diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 80f6b8e9e78e35acba85418e7237d34197541b89..1920d8da173cc0cf24d70137e1f6f3e1ac35ca5f 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -173,7 +173,7 @@ def pre_test_build_mac() { ''' sh ''' cd ${WK}/debug - cmake .. -DBUILD_TEST=true -DBUILD_HTTPS=false + cmake .. -DBUILD_TEST=true -DBUILD_HTTPS=false -DCMAKE_BUILD_TYPE=Release make -j10 ctest -j10 || exit 7 ''' diff --git a/cmake/cmake.define b/cmake/cmake.define index 1b9a06d9e4abe516c68b47e7d7dff20538f2fc13..10f217254152db82518099560ef902b2ebabf499 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -1,6 +1,7 @@ cmake_minimum_required(VERSION 3.0) set(CMAKE_VERBOSE_MAKEFILE OFF) +set(TD_BUILD_TAOSA_INTERNAL FALSE) #set output directory SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib) diff --git a/cmake/cmake.install b/cmake/cmake.install index fd1e080ddab1478f73689e7cced405ae8404fbc2..67634625ce53c6ea48dc03746a39fdb03975c2af 100644 --- a/cmake/cmake.install +++ b/cmake/cmake.install @@ -21,7 +21,7 @@ IF (TD_LINUX) ELSEIF (TD_WINDOWS) SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.bat") INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") - INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} :needAdmin ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Windows ${TD_VER_NUMBER})") + INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} :needAdmin ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Windows ${TD_VER_NUMBER} ${TD_BUILD_TAOSA_INTERNAL})") ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.sh") INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index ab1609f35f15cd625cb2346ffe80ffad7330658e..13b247770ea7eef6b64209ca98787ff6d733bf85 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG 69eee2e + GIT_TAG 213f8b3 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index d01928cfe8f671316a767e3e0191ff87a90f7666..bc3fe878847685022d3490278f0d2880fa3ef3e0 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 5aa25e9 + GIT_TAG f80dd7e SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx b/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx index 7a3ac6bad3b203a9859e235e92d0865f7147d24c..214580c179b97f46f94096411c57bf0004dc68e3 100644 --- a/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx +++ b/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx @@ -47,7 +47,6 @@ Please refer to [OpenTSDB HTTP API](http://opentsdb.net/docs/build/html/api_http :::note - In JSON protocol, strings will be converted to NCHAR type and numeric values will be converted to double type. -- Only data in array format is accepted and so an array must be used even if there is only one row. - The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. ::: diff --git a/docs/en/07-develop/07-tmq.mdx b/docs/en/07-develop/07-tmq.mdx index 17b3f5caa062eaacb4216b7153e899040e702cc1..94a9dbffbd2bc5199edf6f2a6e4561355d967705 100644 --- a/docs/en/07-develop/07-tmq.mdx +++ b/docs/en/07-develop/07-tmq.mdx @@ -117,19 +117,22 @@ class TaosConsumer(): ```go -func NewConsumer(conf *Config) (*Consumer, error) +func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error) -func (c *Consumer) Close() error - -func (c *Consumer) Commit(ctx context.Context, message unsafe.Pointer) error +// rebalanceCb is reserved for compatibility purpose +func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error -func (c *Consumer) FreeMessage(message unsafe.Pointer) +// rebalanceCb is reserved for compatibility purpose +func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error -func (c *Consumer) Poll(timeout time.Duration) (*Result, error) +func (c *Consumer) Poll(timeoutMs int) tmq.Event -func (c *Consumer) Subscribe(topics []string) error +// tmq.TopicPartition is reserved for compatibility purpose +func (c *Consumer) Commit() ([]tmq.TopicPartition, error) func (c *Consumer) Unsubscribe() error + +func (c *Consumer) Close() error ``` @@ -357,50 +360,20 @@ public class MetersDeserializer extends ReferenceDeserializer { ```go -config := tmq.NewConfig() -defer config.Destroy() -err = config.SetGroupID("test") -if err != nil { - panic(err) -} -err = config.SetAutoOffsetReset("earliest") -if err != nil { - panic(err) -} -err = config.SetConnectIP("127.0.0.1") -if err != nil { - panic(err) -} -err = config.SetConnectUser("root") -if err != nil { - panic(err) -} -err = config.SetConnectPass("taosdata") -if err != nil { - panic(err) -} -err = config.SetConnectPort("6030") -if err != nil { - panic(err) -} -err = config.SetMsgWithTableName(true) -if err != nil { - panic(err) -} -err = config.EnableHeartBeat() -if err != nil { - panic(err) -} -err = config.EnableAutoCommit(func(result *wrapper.TMQCommitCallbackResult) { - if result.ErrCode != 0 { - errStr := wrapper.TMQErr2Str(result.ErrCode) - err := errors.NewError(int(result.ErrCode), errStr) - panic(err) - } -}) -if err != nil { - panic(err) +conf := &tmq.ConfigMap{ + "group.id": "test", + "auto.offset.reset": "earliest", + "td.connect.ip": "127.0.0.1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "td.connect.port": "6030", + "client.id": "test_tmq_c", + "enable.auto.commit": "false", + "enable.heartbeat.background": "true", + "experimental.snapshot.enable": "true", + "msg.with.table.name": "true", } +consumer, err := NewConsumer(conf) ``` @@ -523,11 +496,7 @@ consumer.subscribe(topics); ```go -consumer, err := tmq.NewConsumer(config) -if err != nil { - panic(err) -} -err = consumer.Subscribe([]string{"example_tmq_topic"}) +err = consumer.Subscribe("example_tmq_topic", nil) if err != nil { panic(err) } @@ -611,13 +580,17 @@ while(running){ ```go for { - result, err := consumer.Poll(time.Second) - if err != nil { - panic(err) + ev := consumer.Poll(0) + if ev != nil { + switch e := ev.(type) { + case *tmqcommon.DataMessage: + fmt.Println(e.Value()) + case tmqcommon.Error: + fmt.Fprintf(os.Stderr, "%% Error: %v: %v\n", e.Code(), e) + panic(e) + } + consumer.Commit() } - fmt.Println(result) - consumer.Commit(context.Background(), result.Message) - consumer.FreeMessage(result.Message) } ``` @@ -729,7 +702,11 @@ consumer.close(); ```go -consumer.Close() +/* Unsubscribe */ +_ = consumer.Unsubscribe() + +/* Close consumer */ +_ = consumer.Close() ``` diff --git a/docs/en/12-taos-sql/02-database.md b/docs/en/12-taos-sql/02-database.md index a12406fe4353d437c5df9755f8f8e68b0f24282f..476df0a60d593f5215432c08ba4d7560021693e6 100644 --- a/docs/en/12-taos-sql/02-database.md +++ b/docs/en/12-taos-sql/02-database.md @@ -27,10 +27,13 @@ database_option: { | PRECISION {'ms' | 'us' | 'ns'} | REPLICA value | RETENTIONS ingestion_duration:keep_duration ... - | STRICT {'off' | 'on'} | WAL_LEVEL {1 | 2} | VGROUPS value | SINGLE_STABLE {0 | 1} + | STT_TRIGGER value + | TABLE_PREFIX value + | TABLE_SUFFIX value + | TSDB_PAGESIZE value | WAL_RETENTION_PERIOD value | WAL_ROLL_PERIOD value | WAL_RETENTION_SIZE value @@ -61,9 +64,6 @@ database_option: { - PRECISION: specifies the precision at which a database records timestamps. Enter ms for milliseconds, us for microseconds, or ns for nanoseconds. The default value is ms. - REPLICA: specifies the number of replicas that are made of the database. Enter 1 or 3. The default value is 1. The value of the REPLICA parameter cannot exceed the number of dnodes in the cluster. - RETENTIONS: specifies the retention period for data aggregated at various intervals. For example, RETENTIONS 15s:7d,1m:21d,15m:50d indicates that data aggregated every 15 seconds is retained for 7 days, data aggregated every 1 minute is retained for 21 days, and data aggregated every 15 minutes is retained for 50 days. You must enter three aggregation intervals and corresponding retention periods. -- STRICT: specifies whether strong data consistency is enabled. The default value is off. - - on: Strong consistency is enabled and implemented through the Raft consensus algorithm. In this mode, an operation is considered successful once it is confirmed by half of the nodes in the cluster. - - off: Strong consistency is disabled. In this mode, an operation is considered successful when it is initiated by the local node. - WAL_LEVEL: specifies whether fsync is enabled. The default value is 1. - 1: WAL is enabled but fsync is disabled. - 2: WAL and fsync are both enabled. @@ -71,6 +71,10 @@ database_option: { - SINGLE_STABLE: specifies whether the database can contain more than one supertable. - 0: The database can contain multiple supertables. - 1: The database can contain only one supertable. +- STT_TRIGGER: specifies the number of file merges triggered by flushed files. The default is 8, ranging from 1 to 16. For high-frequency scenarios with few tables, it is recommended to use the default configuration or a smaller value for this parameter; For multi-table low-frequency scenarios, it is recommended to configure this parameter with a larger value. +- TABLE_PREFIX：The prefix length in the table name that is ignored when distributing table to vnode based on table name. +- TABLE_SUFFIX：The suffix length in the table name that is ignored when distributing table to vnode based on table name. +- TSDB_PAGESIZE: The page size of the data storage engine in a vnode. The unit is KB. The default is 4 KB. The range is 1 to 16384, that is, 1 KB to 16 MB. - WAL_RETENTION_PERIOD: specifies the time after which WAL files are deleted. This parameter is used for data subscription. Enter a time in seconds. The default value of single copy is 0. A value of 0 indicates that each WAL file is deleted immediately after its contents are written to disk. -1: WAL files are never deleted. The default value of multiple copy is 4 days. - WAL_RETENTION_SIZE: specifies the size at which WAL files are deleted. This parameter is used for data subscription. Enter a size in KB. The default value of single copy is 0. A value of 0 indicates that each WAL file is deleted immediately after its contents are written to disk. -1: WAL files are never deleted. The default value of multiple copy is -1. - WAL_ROLL_PERIOD: specifies the time after which WAL files are rotated. After this period elapses, a new WAL file is created. The default value of single copy is 0. A value of 0 indicates that a new WAL file is created only after the previous WAL file was written to disk. The default values of multiple copy is 1 day. @@ -112,6 +116,10 @@ alter_database_options: alter_database_option: { CACHEMODEL {'none' | 'last_row' | 'last_value' | 'both'} | CACHESIZE value + | BUFFER value + | PAGES value + | REPLICA value + | STT_TRIGGER value | WAL_LEVEL value | WAL_FSYNC_PERIOD value | KEEP value @@ -154,3 +162,19 @@ TRIM DATABASE db_name; ``` The preceding SQL statement deletes data that has expired and orders the remaining data in accordance with the storage configuration. + +## Redistribute Vgroup + +```sql +REDISTRIBUTE VGROUP vgroup_no DNODE dnode_id1 [DNODE dnode_id2] [DNODE dnode_id3] +``` + +Adjust the distribution of vnodes in the vgroup according to the given list of dnodes. + +## Balance Vgroup + +```sql +BALANCE VGROUP +``` + +Automatically adjusts the distribution of vnodes in all vgroups of the cluster, which is equivalent to load balancing the data of the cluster at the vnode level. diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index c087a9e9fb2f0af921aa031d41d124c66fbb0ae7..ee06a7be2d3172210bf35302d5bffbf7a49adabc 100644 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -350,9 +350,9 @@ SELECT AVG(CASE WHEN voltage < 200 or voltage > 250 THEN 220 ELSE voltage END) F ## JOIN -TDengine supports natural joins between supertables, between standard tables, and between subqueries. The difference between natural joins and inner joins is that natural joins require that the fields being joined in the supertables or standard tables must have the same name. Data or tag columns must be joined with the equivalent column in another table. +TDengine supports the `INTER JOIN` based on the timestamp primary key, that is, the `JOIN` condition must contain the timestamp primary key. As long as the requirement of timestamp-based primary key is met, `INTER JOIN` can be made between normal tables, sub-tables, super tables and sub-queries at will, and there is no limit on the number of tables. -For standard tables, only the timestamp (primary key) can be used in join operations. For example: +For standard tables: ```sql SELECT * @@ -360,7 +360,7 @@ FROM temp_tb_1 t1, pressure_tb_1 t2 WHERE t1.ts = t2.ts ``` -For supertables, tags as well as timestamps can be used in join operations. For example: +For supertables: ```sql SELECT * @@ -368,20 +368,15 @@ FROM temp_stable t1, temp_stable t2 WHERE t1.ts = t2.ts AND t1.deviceid = t2.deviceid AND t1.status=0; ``` -Similarly, join operations can be performed on the result sets of multiple subqueries. - -:::note - -The following restriction apply to JOIN statements: +For sub-table and super table： -- The number of tables or supertables in a single join operation cannot exceed 10. -- `FILL` cannot be used in a JOIN statement. -- Arithmetic operations cannot be performed on the result sets of join operation. -- `GROUP BY` is not allowed on a segment of the tables that participate in a join operation. -- `OR` cannot be used in the conditions for join operation -- Join operation can be performed only on tags or timestamps. You cannot perform a join operation on data columns. +```sql +SELECT * +FROM temp_ctable t1, temp_stable t2 +WHERE t1.ts = t2.ts AND t1.deviceid = t2.deviceid AND t1.status=0; +``` -::: +Similarly, join operations can be performed on the result sets of multiple subqueries. ## Nested Query diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index f0daf4b82a2fdc9d519d151451a1bbd10fd93e31..802eb259bf72687f9c75cdb34e3520040d3c9010 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -876,7 +876,8 @@ INTERP(expr) - The number of rows in the result set of `INTERP` is determined by the parameter `EVERY`. Starting from timestamp1, one interpolation is performed for every time interval specified `EVERY` parameter. The parameter `EVERY` must be an integer, with no quotes, with a time unit of: b(nanosecond), u(microsecond), a(millisecond)), s(second), m(minute), h(hour), d(day), or w(week). For example, `EVERY(500a)` will interpolate every 500 milliseconds. - Interpolation is performed based on `FILL` parameter. - `INTERP` can only be used to interpolate in single timeline. So it must be used with `partition by tbname` when it's used on a STable. -- Pseudo column `_irowts` can be used along with `INTERP` to return the timestamps associated with interpolation points(support after version 3.0.1.4). +- Pseudocolumn `_irowts` can be used along with `INTERP` to return the timestamps associated with interpolation points(support after version 3.0.1.4). +- Pseudocolumn `_isfilled` can be used along with `INTERP` to indicate whether the results are original records or data points generated by interpolation algorithm(support after version 3.0.2.3). ### LAST diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index c47d2da0ebf8b8d3ecc2e51fad659ceb423ad46f..e70e962668d85ea6a84afd86657a7b26df90ccf4 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -108,7 +108,7 @@ SHOW STREAMS; When you create a stream, you can use the TRIGGER parameter to specify triggering conditions for it. -For non-windowed processing, triggering occurs in real time. For windowed processing, there are three methods of triggering: +For non-windowed processing, triggering occurs in real time. For windowed processing, there are three methods of triggering，the default value is AT_ONCE: 1. AT_ONCE: triggers on write diff --git a/docs/en/12-taos-sql/20-keywords.md b/docs/en/12-taos-sql/20-keywords.md index 4b479b866b77e1e354d20376ccb869755af76d00..23f85947e30ae2876cf631124e6485c09f2ff8da 100644 --- a/docs/en/12-taos-sql/20-keywords.md +++ b/docs/en/12-taos-sql/20-keywords.md @@ -17,6 +17,7 @@ The following list shows all reserved keywords: - ADD - AFTER - AGGREGATE +- ALIVE - ALL - ALTER - ANALYZE diff --git a/docs/en/12-taos-sql/24-show.md b/docs/en/12-taos-sql/24-show.md index 1bda4a118d796765f2dab6f5fdeae27a58cdd232..2db3e7cb31463e20f024f48e62d06422519ba0e7 100644 --- a/docs/en/12-taos-sql/24-show.md +++ b/docs/en/12-taos-sql/24-show.md @@ -178,6 +178,139 @@ SHOW TABLE DISTRIBUTED table_name; Shows how table data is distributed. +Examples： Below is an example of this command to display the block distribution of table `d0` in detailed format. + +```sql +show table distributed d0\G; +``` + +
+ Show Example + 

+*************************** 1.row ***************************
+_block_dist: Total_Blocks=[5] Total_Size=[93.65 Kb] Average_size=[18.73 Kb] Compression_Ratio=[23.98 %]
+
+Total_Blocks :  Table `d0` contains total 5 blocks
+
+Total_Size:  The total size of all the data blocks in table `d0` is 93.65 KB 
+
+Average_size:  The average size of each block is 18.73 KB
+
+Compression_Ratio: The data compression rate is 23.98%
+ 
+*************************** 2.row ***************************
+_block_dist: Total_Rows=[20000] Inmem_Rows=[0] MinRows=[3616] MaxRows=[4096] Average_Rows=[4000]
+
+Total_Rows: Table `d0` contains 20,000 rows
+
+Inmem_Rows： The rows still in memory, i.e. not committed in disk, is 0, i.e. none such rows
+
+MinRows：  The minimum number of rows in a block is 3,616 
+
+MaxRows： The maximum number of rows in a block is 4,096B
+
+Average_Rows： The average number of rows in a block is 4,000
+
+*************************** 3.row ***************************
+_block_dist: Total_Tables=[1] Total_Files=[2]
+
+Total_Tables:  The number of child tables, 1 in this example
+
+Total_Files：   The number of files storing the table's data, 2 in this example
+
+*************************** 4.row ***************************
+
+_block_dist: --------------------------------------------------------------------------------
+
+*************************** 5.row ***************************
+
+_block_dist: 0100 |
+
+*************************** 6.row ***************************
+
+_block_dist: 0299 |
+
+*************************** 7.row ***************************
+
+_block_dist: 0498 |
+
+*************************** 8.row ***************************
+
+_block_dist: 0697 |
+
+*************************** 9.row ***************************
+
+_block_dist: 0896 |
+
+*************************** 10.row ***************************
+
+_block_dist: 1095 |
+
+*************************** 11.row ***************************
+
+_block_dist: 1294 |
+
+*************************** 12.row ***************************
+
+_block_dist: 1493 |
+
+*************************** 13.row ***************************
+
+_block_dist: 1692 |
+
+*************************** 14.row ***************************
+
+_block_dist: 1891 |
+
+*************************** 15.row ***************************
+
+_block_dist: 2090 |
+
+*************************** 16.row ***************************
+
+_block_dist: 2289 |
+
+*************************** 17.row ***************************
+
+_block_dist: 2488 |
+
+*************************** 18.row ***************************
+
+_block_dist: 2687 |
+
+*************************** 19.row ***************************
+
+_block_dist: 2886 |
+
+*************************** 20.row ***************************
+
+_block_dist: 3085 |
+
+*************************** 21.row ***************************
+
+_block_dist: 3284 |
+
+*************************** 22.row ***************************
+
+_block_dist: 3483 |||||||||||||||||  1 (20.00%)
+
+*************************** 23.row ***************************
+
+_block_dist: 3682 |
+
+*************************** 24.row ***************************
+
+_block_dist: 3881 |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||  4 (80.00%)
+
+Query OK, 24 row(s) in set (0.002444s)
+
+
+
+ + The above show the block distribution percentage according to the number of rows in each block. In the above example, we can get below information: + - `_block_dist: 3483 ||||||||||||||||| 1 (20.00%)` means there is one block whose rows is between 3,483 and 3,681. + - `_block_dist: 3881 ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| 4 (80.00%)` means there are 4 blocks whose rows is between 3,881 and 4,096. - The number of blocks whose rows fall in other range is zero. + ## SHOW TAGS ```sql diff --git a/docs/en/12-taos-sql/29-changes.md b/docs/en/12-taos-sql/29-changes.md index 78b6d5fc05b9b03e1e8b3af268bc357dfaa401bc..f288cd7545c1de25f6cf8b003e084ba3622524a7 100644 --- a/docs/en/12-taos-sql/29-changes.md +++ b/docs/en/12-taos-sql/29-changes.md @@ -54,7 +54,6 @@ The following data types can be used in the schema for standard tables. | 27 | GRANT | Added | Grants permissions to a user. | 28 | KILL TRANSACTION | Added | Terminates an mnode transaction. | 29 | KILL STREAM | Deprecated | Terminated a continuous query. The continuous query feature has been replaced with the stream processing feature. -| 30 | MERGE VGROUP | Added | Merges vgroups. | 31 | REVOKE | Added | Revokes permissions from a user. | 32 | SELECT | Modified | | 33 | SHOW ACCOUNTS | Deprecated | This Enterprise Edition-only statement has been removed. It returns the error "This statement is no longer supported." @@ -76,8 +75,9 @@ The following data types can be used in the schema for standard tables. | 49 | SHOW TRANSACTIONS | Added | Shows all running transactions in the system. | 50 | SHOW DNODE VARIABLES | Added | Shows the configuration of the specified dnode. | 51 | SHOW VNODES | Not supported | Shows information about vnodes in the system. Not supported. -| 52 | SPLIT VGROUP | Added | Splits a vgroup into two vgroups. -| 53 | TRIM DATABASE | Added | Deletes data that has expired and orders the remaining data in accordance with the storage configuration. +| 52 | TRIM DATABASE | Added | Deletes data that has expired and orders the remaining data in accordance with the storage configuration. +| 53 | REDISTRIBUTE VGROUP | Added | Adjust the distribution of VNODES in VGROUP. +| 54 | BALANCE VGROUP | Added | Auto adjust the distribution of VNODES in VGROUP. ## SQL Functions diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx index 7b3440ebac2f630a2f6f39c3b270a2196d1d7739..c37738b3f8ddba9879159e04573798dd61efa27e 100644 --- a/docs/en/14-reference/03-connector/04-java.mdx +++ b/docs/en/14-reference/03-connector/04-java.mdx @@ -878,8 +878,10 @@ The source code of the sample application is under `TDengine/examples/JDBC`: | taos-jdbcdriver version | major changes | | :---------------------: | :--------------------------------------------: | +| 3.0.3 | fix timestamp resolution error for REST connection in jdk17+ version | | 3.0.1 - 3.0.2 | fix the resultSet data is parsed incorrectly sometimes. 3.0.1 is compiled on JDK 11, you are advised to use 3.0.2 in the JDK 8 environment | | 3.0.0 | Support for TDengine 3.0 | +| 2.0.42 | fix wasNull interface return value in WebSocket connection | | 2.0.41 | fix decode method of username and password in REST connection | | 2.0.39 - 2.0.40 | Add REST connection/request timeout parameters | | 2.0.38 | JDBC REST connections add bulk pull function | diff --git a/docs/en/14-reference/03-connector/05-go.mdx b/docs/en/14-reference/03-connector/05-go.mdx index df5b129cea552144d5833190d46e8a78f2fd2fa5..60407c0735bf9bcb42ae54bddcc9afa639a02fcc 100644 --- a/docs/en/14-reference/03-connector/05-go.mdx +++ b/docs/en/14-reference/03-connector/05-go.mdx @@ -355,26 +355,29 @@ The `af` package encapsulates TDengine advanced functions such as connection man #### Subscribe -* `func NewConsumer(conf *Config) (*Consumer, error)` +* `func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error)` Creates consumer group. -* `func (c *Consumer) Subscribe(topics []string) error` +* `func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error` +Note: `rebalanceCb` is reserved for compatibility purpose + +Subscribes a topic. + +* `func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error` +Note: `rebalanceCb` is reserved for compatibility purpose Subscribes to topics. -* `func (c *Consumer) Poll(timeout time.Duration) (*Result, error)` +* `func (c *Consumer) Poll(timeoutMs int) tmq.Event` Polling information. -* `func (c *Consumer) Commit(ctx context.Context, message unsafe.Pointer) error` +* `func (c *Consumer) Commit() ([]tmq.TopicPartition, error)` +Note: `tmq.TopicPartition` is reserved for compatibility purpose Commit information. -* `func (c *Consumer) FreeMessage(message unsafe.Pointer)` - -Free information. - * `func (c *Consumer) Unsubscribe() error` Unsubscribe. @@ -441,25 +444,36 @@ Close consumer. ### Subscribe via WebSocket -* `func NewConsumer(config *Config) (*Consumer, error)` +* `func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error)` - Creates consumer group. +Creates consumer group. + +* `func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error` +Note: `rebalanceCb` is reserved for compatibility purpose -* `func (c *Consumer) Subscribe(topic []string) error` +Subscribes a topic. - Subscribes to topics. +* `func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error` +Note: `rebalanceCb` is reserved for compatibility purpose -* `func (c *Consumer) Poll(timeout time.Duration) (*Result, error)` +Subscribes to topics. - Polling information. +* `func (c *Consumer) Poll(timeoutMs int) tmq.Event` -* `func (c *Consumer) Commit(messageID uint64) error` +Polling information. - Commit information. +* `func (c *Consumer) Commit() ([]tmq.TopicPartition, error)` +Note: `tmq.TopicPartition` is reserved for compatibility purpose + +Commit information. + +* `func (c *Consumer) Unsubscribe() error` + +Unsubscribe. * `func (c *Consumer) Close() error` - Close consumer. +Close consumer. For a complete example see [GitHub sample file](https://github.com/taosdata/driver-go/blob/3.0/examples/tmqoverws/main.go) diff --git a/docs/en/14-reference/03-connector/09-csharp.mdx b/docs/en/14-reference/03-connector/09-csharp.mdx index 85514f58ac1a19c7ae1a725e9b055f10280ebbb6..b73e1c262758df77d5b84c692f7098dacac70c80 100644 --- a/docs/en/14-reference/03-connector/09-csharp.mdx +++ b/docs/en/14-reference/03-connector/09-csharp.mdx @@ -252,14 +252,14 @@ ws://localhost:6041/test |Sample program |Sample program description | |--------------------------------------------------------------------------------------------------------------------|--------------------------------------------| -| [CURD](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/Query/Query.cs) | Table creation, data insertion, and query examples with TDengine.Connector | -| [JSON Tag](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/JSONTag) | Writing and querying JSON tag data with TDengine Connector | -| [stmt](https://github.com/taosdata/taos-connector-dotnet/tree/3.0/examples/Stmt) | Parameter binding with TDengine Connector | -| [schemaless](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/schemaless) | Schemaless writes with TDengine Connector | -| [async query](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/AsyncQuery/QueryAsync.cs) | Asynchronous queries with TDengine Connector | -| [Subscription](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/TMQ/TMQ.cs) | Subscription example with TDengine Connector | -| [Basic WebSocket Usage](https://github.com/taosdata/taos-connector-dotnet/blob/5a4a7cd0dbcda114447cdc6d0c6dedd8e84a52da/examples/WS/WebSocketSample.cs) | WebSocket basic data in and out with TDengine connector | -| [WebSocket Parameter Binding](https://github.com/taosdata/taos-connector-dotnet/blob/5a4a7cd0dbcda114447cdc6d0c6dedd8e84a52da/examples/WS/WebSocketSTMT.cs) | WebSocket parameter binding example | +| [CURD](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/Query/Query.cs) | Table creation, data insertion, and query examples with TDengine.Connector | +| [JSON Tag](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/JSONTag) | Writing and querying JSON tag data with TDengine Connector | +| [stmt](https://github.com/taosdata/taos-connector-dotnet/tree/3.0/examples/NET6Examples/Stmt) | Parameter binding with TDengine Connector | +| [schemaless](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/schemaless) | Schemaless writes with TDengine Connector | +| [async query](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/AsyncQuery/QueryAsync.cs) | Asynchronous queries with TDengine Connector | +| [Subscription](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/TMQ/TMQ.cs) | Subscription example with TDengine Connector | +| [Basic WebSocket Usage](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/FrameWork45/WS/WebSocketSample.cs) | WebSocket basic data in and out with TDengine connector | +| [WebSocket Parameter Binding](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/FrameWork45/WS/WebSocketSTMT.cs) | WebSocket parameter binding example | ## Important update records diff --git a/docs/en/14-reference/04-taosadapter.md b/docs/en/14-reference/04-taosadapter.md index ad00584360acc18dca46998948fd96790965e5b1..9eb6cb921391df3ce494fddaae8caa23cb0f4adc 100644 --- a/docs/en/14-reference/04-taosadapter.md +++ b/docs/en/14-reference/04-taosadapter.md @@ -21,6 +21,7 @@ taosAdapter provides the following features. - Seamless connection to collectd - Seamless connection to StatsD - Supports Prometheus remote_read and remote_write +- Get table's VGroup ID ## taosAdapter architecture diagram @@ -59,6 +60,7 @@ Usage of taosAdapter: --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) + --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" (default 0, means no ttl) -c, --config string config path default /etc/taos/taosadapter.toml --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" @@ -100,6 +102,7 @@ Usage of taosAdapter: --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") + --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL"(default 0, means no ttl) --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) @@ -110,6 +113,7 @@ Usage of taosAdapter: --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") + --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL"(default 0, means no ttl) --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" (default 1h0m0s) --pool.maxConnect int max connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" (default 4000) --pool.maxIdle int max idle connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_IDLE" (default 4000) @@ -131,6 +135,7 @@ Usage of taosAdapter: --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) + --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" (default 0, means no ttl) --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" --version Print the version and exit ``` @@ -174,6 +179,7 @@ See [example/config/taosadapter.toml](https://github.com/taosdata/taosadapter/bl node_export is an exporter for machine metrics. Please visit [https://github.com/prometheus/node_exporter](https://github.com/prometheus/node_exporter) for more information. - Support for Prometheus remote_read and remote_write remote_read and remote_write are interfaces for Prometheus data read and write from/to other data storage solution. Please visit [https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis) for more information. +- Get table's VGroup ID. For more information about VGroup, please refer to [primary-logic-unit](/tdinternal/arch/#primary-logic-unit). ## Interfaces @@ -195,6 +201,7 @@ Support InfluxDB query parameters as follows. - `precision` The time precision used by TDengine - `u` TDengine user name - `p` TDengine password +- `ttl` The time to live of automatically created sub-table. This value cannot be updated. TDengine will use the ttl value of the first data of sub-table to create sub-table. For more information, please refer [Create Table](/taos-sql/table/#create-table) Note: InfluxDB token authorization is not supported at present. Only Basic authorization and query parameter validation are supported. Example: curl --request POST http://127.0.0.1:6041/influxdb/v1/write?db=test --user "root:taosdata" --data-binary "measurement,host=host1 field1=2i,field2=2.0 1577836800000000000" @@ -236,6 +243,10 @@ node_export is an exporter of hardware and OS metrics exposed by the \*NIX kerne +### Get table's VGroup ID + +You can call `http://:6041/rest/vgid?db=&table=` to get table's VGroup ID. For more information about VGroup, please refer to [primary-logic-unit](/tdinternal/arch/#primary-logic-unit). + ## Memory usage optimization methods taosAdapter will monitor its memory usage during operation and adjust it with two thresholds. Valid values are integers between 1 to 100, and represent a percentage of the system's physical memory. diff --git a/docs/en/14-reference/05-taosbenchmark.md b/docs/en/14-reference/05-taosbenchmark.md index 6e08671e344100aa3dcfeba5a10c0d0e304a1a91..4017b12be923a9bcb5696c8b4b57c2d67b5c1378 100644 --- a/docs/en/14-reference/05-taosbenchmark.md +++ b/docs/en/14-reference/05-taosbenchmark.md @@ -92,7 +92,7 @@ taosBenchmark -f -## Command-line argument in detailed +## Command-line argument in detail - **-f/--file ** : specify the configuration file to use. This file includes All parameters. Users should not use this parameter with other parameters on the command-line. There is no default value. @@ -198,19 +198,25 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) - **-R/--disorder-range ** : Specify the timestamp range for the disordered data. It leads the resulting disorder timestamp as the ordered timestamp minus a random value in this range. Valid only if the percentage of disordered data specified by `-O/--disorder` is greater than 0. -- **-F/--prepare_rand ** : +- **-F/--prepared_rand ** : Specify the number of unique values in the generated random data. A value of 1 means that all data are equal. The default value is 10000. - **-a/--replica ** : Specify the number of replicas when creating the database. The default value is 1. +- **-k/--keep-trying ** : + Keep trying if failed to insert, default is no. Available with v3.0.9+. + +- **-z/--trying-interval ** : + Specify interval between keep trying insert. Valid value is a postive number. Only valid when keep trying be enabled. Available with v3.0.9+. + - **-V/--version** : Show version information only. Users should not use it with other parameters. - **-? /--help** : Show help information and exit. Users should not use it with other parameters. -## Configuration file parameters in detailed +## Configuration file parameters in detail ### General configuration parameters @@ -231,6 +237,10 @@ The parameters listed in this section apply to all function modes. `filetype` must be set to `insert` in the insertion scenario. See [General Configuration Parameters](#General Configuration Parameters) +- ** keep_trying ** : Keep trying if failed to insert, default is no. Available with v3.0.9+. + +- ** trying_interval ** : Specify interval between keep trying insert. Valid value is a postive number. Only valid when keep trying be enabled. Available with v3.0.9+. + #### Database related configuration parameters The parameters related to database creation are configured in `dbinfo` in the json configuration file, as follows. The other parameters correspond to the database parameters specified when `create database` in [../../taos-sql/database]. @@ -370,7 +380,7 @@ The configuration parameters for specifying super table tag columns and data col - **num_of_records_per_req** : Writing the number of rows of records per request to TDengine, the default value is 30000. When it is set too large, the TDengine client driver will return the corresponding error message, so you need to lower the setting of this parameter to meet the writing requirements. -- **prepare_rand**: The number of unique values in the generated random data. A value of 1 means that all data are equal. The default value is 10000. +- **prepared_rand**: The number of unique values in the generated random data. A value of 1 means that all data are equal. The default value is 10000. ### Query scenario configuration parameters diff --git a/docs/en/14-reference/06-taosdump.md b/docs/en/14-reference/06-taosdump.md index e73441a96b087062b2e3912ed73010fc3e761bb9..9c63b4dc035b6a6a18f8e17f19782527eef88bca 100644 --- a/docs/en/14-reference/06-taosdump.md +++ b/docs/en/14-reference/06-taosdump.md @@ -19,7 +19,7 @@ Users should not use taosdump to back up raw data, environment settings, hardwar There are two ways to install taosdump: -- Install the taosTools official installer. Please find taosTools from [All download links](https://www.tdengine.com/all-downloads) page and download and install it. +- Install the taosTools official installer. Please find taosTools from [Release History](https://docs.taosdata.com/releases/tools/) page and download and install it. - Compile taos-tools separately and install it. Please refer to the [taos-tools](https://github.com/taosdata/taos-tools) repository for details. diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 274c99509836702dc067ca7f3357acc96be4132c..c3d5de02137b61796cb9361721bf51fa6ac9dfb2 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -142,6 +142,15 @@ The parameters described in this document by the effect that they have on the sy | Meaning | Switch for allowing TDengine to collect and report service usage information | | Value Range | 0: Not allowed; 1: Allowed | | Default Value | 1 | +### crashReporting + +| Attribute | Description | +| -------- | -------------------------------------------- | +| Applicable | Server Only | +| Meaning |Switch for allowing TDengine to collect and report crash related information | +| Value Range | 0,1 0: Not allowed；1：allowed | +| Default Value | 1 | + ## Query Parameters @@ -153,11 +162,11 @@ The parameters described in this document by the effect that they have on the sy | Meaning | Execution policy for query statements | | Unit | None | | Default | 1 | -| Notes | 1: Run queries on vnodes and not on qnodes | +| Value Range | 1: Run queries on vnodes and not on qnodes 2: Run subtasks without scan operators on qnodes and subtasks with scan operators on vnodes. -3: Only run scan operators on vnodes; run all other operators on qnodes. +3: Only run scan operators on vnodes; run all other operators on qnodes. | ### querySmaOptimize @@ -173,6 +182,14 @@ The parameters described in this document by the effect that they have on the sy 1: Enable SMA indexing and perform queries from suitable statements on precomputation results.| +### countAlwaysReturnValue + +| Attribute | Description | +| -------- | -------------------------------- | +| Applicable | Server only | +| Meaning | count()/hyperloglog() return value or not if the result data is NULL | +| Vlue Range | 0：Return empty line，1：Return 0 | +| Default | 1 | ### maxNumOfDistinctRes @@ -307,6 +324,14 @@ The charset that takes effect is UTF-8. | Meaning | All data files are stored in this directory | | Default Value | /var/lib/taos | +### tempDir + +| Attribute | Description | +| -------- | ------------------------------------------ | +| Applicable | Server only | +| Meaning | The directory where to put all the temporary files generated during system running | +| Default | /tmp | + ### minimalTmpDirGB | Attribute | Description | @@ -336,89 +361,6 @@ The charset that takes effect is UTF-8. | Value Range | 0-4096 | | Default Value | 2x the CPU cores | -## Time Parameters - -### statusInterval - -| Attribute | Description | -| -------- | --------------------------- | -| Applicable | Server Only | -| Meaning | the interval of dnode reporting status to mnode | -| Unit | second | -| Value Range | 1-10 | -| Default Value | 1 | - -### shellActivityTimer - -| Attribute | Description | -| -------- | --------------------------------- | -| Applicable | Server and Client | -| Meaning | The interval for TDengine CLI to send heartbeat to mnode | -| Unit | second | -| Value Range | 1-120 | -| Default Value | 3 | - -## Performance Optimization Parameters - -### numOfCommitThreads - -| Attribute | Description | -| -------- | ---------------------- | -| Applicable | Server Only | -| Meaning | Maximum of threads for committing to disk | -| Default Value | | - -## Compression Parameters - -### compressMsgSize - -| Attribute | Description | -| ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Applicable | Server Only | -| Meaning | The threshold for message size to compress the message. | Set the value to 64330 bytes for good message compression. | -| Unit | bytes | -| Value Range | 0: already compress; >0: compress when message exceeds it; -1: always uncompress | -| Default Value | -1 | - -### compressColData - -| Attribute | Description | -| -------- | --------------------------------------------------------------------------------------- | -| Applicable | Server Only | -| Meaning | The threshold for size of column data to trigger compression for the query result | -| Unit | bytes | -| Value Range | 0: always compress; >0: only compress when the size of any column data exceeds the threshold; -1: always uncompress | -| Default Value | -1 | -| Default Value | -1 | -| Note | available from version 2.3.0.0 | | - -## Continuous Query Parameters | - -### minSlidingTime - -| Attribute | Description | -| ------------- | -------------------------------------------------------- | -| Applicable | Server Only | -| Meaning | Minimum sliding time of time window | -| Unit | millisecond or microsecond , depending on time precision | -| Value Range | 10-1000000 | -| Default Value | 10 | - -### minIntervalTime - -| Attribute | Description | -| ------------- | --------------------------- | -| Applicable | Server Only | -| Meaning | Minimum size of time window | -| Unit | millisecond | -| Value Range | 1-1000000 | -| Default Value | 10 | - -:::info -To prevent system resource from being exhausted by multiple concurrent streams, a random delay is applied on each stream automatically. `maxFirstStreamCompDelay` is the maximum delay time before a continuous query is started the first time. `streamCompDelayRatio` is the ratio for calculating delay time, with the size of the time window as base. `maxStreamCompDelay` is the maximum delay time. The actual delay time is a random time not bigger than `maxStreamCompDelay`. If a continuous query fails, `retryStreamComDelay` is the delay time before retrying it, also not bigger than `maxStreamCompDelay`. - -::: - ## Log Parameters ### logDir @@ -665,6 +607,18 @@ To prevent system resource from being exhausted by multiple concurrent streams, | Value Range | 0: not consistent; 1: consistent. | | Default | 1 | +## Compress Parameters + +### compressMsgSize + +| Attribute | Description | +| -------- | ----------------------------- | +| Applicable | Both Client and Server side | +| Meaning | Whether RPC message is compressed | +| Value Range | -1: none message is compressed; 0: all messages are compressed; N (N>0): messages exceeding N bytes are compressed | +| Default | -1 | + + ## Other Parameters ### enableCoreFile @@ -686,172 +640,60 @@ To prevent system resource from being exhausted by multiple concurrent streams, | Value Range | 0: disable UDF; 1: enabled UDF | | Default Value | 1 | -## Parameter Comparison of TDengine 2.x and 3.0 -| # | **Parameter** | **In 2.x** | **In 3.0** | -| --- | :-----------------: | --------------- | --------------- | -| 1 | firstEp | Yes | Yes | -| 2 | secondEp | Yes | Yes | -| 3 | fqdn | Yes | Yes | -| 4 | serverPort | Yes | Yes | -| 5 | maxShellConns | Yes | Yes | -| 6 | monitor | Yes | Yes | -| 7 | monitorFqdn | No | Yes | -| 8 | monitorPort | No | Yes | -| 9 | monitorInterval | Yes | Yes | -| 10 | monitorMaxLogs | No | Yes | -| 11 | monitorComp | No | Yes | -| 12 | telemetryReporting | Yes | Yes | -| 13 | telemetryInterval | No | Yes | -| 14 | telemetryServer | No | Yes | -| 15 | telemetryPort | No | Yes | -| 16 | queryPolicy | No | Yes | -| 17 | querySmaOptimize | No | Yes | -| 18 | queryRsmaTolerance | No | Yes | -| 19 | queryBufferSize | Yes | Yes | -| 20 | maxNumOfDistinctRes | Yes | Yes | -| 21 | minSlidingTime | Yes | Yes | -| 22 | minIntervalTime | Yes | Yes | -| 23 | countAlwaysReturnValue | Yes | Yes | -| 24 | dataDir | Yes | Yes | -| 25 | minimalDataDirGB | Yes | Yes | -| 26 | supportVnodes | No | Yes | -| 27 | tempDir | Yes | Yes | -| 28 | minimalTmpDirGB | Yes | Yes | -| 29 | compressMsgSize | Yes | Yes | -| 30 | compressColData | Yes | Yes | -| 31 | smlChildTableName | Yes | Yes | -| 32 | smlTagName | Yes | Yes | -| 33 | smlDataFormat | No | Yes | -| 34 | statusInterval | Yes | Yes | -| 35 | shellActivityTimer | Yes | Yes | -| 36 | transPullupInterval | No | Yes | -| 37 | mqRebalanceInterval | No | Yes | -| 38 | ttlUnit | No | Yes | -| 39 | ttlPushInterval | No | Yes | -| 40 | numOfTaskQueueThreads | No | Yes | -| 41 | numOfRpcThreads | No | Yes | -| 42 | numOfCommitThreads | Yes | Yes | -| 43 | numOfMnodeReadThreads | No | Yes | -| 44 | numOfVnodeQueryThreads | No | Yes | -| 45 | ratioOfVnodeStreamThreads | No | Yes | -| 46 | numOfVnodeFetchThreads | No | Yes | -| 47 | numOfVnodeRsmaThreads | No | Yes | -| 48 | numOfQnodeQueryThreads | No | Yes | -| 49 | numOfQnodeFetchThreads | No | Yes | -| 50 | numOfSnodeSharedThreads | No | Yes | -| 51 | numOfSnodeUniqueThreads | No | Yes | -| 52 | rpcQueueMemoryAllowed | No | Yes | -| 53 | logDir | Yes | Yes | -| 54 | minimalLogDirGB | Yes | Yes | -| 55 | numOfLogLines | Yes | Yes | -| 56 | asyncLog | Yes | Yes | -| 57 | logKeepDays | Yes | Yes | -| 60 | debugFlag | Yes | Yes | -| 61 | tmrDebugFlag | Yes | Yes | -| 62 | uDebugFlag | Yes | Yes | -| 63 | rpcDebugFlag | Yes | Yes | -| 64 | jniDebugFlag | Yes | Yes | -| 65 | qDebugFlag | Yes | Yes | -| 66 | cDebugFlag | Yes | Yes | -| 67 | dDebugFlag | Yes | Yes | -| 68 | vDebugFlag | Yes | Yes | -| 69 | mDebugFlag | Yes | Yes | -| 70 | wDebugFlag | Yes | Yes | -| 71 | sDebugFlag | Yes | Yes | -| 72 | tsdbDebugFlag | Yes | Yes | -| 73 | tqDebugFlag | No | Yes | -| 74 | fsDebugFlag | Yes | Yes | -| 75 | udfDebugFlag | No | Yes | -| 76 | smaDebugFlag | No | Yes | -| 77 | idxDebugFlag | No | Yes | -| 78 | tdbDebugFlag | No | Yes | -| 79 | metaDebugFlag | No | Yes | -| 80 | timezone | Yes | Yes | -| 81 | locale | Yes | Yes | -| 82 | charset | Yes | Yes | -| 83 | udf | Yes | Yes | -| 84 | enableCoreFile | Yes | Yes | -| 85 | arbitrator | Yes | No | -| 86 | numOfThreadsPerCore | Yes | No | -| 87 | numOfMnodes | Yes | No | -| 88 | vnodeBak | Yes | No | -| 89 | balance | Yes | No | -| 90 | balanceInterval | Yes | No | -| 91 | offlineThreshold | Yes | No | -| 92 | role | Yes | No | -| 93 | dnodeNopLoop | Yes | No | -| 94 | keepTimeOffset | Yes | No | -| 95 | rpcTimer | Yes | No | -| 96 | rpcMaxTime | Yes | No | -| 97 | rpcForceTcp | Yes | No | -| 98 | tcpConnTimeout | Yes | No | -| 99 | syncCheckInterval | Yes | No | -| 100 | maxTmrCtrl | Yes | No | -| 101 | monitorReplica | Yes | No | -| 102 | smlTagNullName | Yes | No | -| 103 | keepColumnName | Yes | No | -| 104 | ratioOfQueryCores | Yes | No | -| 105 | maxStreamCompDelay | Yes | No | -| 106 | maxFirstStreamCompDelay | Yes | No | -| 107 | retryStreamCompDelay | Yes | No | -| 108 | streamCompDelayRatio | Yes | No | -| 109 | maxVgroupsPerDb | Yes | No | -| 110 | maxTablesPerVnode | Yes | No | -| 111 | minTablesPerVnode | Yes | No | -| 112 | tableIncStepPerVnode | Yes | No | -| 113 | cache | Yes | No | -| 114 | blocks | Yes | No | -| 115 | days | Yes | No | -| 116 | keep | Yes | No | -| 117 | minRows | Yes | No | -| 118 | maxRows | Yes | No | -| 119 | quorum | Yes | No | -| 120 | comp | Yes | No | -| 121 | walLevel | Yes | No | -| 122 | fsync | Yes | No | -| 123 | replica | Yes | No | -| 124 | partitions | Yes | No | -| 125 | quorum | Yes | No | -| 126 | update | Yes | No | -| 127 | cachelast | Yes | No | -| 128 | maxSQLLength | Yes | No | -| 129 | maxWildCardsLength | Yes | No | -| 130 | maxRegexStringLen | Yes | No | -| 131 | maxNumOfOrderedRes | Yes | No | -| 132 | maxConnections | Yes | No | -| 133 | mnodeEqualVnodeNum | Yes | No | -| 134 | http | Yes | No | -| 135 | httpEnableRecordSql | Yes | No | -| 136 | httpMaxThreads | Yes | No | -| 137 | restfulRowLimit | Yes | No | -| 138 | httpDbNameMandatory | Yes | No | -| 139 | httpKeepAlive | Yes | No | -| 140 | enableRecordSql | Yes | No | -| 141 | maxBinaryDisplayWidth | Yes | No | -| 142 | stream | Yes | No | -| 143 | retrieveBlockingModel | Yes | No | -| 144 | tsdbMetaCompactRatio | Yes | No | -| 145 | defaultJSONStrType | Yes | No | -| 146 | walFlushSize | Yes | No | -| 147 | keepTimeOffset | Yes | No | -| 148 | flowctrl | Yes | No | -| 149 | slaveQuery | Yes | No | -| 150 | adjustMaster | Yes | No | -| 151 | topicBinaryLen | Yes | No | -| 152 | telegrafUseFieldNum | Yes | No | -| 153 | deadLockKillQuery | Yes | No | -| 154 | clientMerge | Yes | No | -| 155 | sdbDebugFlag | Yes | No | -| 156 | odbcDebugFlag | Yes | No | -| 157 | httpDebugFlag | Yes | No | -| 158 | monDebugFlag | Yes | No | -| 159 | cqDebugFlag | Yes | No | -| 160 | shortcutFlag | Yes | No | -| 161 | probeSeconds | Yes | No | -| 162 | probeKillSeconds | Yes | No | -| 163 | probeInterval | Yes | No | -| 164 | lossyColumns | Yes | No | -| 165 | fPrecision | Yes | No | -| 166 | dPrecision | Yes | No | -| 167 | maxRange | Yes | No | -| 168 | range | Yes | No | + +## 3.0 Parameters + +| # | **参数** | **Applicable to 2.x ** | **Applicable to 3.0 ** | Current behavior in 3.0 | +| --- | :---------------------: | --------------- | --------------- | ------------------------------------------------- | +| 1 | firstEp | Yes | Yes | | +| 2 | secondEp | Yes | Yes | | +| 3 | fqdn | Yes | Yes | | +| 4 | serverPort | Yes | Yes | | +| 5 | maxShellConns | Yes | Yes | | +| 6 | monitor | Yes | Yes | | +| 7 | monitorFqdn | No | Yes | | +| 8 | monitorPort | No | Yes | | +| 9 | monitorInterval | Yes | Yes | | +| 10 | queryPolicy | No | Yes | | +| 11 | querySmaOptimize | No | Yes | | +| 12 | maxNumOfDistinctRes | Yes | Yes | | +| 15 | countAlwaysReturnValue | Yes | Yes | | +| 16 | dataDir | Yes | Yes | | +| 17 | minimalDataDirGB | Yes | Yes | | +| 18 | supportVnodes | No | Yes | | +| 19 | tempDir | Yes | Yes | | +| 20 | minimalTmpDirGB | Yes | Yes | | +| 21 | smlChildTableName | Yes | Yes | | +| 22 | smlTagName | Yes | Yes | | +| 23 | smlDataFormat | No | Yes | | +| 24 | statusInterval | Yes | Yes | | +| 25 | logDir | Yes | Yes | | +| 26 | minimalLogDirGB | Yes | Yes | | +| 27 | numOfLogLines | Yes | Yes | | +| 28 | asyncLog | Yes | Yes | | +| 29 | logKeepDays | Yes | Yes | | +| 30 | debugFlag | Yes | Yes | | +| 31 | tmrDebugFlag | Yes | Yes | | +| 32 | uDebugFlag | Yes | Yes | | +| 33 | rpcDebugFlag | Yes | Yes | | +| 34 | jniDebugFlag | Yes | Yes | | +| 35 | qDebugFlag | Yes | Yes | | +| 36 | cDebugFlag | Yes | Yes | | +| 37 | dDebugFlag | Yes | Yes | | +| 38 | vDebugFlag | Yes | Yes | | +| 39 | mDebugFlag | Yes | Yes | | +| 40 | wDebugFlag | Yes | Yes | | +| 41 | sDebugFlag | Yes | Yes | | +| 42 | tsdbDebugFlag | Yes | Yes | | +| 43 | tqDebugFlag | No | Yes | | +| 44 | fsDebugFlag | Yes | Yes | | +| 45 | udfDebugFlag | No | Yes | | +| 46 | smaDebugFlag | No | Yes | | +| 47 | idxDebugFlag | No | Yes | | +| 48 | tdbDebugFlag | No | Yes | | +| 49 | metaDebugFlag | No | Yes | | +| 50 | timezone | Yes | Yes | | +| 51 | locale | Yes | Yes | | +| 52 | charset | Yes | Yes | | +| 53 | udf | Yes | Yes | | +| 54 | enableCoreFile | Yes | Yes | | diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index 32bdc21e7c0552f5f891b38f806e48efb6c419ac..4e0239eb80974647ac5ab5e2f3e14bedb831859e 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -10,6 +10,22 @@ For TDengine 2.x installation packages by version, please visit [here](https://w import Release from "/components/ReleaseV3"; +## 3.0.2.3 + + + +## 3.0.2.2 + + + +## 3.0.2.1 + + + +## 3.0.2.0 + + + ## 3.0.1.8 diff --git a/docs/en/28-releases/02-tools.md b/docs/en/28-releases/02-tools.md index 7126b5a997043231d1cf93d633b8cd71e5f6275e..2d20c26c56237fd1834167f0bdf6fb2b42ffba33 100644 --- a/docs/en/28-releases/02-tools.md +++ b/docs/en/28-releases/02-tools.md @@ -10,6 +10,22 @@ For other historical version installers, please visit [here](https://www.taosdat import Release from "/components/ReleaseV3"; +## 2.4.1 + + + +## 2.4.0 + + + +## 2.3.3 + + + +## 2.3.2 + + + ## 2.3.0 diff --git a/docs/examples/csharp/stmtInsert/Program.cs b/docs/examples/csharp/stmtInsert/Program.cs index 87e1971feb8499c515206f05a1e916070ac57f4c..80cadb2ff8b596a0484d05ff15aeaa50f22ff859 100644 --- a/docs/examples/csharp/stmtInsert/Program.cs +++ b/docs/examples/csharp/stmtInsert/Program.cs @@ -42,7 +42,7 @@ namespace TDengineExample // 5. execute res = TDengine.StmtExecute(stmt); - CheckStmtRes(res, "faild to execute"); + CheckStmtRes(res, "failed to execute"); // 6. free TaosMultiBind.FreeTaosBind(tags); @@ -92,7 +92,7 @@ namespace TDengineExample int code = TDengine.StmtClose(stmt); if (code != 0) { - throw new Exception($"falied to close stmt, {code} reason: {TDengine.StmtErrorStr(stmt)} "); + throw new Exception($"failed to close stmt, {code} reason: {TDengine.StmtErrorStr(stmt)} "); } } } diff --git a/docs/examples/go/go.mod b/docs/examples/go/go.mod index 2bc1a74cb6ef14221fa384701773dc73fe3b161d..6696852312f39b93df189beba89f08c1945ef9f1 100644 --- a/docs/examples/go/go.mod +++ b/docs/examples/go/go.mod @@ -2,5 +2,5 @@ module goexample go 1.17 -require github.com/taosdata/driver-go/v3 3.0 +require github.com/taosdata/driver-go/v3 3.1.0 diff --git a/docs/examples/go/sub/main.go b/docs/examples/go/sub/main.go index 7721eed1346f254e104bb41a2135171ebd518e2c..1f7218936fbe457615562ded1b938daca95225cb 100644 --- a/docs/examples/go/sub/main.go +++ b/docs/examples/go/sub/main.go @@ -1,17 +1,12 @@ package main import ( - "context" - "encoding/json" "fmt" - "strconv" - "time" + "os" "github.com/taosdata/driver-go/v3/af" "github.com/taosdata/driver-go/v3/af/tmq" - "github.com/taosdata/driver-go/v3/common" - "github.com/taosdata/driver-go/v3/errors" - "github.com/taosdata/driver-go/v3/wrapper" + tmqcommon "github.com/taosdata/driver-go/v3/common/tmq" ) func main() { @@ -24,97 +19,60 @@ func main() { if err != nil { panic(err) } - _, err = db.Exec("create topic if not exists example_tmq_topic with meta as DATABASE example_tmq") + _, err = db.Exec("create topic if not exists example_tmq_topic as DATABASE example_tmq") if err != nil { panic(err) } - config := tmq.NewConfig() - defer config.Destroy() - err = config.SetGroupID("test") if err != nil { panic(err) } - err = config.SetAutoOffsetReset("earliest") - if err != nil { - panic(err) - } - err = config.SetConnectIP("127.0.0.1") - if err != nil { - panic(err) - } - err = config.SetConnectUser("root") - if err != nil { - panic(err) - } - err = config.SetConnectPass("taosdata") + consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{ + "group.id": "test", + "auto.offset.reset": "earliest", + "td.connect.ip": "127.0.0.1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "td.connect.port": "6030", + "client.id": "test_tmq_client", + "enable.auto.commit": "false", + "enable.heartbeat.background": "true", + "experimental.snapshot.enable": "true", + "msg.with.table.name": "true", + }) if err != nil { panic(err) } - err = config.SetConnectPort("6030") + err = consumer.Subscribe("example_tmq_topic", nil) if err != nil { panic(err) } - err = config.SetMsgWithTableName(true) + _, err = db.Exec("create table example_tmq.t1 (ts timestamp,v int)") if err != nil { panic(err) } - err = config.EnableHeartBeat() + _, err = db.Exec("insert into example_tmq.t1 values(now,1)") if err != nil { panic(err) } - err = config.EnableAutoCommit(func(result *wrapper.TMQCommitCallbackResult) { - if result.ErrCode != 0 { - errStr := wrapper.TMQErr2Str(result.ErrCode) - err := errors.NewError(int(result.ErrCode), errStr) - panic(err) + for i := 0; i < 5; i++ { + ev := consumer.Poll(0) + if ev != nil { + switch e := ev.(type) { + case *tmqcommon.DataMessage: + fmt.Println(e.String()) + case tmqcommon.Error: + fmt.Fprintf(os.Stderr, "%% Error: %v: %v\n", e.Code(), e) + panic(e) + } + consumer.Commit() } - }) - if err != nil { - panic(err) } - consumer, err := tmq.NewConsumer(config) + err = consumer.Unsubscribe() if err != nil { panic(err) } - err = consumer.Subscribe([]string{"example_tmq_topic"}) + err = consumer.Close() if err != nil { panic(err) } - _, err = db.Exec("create table example_tmq.t1 (ts timestamp,v int)") - if err != nil { - panic(err) - } - for { - result, err := consumer.Poll(time.Second) - if err != nil { - panic(err) - } - if result.Type != common.TMQ_RES_TABLE_META { - panic("want message type 2 got " + strconv.Itoa(int(result.Type))) - } - data, _ := json.Marshal(result.Meta) - fmt.Println(string(data)) - consumer.Commit(context.Background(), result.Message) - consumer.FreeMessage(result.Message) - break - } - _, err = db.Exec("insert into example_tmq.t1 values(now,1)") - if err != nil { - panic(err) - } - for { - result, err := consumer.Poll(time.Second) - if err != nil { - panic(err) - } - if result.Type != common.TMQ_RES_DATA { - panic("want message type 1 got " + strconv.Itoa(int(result.Type))) - } - data, _ := json.Marshal(result.Data) - fmt.Println(string(data)) - consumer.Commit(context.Background(), result.Message) - consumer.FreeMessage(result.Message) - break - } - consumer.Close() } diff --git a/docs/examples/python/connect_rest_examples.py b/docs/examples/python/connect_rest_examples.py index 900ec1022ec81ac2db761d918d1ec11c9bb26852..dba00b5a8279a3cbb3cab0a2d8b26bb312364479 100644 --- a/docs/examples/python/connect_rest_examples.py +++ b/docs/examples/python/connect_rest_examples.py @@ -15,10 +15,10 @@ cursor.execute("CREATE DATABASE power") cursor.execute("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)") # insert data -cursor.execute("""INSERT INTO power.d1001 USING power.meters TAGS(California.SanFrancisco, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) - power.d1002 USING power.meters TAGS(California.SanFrancisco, 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) - power.d1003 USING power.meters TAGS(California.LosAngeles, 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) - power.d1004 USING power.meters TAGS(California.LosAngeles, 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)""") +cursor.execute("""INSERT INTO power.d1001 USING power.meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) + power.d1002 USING power.meters TAGS('California.SanFrancisco', 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) + power.d1003 USING power.meters TAGS('California.LosAngeles', 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) + power.d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)""") print("inserted row count:", cursor.rowcount) # query data diff --git a/docs/examples/python/connection_usage_native_reference.py b/docs/examples/python/connection_usage_native_reference.py index 4803511e427bf4d906fd3a14ff6faf5a000da96c..a7179b4cf859eb440b535a797eeb8e2be1e33589 100644 --- a/docs/examples/python/connection_usage_native_reference.py +++ b/docs/examples/python/connection_usage_native_reference.py @@ -8,7 +8,7 @@ conn.execute("CREATE DATABASE test") # change database. same as execute "USE db" conn.select_db("test") conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") -affected_row: int = conn.execute("INSERT INTO t1 USING weather TAGS(1) VALUES (now, 23.5) (now+1m, 23.5) (now+2m 24.4)") +affected_row: int = conn.execute("INSERT INTO t1 USING weather TAGS(1) VALUES (now, 23.5) (now+1m, 23.5) (now+2m, 24.4)") print("affected_row", affected_row) # output: # affected_row 3 diff --git a/docs/examples/python/native_insert_example.py b/docs/examples/python/native_insert_example.py index 94fd00a6e9d1dcd2119693c4b5c862d36c219a3d..cdde7d23d24d12e11c67b6c6acc0e0b089fb5335 100644 --- a/docs/examples/python/native_insert_example.py +++ b/docs/examples/python/native_insert_example.py @@ -25,10 +25,10 @@ def create_stable(conn: taos.TaosConnection): # The generated SQL is: -# INSERT INTO d1001 USING meters TAGS(California.SanFrancisco, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) -# d1002 USING meters TAGS(California.SanFrancisco, 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) -# d1003 USING meters TAGS(California.LosAngeles, 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) -# d1004 USING meters TAGS(California.LosAngeles, 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000) +# INSERT INTO d1001 USING meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) +# d1002 USING meters TAGS('California.SanFrancisco', 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) +# d1003 USING meters TAGS('California.LosAngeles', 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) +# d1004 USING meters TAGS('California.LosAngeles', 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000) def get_sql(): global lines diff --git a/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx b/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx index 89818409c5032166e77a50f07ea1b54bd66617cb..e1fd3dacc8d91ea4ce3efde6f71a645cc65140ea 100644 --- a/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx +++ b/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx @@ -47,7 +47,6 @@ OpenTSDB JSON 格式协议采用一个 JSON 字符串表示一行或多行数据 :::note - 对于 JSON 格式协议，TDengine 并不会自动把所有标签转成 NCHAR 类型， 字符串将将转为 NCHAR 类型， 数值将同样转换为 DOUBLE 类型。 -- TDengine 只接收 JSON **数组格式**的字符串，即使一行数据也需要转换成数组形式。 - 默认生成的子表名是根据规则生成的唯一 ID 值。用户也可以通过在 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 `"tags": { "host": "web02","dc": "lga","tname":"cpu1"}` 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 ::: diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx index 1f5a089aaa2a051e238eedc0315c37cad643b33f..039e9eb6356f55d2cadd83cd4c3ce5e3082c04d1 100644 --- a/docs/zh/07-develop/07-tmq.mdx +++ b/docs/zh/07-develop/07-tmq.mdx @@ -115,19 +115,22 @@ class TaosConsumer(): ```go -func NewConsumer(conf *Config) (*Consumer, error) +func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error) -func (c *Consumer) Close() error - -func (c *Consumer) Commit(ctx context.Context, message unsafe.Pointer) error +// 出于兼容目的保留 rebalanceCb 参数，当前未使用 +func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error -func (c *Consumer) FreeMessage(message unsafe.Pointer) +// 出于兼容目的保留 rebalanceCb 参数，当前未使用 +func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error -func (c *Consumer) Poll(timeout time.Duration) (*Result, error) +func (c *Consumer) Poll(timeoutMs int) tmq.Event -func (c *Consumer) Subscribe(topics []string) error +// 出于兼容目的保留 tmq.TopicPartition 参数，当前未使用 +func (c *Consumer) Commit() ([]tmq.TopicPartition, error) func (c *Consumer) Unsubscribe() error + +func (c *Consumer) Close() error ``` @@ -355,50 +358,20 @@ public class MetersDeserializer extends ReferenceDeserializer { ```go -config := tmq.NewConfig() -defer config.Destroy() -err = config.SetGroupID("test") -if err != nil { - panic(err) -} -err = config.SetAutoOffsetReset("earliest") -if err != nil { - panic(err) -} -err = config.SetConnectIP("127.0.0.1") -if err != nil { - panic(err) -} -err = config.SetConnectUser("root") -if err != nil { - panic(err) -} -err = config.SetConnectPass("taosdata") -if err != nil { - panic(err) -} -err = config.SetConnectPort("6030") -if err != nil { - panic(err) -} -err = config.SetMsgWithTableName(true) -if err != nil { - panic(err) -} -err = config.EnableHeartBeat() -if err != nil { - panic(err) -} -err = config.EnableAutoCommit(func(result *wrapper.TMQCommitCallbackResult) { - if result.ErrCode != 0 { - errStr := wrapper.TMQErr2Str(result.ErrCode) - err := errors.NewError(int(result.ErrCode), errStr) - panic(err) - } -}) -if err != nil { - panic(err) +conf := &tmq.ConfigMap{ + "group.id": "test", + "auto.offset.reset": "earliest", + "td.connect.ip": "127.0.0.1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "td.connect.port": "6030", + "client.id": "test_tmq_c", + "enable.auto.commit": "false", + "enable.heartbeat.background": "true", + "experimental.snapshot.enable": "true", + "msg.with.table.name": "true", } +consumer, err := NewConsumer(conf) ``` @@ -532,11 +505,7 @@ consumer.subscribe(topics); ```go -consumer, err := tmq.NewConsumer(config) -if err != nil { - panic(err) -} -err = consumer.Subscribe([]string{"example_tmq_topic"}) +err = consumer.Subscribe("example_tmq_topic", nil) if err != nil { panic(err) } @@ -620,13 +589,17 @@ while(running){ ```go for { - result, err := consumer.Poll(time.Second) - if err != nil { - panic(err) + ev := consumer.Poll(0) + if ev != nil { + switch e := ev.(type) { + case *tmqcommon.DataMessage: + fmt.Println(e.Value()) + case tmqcommon.Error: + fmt.Fprintf(os.Stderr, "%% Error: %v: %v\n", e.Code(), e) + panic(e) + } + consumer.Commit() } - fmt.Println(result) - consumer.Commit(context.Background(), result.Message) - consumer.FreeMessage(result.Message) } ``` @@ -738,7 +711,11 @@ consumer.close(); ```go -consumer.Close() +/* Unsubscribe */ +_ = consumer.Unsubscribe() + +/* Close consumer */ +_ = consumer.Close() ``` diff --git a/docs/zh/08-connector/05-schemaless-api.mdx b/docs/zh/08-connector/05-schemaless-api.mdx index f6d7e09212c71ed7b142eeabf2a2d69c5f79d439..8f0a9273b9988c191681063adfb67ebfd03615d3 100644 --- a/docs/zh/08-connector/05-schemaless-api.mdx +++ b/docs/zh/08-connector/05-schemaless-api.mdx @@ -19,6 +19,7 @@ TDengine 提供了兼容 InfluxDB (v1) 和 OpenTSDB 行协议的 Schemaless API - `precision` TDengine 使用的时间精度 - `u` TDengine 用户名 - `p` TDengine 密码 +- `ttl` 自动创建的子表生命周期，以子表的第一条数据的 TTL 参数为准，不可更新。更多信息请参考[创建表文档](taos-sql/table/#创建表)的 TTL 参数 注意： 目前不支持 InfluxDB 的 token 验证方式，仅支持 Basic 验证和查询参数验证。 diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx index 1ee59d2df429f2bd64b1dfdcc2ef81f12b1272d0..fc6dc571383b8b003ee6e8c8dce73fa7645978e0 100644 --- a/docs/zh/08-connector/14-java.mdx +++ b/docs/zh/08-connector/14-java.mdx @@ -68,39 +68,38 @@ TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对 ### 安装连接器 - + - 目前 taos-jdbcdriver 已经发布到 [Sonatype Repository](https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver) - 仓库，且各大仓库都已同步。 +目前 taos-jdbcdriver 已经发布到 [Sonatype Repository](https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver) 仓库，且各大仓库都已同步。 - - [sonatype](https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver) - - [mvnrepository](https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver) - - [maven.aliyun](https://maven.aliyun.com/mvn/search) +- [sonatype](https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver) +- [mvnrepository](https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver) +- [maven.aliyun](https://maven.aliyun.com/mvn/search) - Maven 项目中，在 pom.xml 中添加以下依赖： +Maven 项目中，在 pom.xml 中添加以下依赖： - ```xml-dtd - - com.taosdata.jdbc - taos-jdbcdriver - 3.0.0 - - ``` +```xml-dtd + + com.taosdata.jdbc + taos-jdbcdriver + 3.0.0 + +``` - - + + - 可以通过下载 TDengine 的源码，自己编译最新版本的 Java connector +可以通过下载 TDengine 的源码，自己编译最新版本的 Java connector - ```shell - git clone https://github.com/taosdata/taos-connector-jdbc.git - cd taos-connector-jdbc - mvn clean install -Dmaven.test.skip=true - ``` +```shell +git clone https://github.com/taosdata/taos-connector-jdbc.git +cd taos-connector-jdbc +mvn clean install -Dmaven.test.skip=true +``` - 编译后，在 target 目录下会产生 taos-jdbcdriver-3.0.*-dist.jar 的 jar 包，并自动将编译的 jar 文件放在本地的 Maven 仓库中。 +编译后，在 target 目录下会产生 taos-jdbcdriver-3.0.*-dist.jar 的 jar 包，并自动将编译的 jar 文件放在本地的 Maven 仓库中。 - + ## 建立连接 @@ -111,125 +110,117 @@ TDengine 的 JDBC URL 规范格式为： 对于建立连接，原生连接与 REST 连接有细微不同。 - + - ```java - Class.forName("com.taosdata.jdbc.TSDBDriver"); - String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata"; - Connection conn = DriverManager.getConnection(jdbcUrl); - ``` +```java +Class.forName("com.taosdata.jdbc.TSDBDriver"); +String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata"; +Connection conn = DriverManager.getConnection(jdbcUrl); +``` - 以上示例，使用了 JDBC 原生连接的 TSDBDriver，建立了到 hostname 为 taosdemo.com，端口为 6030（TDengine 的默认端口），数据库名为 test 的连接。这个 URL - 中指定用户名（user）为 root，密码（password）为 taosdata。 +以上示例，使用了 JDBC 原生连接的 TSDBDriver，建立了到 hostname 为 taosdemo.com，端口为 6030（TDengine 的默认端口），数据库名为 test 的连接。这个 URL +中指定用户名（user）为 root，密码（password）为 taosdata。 - **注意**：使用 JDBC 原生连接，taos-jdbcdriver 需要依赖客户端驱动（Linux 下是 libtaos.so；Windows 下是 taos.dll；macOS 下是 libtaos.dylib）。 +**注意**：使用 JDBC 原生连接，taos-jdbcdriver 需要依赖客户端驱动（Linux 下是 libtaos.so；Windows 下是 taos.dll；macOS 下是 libtaos.dylib）。 - url 中的配置参数如下： +url 中的配置参数如下： - - user：登录 TDengine 用户名，默认值 'root'。 - - password：用户登录密码，默认值 'taosdata'。 - - cfgdir：客户端配置文件目录路径，Linux OS 上默认值 `/etc/taos`，Windows OS 上默认值 `C:/TDengine/cfg`。 - - charset：客户端使用的字符集，默认值为系统字符集。 - - locale：客户端语言环境，默认值系统当前 locale。 - - timezone：客户端使用的时区，默认值为系统当前时区。 - - batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：true。开启批量拉取同时获取一批数据在查询数据量较大时批量拉取可以有效的提升查询性能。 - - batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败将继续执行下面的 SQL。false：不再执行失败 SQL - 后的任何语句。默认值为：false。 +- user：登录 TDengine 用户名，默认值 'root'。 +- password：用户登录密码，默认值 'taosdata'。 +- cfgdir：客户端配置文件目录路径，Linux OS 上默认值 `/etc/taos`，Windows OS 上默认值 `C:/TDengine/cfg`。 +- charset：客户端使用的字符集，默认值为系统字符集。 +- locale：客户端语言环境，默认值系统当前 locale。 +- timezone：客户端使用的时区，默认值为系统当前时区。 +- batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：true。开启批量拉取同时获取一批数据在查询数据量较大时批量拉取可以有效的提升查询性能。 +- batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败将继续执行下面的 SQL。false：不再执行失败 SQL 后的任何语句。默认值为：false。 - JDBC 原生连接的使用请参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1955.html)。 +JDBC 原生连接的使用请参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1955.html)。 - **使用 TDengine 客户端驱动配置文件建立连接 ** +**使用 TDengine 客户端驱动配置文件建立连接 ** - 当使用 JDBC 原生连接连接 TDengine 集群时，可以使用 TDengine 客户端驱动配置文件，在配置文件中指定集群的 firstEp、secondEp 等参数。如下所示： +当使用 JDBC 原生连接连接 TDengine 集群时，可以使用 TDengine 客户端驱动配置文件，在配置文件中指定集群的 firstEp、secondEp 等参数。如下所示： - 1. 在 Java 应用中不指定 hostname 和 port +1. 在 Java 应用中不指定 hostname 和 port - ```java - public Connection getConn() throws Exception{ - Class.forName("com.taosdata.jdbc.TSDBDriver"); - String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata"; - Properties connProps = new Properties(); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - Connection conn = DriverManager.getConnection(jdbcUrl, connProps); - return conn; - } - ``` +```java +public Connection getConn() throws Exception{ + Class.forName("com.taosdata.jdbc.TSDBDriver"); + String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata"; + Properties connProps = new Properties(); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + Connection conn = DriverManager.getConnection(jdbcUrl, connProps); + return conn; +} +``` - 2. 在配置文件中指定 firstEp 和 secondEp +2. 在配置文件中指定 firstEp 和 secondEp - ```shell - # first fully qualified domain name (FQDN) for TDengine system - firstEp cluster_node1:6030 +```shell +# first fully qualified domain name (FQDN) for TDengine system +firstEp cluster_node1:6030 - # second fully qualified domain name (FQDN) for TDengine system, for cluster only - secondEp cluster_node2:6030 +# second fully qualified domain name (FQDN) for TDengine system, for cluster only +secondEp cluster_node2:6030 - # default system charset - # charset UTF-8 +# default system charset +# charset UTF-8 - # system locale - # locale en_US.UTF-8 - ``` +# system locale +# locale en_US.UTF-8 +``` - 以上示例，jdbc 会使用客户端的配置文件，建立到 hostname 为 cluster_node1、端口为 6030、数据库名为 test 的连接。当集群中 firstEp 节点失效时，JDBC 会尝试使用 secondEp - 连接集群。 +以上示例，jdbc 会使用客户端的配置文件，建立到 hostname 为 cluster_node1、端口为 6030、数据库名为 test 的连接。当集群中 firstEp 节点失效时，JDBC 会尝试使用 secondEp 连接集群。 - TDengine 中，只要保证 firstEp 和 secondEp 中一个节点有效，就可以正常建立到集群的连接。 +TDengine 中，只要保证 firstEp 和 secondEp 中一个节点有效，就可以正常建立到集群的连接。 - > **注意**：这里的配置文件指的是调用 JDBC Connector 的应用程序所在机器上的配置文件，Linux OS 上默认值 /etc/taos/taos.cfg ，Windows OS 上默认值 - C://TDengine/cfg/taos.cfg。 +> **注意**：这里的配置文件指的是调用 JDBC Connector 的应用程序所在机器上的配置文件，Linux OS 上默认值 /etc/taos/taos.cfg ，Windows OS 上默认值 C://TDengine/cfg/taos.cfg。 - - + + - ```java - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - String jdbcUrl = "jdbc:TAOS-RS://taosdemo.com:6041/test?user=root&password=taosdata"; - Connection conn = DriverManager.getConnection(jdbcUrl); - ``` +```java +Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); +String jdbcUrl = "jdbc:TAOS-RS://taosdemo.com:6041/test?user=root&password=taosdata"; +Connection conn = DriverManager.getConnection(jdbcUrl); +``` - 以上示例，使用了 JDBC REST 连接的 RestfulDriver，建立了到 hostname 为 taosdemo.com，端口为 6041，数据库名为 test 的连接。这个 URL 中指定用户名（user）为 - root，密码（password）为 taosdata。 +以上示例，使用了 JDBC REST 连接的 RestfulDriver，建立了到 hostname 为 taosdemo.com，端口为 6041，数据库名为 test 的连接。这个 URL 中指定用户名（user）为 root，密码（password）为 taosdata。 - 使用 JDBC REST 连接，不需要依赖客户端驱动。与 JDBC 原生连接相比，仅需要： +使用 JDBC REST 连接，不需要依赖客户端驱动。与 JDBC 原生连接相比，仅需要： - 1. driverClass 指定为“com.taosdata.jdbc.rs.RestfulDriver”； - 2. jdbcUrl 以“jdbc:TAOS-RS://”开头； - 3. 使用 6041 作为连接端口。 +1. driverClass 指定为“com.taosdata.jdbc.rs.RestfulDriver”； +2. jdbcUrl 以“jdbc:TAOS-RS://”开头； +3. 使用 6041 作为连接端口。 - url 中的配置参数如下： +url 中的配置参数如下： - - user：登录 TDengine 用户名，默认值 'root'。 - - password：用户登录密码，默认值 'taosdata'。 - - batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。逐行拉取结果集使用 HTTP 方式进行数据传输。JDBC REST - 连接支持批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP，WebSocket 可以使 JDBC REST 连接支持大数据量查询，并提升查询性能。 - - charset: 当开启批量拉取数据时，指定解析字符串数据的字符集。 - - batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 SQL 了。false：不再执行失败 SQL - 后的任何语句。默认值为：false。 - - httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 5000。 - - httpSocketTimeout: socket 超时时间，单位 ms，默认值为 5000。仅在 batchfetch 设置为 false 时生效。 - - messageWaitTimeout: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 batchfetch 设置为 true 时生效。 - - useSSL: 连接中是否使用 SSL。 +- user：登录 TDengine 用户名，默认值 'root'。 +- password：用户登录密码，默认值 'taosdata'。 +- batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。逐行拉取结果集使用 HTTP 方式进行数据传输。JDBC REST 连接支持批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP，WebSocket 可以使 JDBC REST 连接支持大数据量查询，并提升查询性能。 +- charset: 当开启批量拉取数据时，指定解析字符串数据的字符集。 +- batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 SQL 了。false：不再执行失败 SQL 后的任何语句。默认值为：false。 +- httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 5000。 +- httpSocketTimeout: socket 超时时间，单位 ms，默认值为 5000。仅在 batchfetch 设置为 false 时生效。 +- messageWaitTimeout: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 batchfetch 设置为 true 时生效。 +- useSSL: 连接中是否使用 SSL。 - **注意**：部分配置项（比如：locale、timezone）在 REST 连接中不生效。 +**注意**：部分配置项（比如：locale、timezone）在 REST 连接中不生效。 - :::note +:::note - - 与原生连接方式不同，REST 接口是无状态的。在使用 JDBC REST 连接时，需要在 SQL 中指定表、超级表的数据库名称。例如： +- 与原生连接方式不同，REST 接口是无状态的。在使用 JDBC REST 连接时，需要在 SQL 中指定表、超级表的数据库名称。例如： - ```sql - INSERT INTO test.t1 USING test.weather (ts, temperature) TAGS('California.SanFrancisco') VALUES(now, 24.6); - ``` +```sql +INSERT INTO test.t1 USING test.weather (ts, temperature) TAGS('California.SanFrancisco') VALUES(now, 24.6); +``` - - 如果在 url 中指定了 dbname，那么，JDBC REST 连接会默认使用/rest/sql/dbname 作为 restful 请求的 url，在 SQL 中不需要指定 dbname。例如：url 为 - jdbc:TAOS-RS://127.0.0.1:6041/test，那么，可以执行 sql：insert into t1 using weather(ts, temperature) - tags('California.SanFrancisco') values(now, 24.6); +- 如果在 url 中指定了 dbname，那么，JDBC REST 连接会默认使用/rest/sql/dbname 作为 restful 请求的 url，在 SQL 中不需要指定 dbname。例如：url 为 jdbc:TAOS-RS://127.0.0.1:6041/test，那么，可以执行 sql：insert into t1 using weather(ts, temperature) tags('California.SanFrancisco') values(now, 24.6); - ::: +::: - + ### 指定 URL 和 Properties 获取连接 @@ -890,8 +881,10 @@ public static void main(String[] args) throws Exception { | taos-jdbcdriver 版本 | 主要变化 | | :------------------: | :----------------------------: | +| 3.0.3 | 修复 REST 连接在 jdk17+ 版本时间戳解析错误问题 | | 3.0.1 - 3.0.2 | 修复一些情况下结果集数据解析错误的问题。3.0.1 在 JDK 11 环境编译，JDK 8 环境下建议使用 3.0.2 版本 | | 3.0.0 | 支持 TDengine 3.0 | +| 2.0.42 | 修在 WebSocket 连接中 wasNull 接口返回值 | | 2.0.41 | 修正 REST 连接中用户名和密码转码方式 | | 2.0.39 - 2.0.40 | 增加 REST 连接/请求 超时设置 | | 2.0.38 | JDBC REST 连接增加批量拉取功能 | @@ -928,7 +921,7 @@ public static void main(String[] args) throws Exception { **原因**：taos-jdbcdriver 3.0.1 版本需要在 JDK 11+ 环境使用。 -**解决方法**： 更换 taos-jdbcdriver 3.0.2 版本。 +**解决方法**： 更换 taos-jdbcdriver 3.0.2+ 版本。 其它问题请参考 [FAQ](../../../train-faq/faq) diff --git a/docs/zh/08-connector/20-go.mdx b/docs/zh/08-connector/20-go.mdx index 0fc4007f6362697222b425c8c2c803b911b9ac8a..2aa1a58e49f34b412f12bd0d67586dc6e56cf0bc 100644 --- a/docs/zh/08-connector/20-go.mdx +++ b/docs/zh/08-connector/20-go.mdx @@ -15,7 +15,7 @@ import GoOpenTSDBTelnet from "../07-develop/03-insert-data/_go_opts_telnet.mdx" import GoOpenTSDBJson from "../07-develop/03-insert-data/_go_opts_json.mdx" import GoQuery from "../07-develop/04-query-data/_go.mdx" -`driver-go` 是 TDengine 的官方 Go 语言连接器，实现了 Go 语言[ database/sql ](https://golang.org/pkg/database/sql/) 包的接口。Go 开发人员可以通过它开发存取 TDengine 集群数据的应用软件。 +`driver-go` 是 TDengine 的官方 Go 语言连接器，实现了 Go 语言 [database/sql](https://golang.org/pkg/database/sql/) 包的接口。Go 开发人员可以通过它开发存取 TDengine 集群数据的应用软件。 `driver-go` 提供两种建立连接的方式。一种是**原生连接**，它通过 TDengine 客户端驱动程序（taosc）原生连接 TDengine 运行实例，支持数据写入、查询、订阅、schemaless 接口和参数绑定接口等功能。另外一种是 **REST 连接**，它通过 taosAdapter 提供的 REST 接口连接 TDengine 运行实例。REST 连接实现的功能特性集合和原生连接有少量不同。 @@ -112,6 +112,7 @@ REST 连接支持所有能运行 Go 的平台。 ```text username:password@protocol(address)/dbname?param=value ``` + ### 使用连接器进行连接 @@ -176,6 +177,7 @@ func main() { } } ``` + @@ -207,6 +209,7 @@ func main() { } } ``` + @@ -357,33 +360,32 @@ func main() { #### 订阅 -* `func NewConsumer(conf *Config) (*Consumer, error)` - -创建消费者。 +* `func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error)` -* `func (c *Consumer) Subscribe(topics []string) error` + 创建消费者。 -订阅主题。 +* `func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error` +注意：出于兼容目的保留 `rebalanceCb` 参数，当前未使用 -* `func (c *Consumer) Poll(timeout time.Duration) (*Result, error)` + 订阅单个主题。 -轮询消息。 +* `func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error` +注意：出于兼容目的保留 `rebalanceCb` 参数，当前未使用 -* `func (c *Consumer) Commit(ctx context.Context, message unsafe.Pointer) error` + 订阅主题。 -提交消息。 +* `func (c *Consumer) Poll(timeoutMs int) tmq.Event` -* `func (c *Consumer) FreeMessage(message unsafe.Pointer)` + 轮询消息。 -释放消息。 +* `func (c *Consumer) Commit() ([]tmq.TopicPartition, error)` +注意：出于兼容目的保留 `tmq.TopicPartition` 参数，当前未使用 -* `func (c *Consumer) Unsubscribe() error` - -取消订阅。 + 提交消息。 * `func (c *Consumer) Close() error` -关闭消费者。 + 关闭连接。 #### schemaless @@ -443,25 +445,32 @@ func main() { ### 通过 WebSocket 订阅 -* `func NewConsumer(config *Config) (*Consumer, error)` +* `func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error)` + + 创建消费者。 + +* `func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error` +注意：出于兼容目的保留 `rebalanceCb` 参数，当前未使用 - 创建消费者。 + 订阅单个主题。 -* `func (c *Consumer) Subscribe(topic []string) error` +* `func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error` +注意：出于兼容目的保留 `rebalanceCb` 参数，当前未使用 - 订阅主题。 + 订阅主题。 -* `func (c *Consumer) Poll(timeout time.Duration) (*Result, error)` +* `func (c *Consumer) Poll(timeoutMs int) tmq.Event` - 轮询消息。 + 轮询消息。 -* `func (c *Consumer) Commit(messageID uint64) error` +* `func (c *Consumer) Commit() ([]tmq.TopicPartition, error)` +注意：出于兼容目的保留 `tmq.TopicPartition` 参数，当前未使用 - 提交消息。 + 提交消息。 * `func (c *Consumer) Close() error` - 关闭消费者。 + 关闭连接。 完整订阅示例参见 [GitHub 示例文件](https://github.com/taosdata/driver-go/blob/3.0/examples/tmqoverws/main.go) diff --git a/docs/zh/08-connector/40-csharp.mdx b/docs/zh/08-connector/40-csharp.mdx index a1a161d4eea9e2534ebb3a573211dcfae5dbb21f..c70a8a633e4df46df208c829cce772912cb36f06 100644 --- a/docs/zh/08-connector/40-csharp.mdx +++ b/docs/zh/08-connector/40-csharp.mdx @@ -253,14 +253,14 @@ namespace TDengineExample |示例程序 | 示例程序描述 | |--------------------------------------------------------------------------------------------------------------------|--------------------------------------------| -| [CURD](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/Query/Query.cs) | 使用 TDengine.Connector 实现的建表、插入、查询示例 | -| [JSON Tag](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/JSONTag) | 使用 TDengine.Connector 实现的写入和查询 JSON tag 类型数据的示例 | -| [stmt](https://github.com/taosdata/taos-connector-dotnet/tree/3.0/examples/Stmt) | 使用 TDengine.Connector 实现的参数绑定插入和查询的示例 | -| [schemaless](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/schemaless) | 使用 TDengine.Connector 实现的使用 schemaless 写入的示例 | -| [async query](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/AsyncQuery/QueryAsync.cs) | 使用 TDengine.Connector 实现的异步查询的示例 | -| [数据订阅（TMQ）](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/TMQ/TMQ.cs) | 使用 TDengine.Connector 实现的订阅数据的示例 | -| [Basic WebSocket Usage](https://github.com/taosdata/taos-connector-dotnet/blob/5a4a7cd0dbcda114447cdc6d0c6dedd8e84a52da/examples/WS/WebSocketSample.cs) | 使用 TDengine.Connector 的 WebSocket 基本的示例 | -| [Basic WebSocket STMT](https://github.com/taosdata/taos-connector-dotnet/blob/5a4a7cd0dbcda114447cdc6d0c6dedd8e84a52da/examples/WS/WebSocketSTMT.cs) | 使用 TDengine.Connector 的 WebSocket STMT 基本的示例 | +| [CURD](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/Query/Query.cs) | 使用 TDengine.Connector 实现的建表、插入、查询示例 | +| [JSON Tag](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/JSONTag) | 使用 TDengine.Connector 实现的写入和查询 JSON tag 类型数据的示例 | +| [stmt](https://github.com/taosdata/taos-connector-dotnet/tree/3.0/examples/NET6Examples/Stmt) | 使用 TDengine.Connector 实现的参数绑定插入和查询的示例 | +| [schemaless](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/schemaless) | 使用 TDengine.Connector 实现的使用 schemaless 写入的示例 | +| [async query](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/AsyncQuery/QueryAsync.cs) | 使用 TDengine.Connector 实现的异步查询的示例 | +| [数据订阅（TMQ）](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/TMQ/TMQ.cs) | 使用 TDengine.Connector 实现的订阅数据的示例 | +| [Basic WebSocket Usage](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/FrameWork45/WS/WebSocketSample.cs) | 使用 TDengine.Connector 的 WebSocket 基本的示例 | +| [Basic WebSocket STMT](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/FrameWork45/WS/WebSocketSTMT.cs) | 使用 TDengine.Connector 的 WebSocket STMT 基本的示例 | ## 重要更新记录 diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index 3918f240b4f9037db92af80df26dd471868b29de..7d9566f4f257ed7a0ffdcdfb06d0d7116bf24043 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -27,10 +27,13 @@ database_option: { | PRECISION {'ms' | 'us' | 'ns'} | REPLICA value | RETENTIONS ingestion_duration:keep_duration ... - | STRICT {'off' | 'on'} | WAL_LEVEL {1 | 2} | VGROUPS value | SINGLE_STABLE {0 | 1} + | STT_TRIGGER value + | TABLE_PREFIX value + | TABLE_SUFFIX value + | TSDB_PAGESIZE value | WAL_RETENTION_PERIOD value | WAL_ROLL_PERIOD value | WAL_RETENTION_SIZE value @@ -61,9 +64,6 @@ database_option: { - PRECISION：数据库的时间戳精度。ms 表示毫秒，us 表示微秒，ns 表示纳秒，默认 ms 毫秒。 - REPLICA：表示数据库副本数，取值为 1 或 3，默认为 1。在集群中使用，副本数必须小于或等于 DNODE 的数目。 - RETENTIONS：表示数据的聚合周期和保存时长，如 RETENTIONS 15s:7d,1m:21d,15m:50d 表示数据原始采集周期为 15 秒，原始数据保存 7 天；按 1 分钟聚合的数据保存 21 天；按 15 分钟聚合的数据保存 50 天。目前支持且只支持三级存储周期。 -- STRICT：表示数据同步的一致性要求，默认为 off。 - - on 表示强一致，即运行标准的 raft 协议，半数提交返回成功。 - - off 表示弱一致，本地提交即返回成功。 - WAL_LEVEL：WAL 级别，默认为 1。 - 1：写 WAL，但不执行 fsync。 - 2：写 WAL，而且执行 fsync。 @@ -71,6 +71,10 @@ database_option: { - SINGLE_STABLE：表示此数据库中是否只可以创建一个超级表，用于超级表列非常多的情况。 - 0：表示可以创建多张超级表。 - 1：表示只可以创建一张超级表。 +- STT_TRIGGER：表示落盘文件触发文件合并的个数。默认为 1，范围 1 到 16。对于少表高频场景，此参数建议使用默认配置，或较小的值；而对于多表低频场景，此参数建议配置较大的值。 +- TABLE_PREFIX：内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的前缀的长度。 +- TABLE_SUFFIX：内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的后缀的长度。 +- TSDB_PAGESIZE：一个 VNODE 中时序数据存储引擎的页大小，单位为 KB，默认为 4 KB。范围为 1 到 16384，即 1 KB到 16 MB。 - WAL_RETENTION_PERIOD：wal 文件的额外保留策略，用于数据订阅。wal 的保存时长，单位为 s。单副本默认为 0，即落盘后立即删除。-1 表示不删除。多副本默认为 4 天。 - WAL_RETENTION_SIZE：wal 文件的额外保留策略，用于数据订阅。wal 的保存的最大上限，单位为 KB。单副本默认为 0，即落盘后立即删除。多副本默认为-1，表示不删除。 - WAL_ROLL_PERIOD：wal 文件切换时长，单位为 s。当 wal 文件创建并写入后，经过该时间，会自动创建一个新的 wal 文件。单副本默认为 0，即仅在落盘时创建新文件。多副本默认为 1 天。 @@ -112,6 +116,10 @@ alter_database_options: alter_database_option: { CACHEMODEL {'none' | 'last_row' | 'last_value' | 'both'} | CACHESIZE value + | BUFFER value + | PAGES value + | REPLICA value + | STT_TRIGGER value | WAL_LEVEL value | WAL_FSYNC_PERIOD value | KEEP value @@ -154,3 +162,19 @@ TRIM DATABASE db_name; ``` 删除过期数据，并根据多级存储的配置归整数据。 + +## 调整VGROUP中VNODE的分布 + +```sql +REDISTRIBUTE VGROUP vgroup_no DNODE dnode_id1 [DNODE dnode_id2] [DNODE dnode_id3] +``` + +按照给定的dnode列表，调整vgroup中的vnode分布。因为副本数目最大为3，所以最多输入3个dnode。 + +## 自动调整VGROUP中VNODE的分布 + +```sql +BALANCE VGROUP +``` + +自动调整集群所有vgroup中的vnode分布，相当于在vnode级别对集群进行数据的负载均衡操作。 \ No newline at end of file diff --git a/docs/zh/12-taos-sql/04-stable.md b/docs/zh/12-taos-sql/04-stable.md index bd32da68aecc4c42be110e7716c1b2b4ebdab37c..c5933228decfb3eb865cad058fa6fbba77fefb58 100644 --- a/docs/zh/12-taos-sql/04-stable.md +++ b/docs/zh/12-taos-sql/04-stable.md @@ -139,10 +139,10 @@ alter_table_option: { - ADD COLUMN：添加列。 - DROP COLUMN：删除列。 -- MODIFY COLUMN：修改列定义，如果数据列的类型是可变长类型，那么可以使用此指令修改其宽度，只能改大，不能改小。 +- MODIFY COLUMN：修改列的宽度，数据列的类型必须是 nchar 和 binary，使用此指令可以修改其宽度，只能改大，不能改小。 - ADD TAG：给超级表添加一个标签。 - DROP TAG：删除超级表的一个标签。从超级表删除某个标签后，该超级表下的所有子表也会自动删除该标签。 -- MODIFY TAG：修改超级表的一个标签的定义。如果标签的类型是可变长类型，那么可以使用此指令修改其宽度，只能改大，不能改小。 +- MODIFY TAG：修改超级表的一个标签的列宽度。标签的类型只能是 nchar 和 binary，使用此指令可以修改其宽度，只能改大，不能改小。 - RENAME TAG：修改超级表的一个标签的名称。从超级表修改某个标签名后，该超级表下的所有子表也会自动更新该标签名。 ### 增加列 diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index 3b681f401c826a8b93ba446ad2804cb37c8c7bf7..8244c0f27a271347938e8b55cd5d0f4938df0ea7 100644 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -350,9 +350,9 @@ SELECT AVG(CASE WHEN voltage < 200 or voltage > 250 THEN 220 ELSE voltage END) F ## JOIN 子句 -TDengine 支持“普通表与普通表之间”、“超级表与超级表之间”、“子查询与子查询之间” 进行自然连接。自然连接与内连接的主要区别是，自然连接要求参与连接的字段在不同的表/超级表中必须是同名字段。也即，TDengine 在连接关系的表达中，要求必须使用同名数据列/标签列的相等关系。 +TDengine 支持基于时间戳主键的内连接，即 JOIN 条件必须包含时间戳主键。只要满足基于时间戳主键这个要求，普通表、子表、超级表和子查询之间可以随意的进行内连接，且对表个数没有限制。 -在普通表与普通表之间的 JOIN 操作中，只能使用主键时间戳之间的相等关系。例如： +普通表与普通表之间的 JOIN 操作： ```sql SELECT * @@ -360,7 +360,7 @@ FROM temp_tb_1 t1, pressure_tb_1 t2 WHERE t1.ts = t2.ts ``` -在超级表与超级表之间的 JOIN 操作中，除了主键时间戳一致的条件外，还要求引入能实现一一对应的标签列的相等关系。例如： +超级表与超级表之间的 JOIN 操作： ```sql SELECT * @@ -368,20 +368,15 @@ FROM temp_stable t1, temp_stable t2 WHERE t1.ts = t2.ts AND t1.deviceid = t2.deviceid AND t1.status=0; ``` -类似地，也可以对多个子查询的查询结果进行 JOIN 操作。 - -:::note - -JOIN 语句存在如下限制要求： +子表与超级表之间的 JOIN 操作： -- 参与一条语句中 JOIN 操作的表/超级表最多可以有 10 个。 -- 在包含 JOIN 操作的查询语句中不支持 FILL。 -- 暂不支持参与 JOIN 操作的表之间聚合后的四则运算。 -- 不支持只对其中一部分表做 GROUP BY。 -- JOIN 查询的不同表的过滤条件之间不能为 OR。 -- JOIN 查询要求连接条件不能是普通列，只能针对标签和主时间字段列（第一列）。 +```sql +SELECT * +FROM temp_ctable t1, temp_stable t2 +WHERE t1.ts = t2.ts AND t1.deviceid = t2.deviceid AND t1.status=0; +``` -::: +类似地，也可以对多个子查询的查询结果进行 JOIN 操作。 ## 嵌套查询 diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index afe90b8a93f5ef6f7b36d43619fb5ba222180ca3..81fdb46f25a609f829401509aa87fb6c29350f90 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -880,6 +880,7 @@ INTERP(expr) - INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。 - INTERP 只能在一个时间序列内进行插值，因此当作用于超级表时必须跟 partition by tbname 一起使用。 - INTERP 可以与伪列 _irowts 一起使用，返回插值点所对应的时间戳(3.0.1.4版本以后支持)。 +- INTERP 可以与伪列 _isfilled 一起使用，显示返回结果是否为原始记录或插值算法产生的数据(3.0.2.3版本以后支持)。 ### LAST diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index a70d559a860c4e6ec44cbd13c34f7f306ab452cb..9e2fde526e1e63e3fbfb6124420caee3491cf965 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -114,7 +114,7 @@ SELECT * from information_schema.`ins_streams`; 在创建流时，可以通过 TRIGGER 指令指定流式计算的触发模式。 -对于非窗口计算，流式计算的触发是实时的；对于窗口计算，目前提供 3 种触发模式： +对于非窗口计算，流式计算的触发是实时的；对于窗口计算，目前提供 3 种触发模式，默认为 AT_ONCE： 1. AT_ONCE：写入立即触发 diff --git a/docs/zh/12-taos-sql/20-keywords.md b/docs/zh/12-taos-sql/20-keywords.md index 8013698fced2f31ca21dfa220066b027e71cb856..8fd704ef551c9ebda61678bd129917e67cbfbef8 100644 --- a/docs/zh/12-taos-sql/20-keywords.md +++ b/docs/zh/12-taos-sql/20-keywords.md @@ -18,6 +18,7 @@ description: TDengine 保留关键字的详细列表 - ADD - AFTER - AGGREGATE +- ALIVE - ALL - ALTER - ANALYZE diff --git a/docs/zh/12-taos-sql/24-show.md b/docs/zh/12-taos-sql/24-show.md index 4bd1e52284f46d94cfd892c82a7fbe119b685532..2b875199b501136afef4ce0ba1cbcdd1e0abc933 100644 --- a/docs/zh/12-taos-sql/24-show.md +++ b/docs/zh/12-taos-sql/24-show.md @@ -179,6 +179,81 @@ SHOW TABLE DISTRIBUTED table_name; 显示表的数据分布信息。 +示例说明： + +语句： show table distributed d0\G; 竖行显示表 d0 的 BLOCK 分布情况 + +
+ 显示示例 + 

+
+*************************** 1.row ***************************
+
+_block_dist: Total_Blocks=[5] Total_Size=[93.65 Kb] Average_size=[18.73 Kb] Compression_Ratio=[23.98 %]
+
+Total_Blocks :  表d0 占用的 block 个数为 5 个
+
+Total_Size.    :  表 d0 所有 block 在文件中占用的大小为 93.65 KB 
+
+Average_size:  平均每个 block 在文件中占用的空间大小为 18.73 KB
+
+Compression_Ratio: 数据压缩率为 23.98%
+
+ 
+*************************** 2.row ***************************
+
+_block_dist: Total_Rows=[20000] Inmem_Rows=[0] MinRows=[3616] MaxRows=[4096] Average_Rows=[4000]
+
+Total_Rows: 统计表 d0 的所有行数 为20000 行
+
+Inmem_Rows： 表示仍然还存放在内存中的行数，即没有落盘的行数，为 0行，表示没有
+
+MinRows：  BLOCK 中最小的行数，为 3616 行
+
+MaxRows： BLOCK 中最大的行数，为 4096行
+
+Average_Rows： BLOCK 中的平均行数，为4000 行
+
+
+*************************** 3.row ***************************
+
+_block_dist: Total_Tables=[1] Total_Files=[2]
+
+Total_Tables:  表示子表的个数，这里为1
+
+Total_Files：   表数据保存在几个文件中，这里保存在 2 个文件中
+
+
+*************************** 5.row ***************************
+
+_block_dist: 0100 |
+
+*************************** 6.row ***************************
+
+_block_dist: 0299 |
+
+......
+
+*************************** 22.row ***************************
+
+_block_dist: 3483 |||||||||||||||||  1 (20.00%)
+
+*************************** 23.row ***************************
+
+_block_dist: 3682 |
+
+*************************** 24.row ***************************
+
+_block_dist: 3881 |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||  4 (80.00%)
+
+Query OK, 24 row(s) in set (0.002444s)
+
+
+
+ + 上面是块中包含数据行数的块儿分布情况图，这里的 0100 0299 0498 … 表示的是每个块中包含的数据行数，上面的意思就是这个表的 5 个块，分布在 3483 ~3681 行的块有 1 个，占整个块的 20%，分布在 3881 ~ 4096（最大行数）的块数为 4 个，占整个块的 80%， 其它区域内分布块数为 0。 + + ## SHOW TAGS ```sql diff --git a/docs/zh/12-taos-sql/29-changes.md b/docs/zh/12-taos-sql/29-changes.md index 1ab7f4f0169932576edf9087a825367573713b03..9d67533cdea5c055c8d3ff58000fb80ac640e271 100644 --- a/docs/zh/12-taos-sql/29-changes.md +++ b/docs/zh/12-taos-sql/29-changes.md @@ -54,7 +54,6 @@ description: "TDengine 3.0 版本的语法变更说明" | 27 | GRANT | 新增 | 授予用户权限。 | 28 | KILL TRANSACTION | 新增 | 终止管理节点的事务。 | 29 | KILL STREAM | 废除 | 终止连续查询。3.0版本不再支持连续查询，而是用更通用的流计算来代替。 -| 30 | MERGE VGROUP | 新增 | 合并VGROUP。 | 31 | REVOKE | 新增 | 回收用户权限。 | 32 | SELECT | 调整 |
  • SELECT关闭隐式结果列，输出列均需要由SELECT子句来指定。
  • DISTINCT功能全面支持。2.x版本只支持对标签列去重，并且不可以和JOIN、GROUP BY等子句混用。
  • JOIN功能增强。增加支持：JOIN后WHERE条件中有OR条件；JOIN后的多表运算；JOIN后的多表GROUP BY。
  • FROM后子查询功能大幅增强。不限制子查询嵌套层数；支持子查询和UNION ALL混合使用；移除其他一些之前版本的语法限制。
  • WHERE后可以使用任意的标量表达式。
  • GROUP BY功能增强。支持任意标量表达式及其组合的分组。
  • SESSION可以用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。
  • STATE_WINDOW可以用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。
  • ORDER BY功能大幅增强。不再必须和GROUP BY子句一起使用；不再有排序表达式个数的限制；增加支持NULLS FIRST/LAST语法功能；支持符合语法语义的任意表达式。
  • 新增PARTITION BY语法。替代原来的GROUP BY tags。
| 33 | SHOW ACCOUNTS | 废除 | 2.x中为企业版功能，3.0不再支持。语法暂时保留了，执行报“This statement is no longer supported”错误。 @@ -76,8 +75,9 @@ description: "TDengine 3.0 版本的语法变更说明" | 49 | SHOW TRANSACTIONS | 新增 | 显示当前系统中正在执行的事务的信息。 | 50 | SHOW DNODE VARIABLES | 新增 |显示指定DNODE的配置参数。 | 51 | SHOW VNODES | 暂不支持 | 显示当前系统中VNODE的信息。3.0.0版本暂不支持。 -| 52 | SPLIT VGROUP | 新增 | 拆分VGROUP。 -| 53 | TRIM DATABASE | 新增 | 删除过期数据，并根据多级存储的配置归整数据。 +| 52 | TRIM DATABASE | 新增 | 删除过期数据，并根据多级存储的配置归整数据。 +| 53 | REDISTRIBUTE VGROUP | 新增 | 调整VGROUP中VNODE的分布。 +| 54 | BALANCE VGROUP | 新增 | 自动调整VGROUP中VNODE的分布。 ## SQL 函数变更 @@ -94,6 +94,7 @@ description: "TDengine 3.0 版本的语法变更说明" | 9 | SAMPLE | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 | 10 | STATECOUNT | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 | 11 | STATEDURATION | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 +| 12 | TIMETRUNCATE | 增强 | 增加ignore_timezone参数，可选是否使用，默认值为1. ## SCHEMALESS 变更 diff --git a/docs/zh/14-reference/04-taosadapter.md b/docs/zh/14-reference/04-taosadapter.md index 4e1e7b7ace09f7cb43e5049ebefe21a24525b595..5c155bdd6ebe9f1a4a38886f30b753cbda4ab6ab 100644 --- a/docs/zh/14-reference/04-taosadapter.md +++ b/docs/zh/14-reference/04-taosadapter.md @@ -21,6 +21,7 @@ taosAdapter 提供以下功能： - 无缝连接到 collectd - 无缝连接到 StatsD - 支持 Prometheus remote_read 和 remote_write +- 获取 table 所在的虚拟节点组（VGroup）的 VGroup ID ## taosAdapter 架构图 @@ -59,6 +60,7 @@ Usage of taosAdapter: --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) + --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" (default 0, means no ttl) -c, --config string config path default /etc/taos/taosadapter.toml --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" @@ -100,6 +102,7 @@ Usage of taosAdapter: --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") + --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL"(default 0, means no ttl) --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) @@ -110,6 +113,7 @@ Usage of taosAdapter: --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") + --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL"(default 0, means no ttl) --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" (default 1h0m0s) --pool.maxConnect int max connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" (default 4000) --pool.maxIdle int max idle connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_IDLE" (default 4000) @@ -131,6 +135,7 @@ Usage of taosAdapter: --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) + --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" (default 0, means no ttl) --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" --version Print the version and exit ``` @@ -174,6 +179,7 @@ AllowWebSockets node_export 是一个机器指标的导出器。请访问 [https://github.com/prometheus/node_exporter](https://github.com/prometheus/node_exporter) 了解更多信息。 - 支持 Prometheus remote_read 和 remote_write remote_read 和 remote_write 是 Prometheus 数据读写分离的集群方案。请访问[https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis) 了解更多信息。 +- 获取 table 所在的虚拟节点组（VGroup）的 VGroup ID。关于虚拟节点组（VGroup）的更多信息，请访问[整体架构文档](/tdinternal/arch/#主要逻辑单元) 。 ## 接口 @@ -195,6 +201,7 @@ AllowWebSockets - `precision` TDengine 使用的时间精度 - `u` TDengine 用户名 - `p` TDengine 密码 +- `ttl` 自动创建的子表生命周期，以子表的第一条数据的 TTL 参数为准，不可更新。更多信息请参考[创建表文档](taos-sql/table/#创建表)的 TTL 参数。 注意： 目前不支持 InfluxDB 的 token 验证方式，仅支持 Basic 验证和查询参数验证。 示例： curl --request POST http://127.0.0.1:6041/influxdb/v1/write?db=test --user "root:taosdata" --data-binary "measurement,host=host1 field1=2i,field2=2.0 1577836800000000000" @@ -235,6 +242,10 @@ Prometheus 使用的由 \*NIX 内核暴露的硬件和操作系统指标的输 +### 获取 table 的 VGroup ID + +可以访问 http 接口 `http://:6041/rest/vgid?db=&table=
` 获取 table 的 VGroup ID。关于虚拟节点组（VGroup）的更多信息，请访问[整体架构文档](/tdinternal/arch/#主要逻辑单元) 。 + ## 内存使用优化方法 taosAdapter 将监测自身运行过程中内存使用率并通过两个阈值进行调节。有效值范围为 -1 到 100 的整数，单位为系统物理内存的百分比。 @@ -277,7 +288,7 @@ http 返回内容： ## taosAdapter 监控指标 -taosAdapter 采集 http 相关指标、cpu 百分比和内存百分比。 +taosAdapter 采集 http 相关指标、CPU 百分比和内存百分比。 ### http 接口 @@ -289,13 +300,13 @@ http://:6041/metrics ### 写入 TDengine -taosAdapter 支持将 http 监控、cpu 百分比和内存百分比写入 TDengine。 +taosAdapter 支持将 http 监控、CPU 百分比和内存百分比写入 TDengine。 有关配置参数 | **配置项** | **描述** | **默认值** | |-------------------------|--------------------------------------------|----------| -| monitor.collectDuration | cpu 和内存采集间隔 | 3s | +| monitor.collectDuration | CPU 和内存采集间隔 | 3s | | monitor.identity | 当前taosadapter 的标识符如果不设置将使用 'hostname:port' | | | monitor.incgroup | 是否是 cgroup 中运行(容器中运行设置为 true) | false | | monitor.writeToTD | 是否写入到 TDengine | false | diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md index 76dd5f12d89b223292a8c868fd66135ec204319b..9091e71d1f9b118660d5f75ef9be5627143393b0 100644 --- a/docs/zh/14-reference/05-taosbenchmark.md +++ b/docs/zh/14-reference/05-taosbenchmark.md @@ -204,6 +204,10 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) - **-a/--replica ** : 创建数据库时指定其副本数，默认值为 1 。 +- ** -k/--keep-trying ** : 失败后进行重试的次数，默认不重试。需使用 v3.0.9 以上版本。 + +- ** -z/--trying-interval ** : 失败重试间隔时间，单位为毫秒，仅在 -k 指定重试后有效。需使用 v3.0.9 以上版本。 + - **-V/--version** : 显示版本信息并退出。不能与其它参数混用。 @@ -231,6 +235,10 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) 插入场景下 `filetype` 必须设置为 `insert`，该参数及其它通用参数详见[通用配置参数](#通用配置参数) +- ** keep_trying ** : 失败后进行重试的次数，默认不重试。需使用 v3.0.9 以上版本。 + +- ** trying_interval ** : 失败重试间隔时间，单位为毫秒，仅在 keep_trying 指定重试后有效。需使用 v3.0.9 以上版本。 + #### 数据库相关配置参数 创建数据库时的相关参数在 json 配置文件中的 `dbinfo` 中配置，个别具体参数如下。其余参数均与 TDengine 中 `create database` 时所指定的数据库参数相对应，详见[../../taos-sql/database] diff --git a/docs/zh/14-reference/06-taosdump.md b/docs/zh/14-reference/06-taosdump.md index 625499a94926ac3f86e4d34976c70a0bfe7b0954..8a031d147377713d0478d419798240f6f1194377 100644 --- a/docs/zh/14-reference/06-taosdump.md +++ b/docs/zh/14-reference/06-taosdump.md @@ -22,7 +22,7 @@ taosdump 是一个逻辑备份工具，它不应被用于备份任何原始数 taosdump 有两种安装方式: -- 安装 taosTools 官方安装包, 请从[所有下载链接](https://www.taosdata.com/all-downloads)页面找到 taosTools 并下载安装。 +- 安装 taosTools 官方安装包, 请从[发布历史页面](https://docs.taosdata.com/releases/tools/)页面找到 taosTools 并下载安装。 - 单独编译 taos-tools 并安装, 详情请参考 [taos-tools](https://github.com/taosdata/taos-tools) 仓库。 diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 959438f48c288769186d22a9013fe10b738fd48f..135d97e8fbe44daa07a1461f0146bc80bd9c66c8 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -136,12 +136,21 @@ taos --dump-config ### telemetryReporting -| 属性 | 说明 | -| -------- | ---------------------------------------- | -| 适用范围 | 仅服务端适用 | -| 含义 | 是否允许 TDengine 采集和上报基本使用信息 | -| 取值范围 | 0：不允许 1：允许 | -| 缺省值 | 1 | +| 属性 | 说明 | +| -------- | -------------------------------------------- | +| 适用范围 | 仅服务端适用 | +| 含义 |是否上传 telemetry | +| 取值范围 | 0,1 0: 不上传；1：上传 | +| 缺省值 | 1 | + +### crashReporting + +| 属性 | 说明 | +| -------- | -------------------------------------------- | +| 适用范围 | 仅服务端适用 | +| 含义 |是否上传 crash 信息 | +| 取值范围 | 0,1 0: 不上传；1：上传 | +| 缺省值 | 1 | ## 查询相关 @@ -191,6 +200,15 @@ taos --dump-config | 取值范围 | 0 表示包含函数名，1 表示不包含函数名。 | | 缺省值 | 0 | +### countAlwaysReturnValue + +| 属性 | 说明 | +| -------- | -------------------------------- | +| 适用范围 | 仅服务端适用 | +| 含义 | count/hyperloglog函数在数据为空或者NULL的情况下是否返回值 | +| 取值范围 | 0：返回空行，1：返回 0 | +| 缺省值 | 1 | + ## 区域相关 ### timezone @@ -306,12 +324,20 @@ charset 的有效值是 UTF-8。 | 含义 | 数据文件目录，所有的数据文件都将写入该目录 | | 缺省值 | /var/lib/taos | +### tempDir + +| 属性 | 说明 | +| -------- | ------------------------------------------ | +| 适用范围 | 仅服务端适用 | +| 含义 | 该参数指定所有系统运行过程中的临时文件生成的目录 | +| 缺省值 | /tmp | + ### minimalTmpDirGB | 属性 | 说明 | | -------- | ------------------------------------------------ | | 适用范围 | 服务端和客户端均适用 | -| 含义 | 当日志文件夹的磁盘大小小于该值时，停止写临时文件 | +| 含义 | tempDir 所指定的临时文件目录所需要保留的最小空间 | | 单位 | GB | | 缺省值 | 1.0 | @@ -320,7 +346,7 @@ charset 的有效值是 UTF-8。 | 属性 | 说明 | | -------- | ------------------------------------------------ | | 适用范围 | 仅服务端适用 | -| 含义 | 当日志文件夹的磁盘大小小于该值时，停止写时序数据 | +| 含义 | dataDir 指定的时序数据存储目录所需要保留的最小 | | 单位 | GB | | 缺省值 | 2.0 | @@ -335,27 +361,7 @@ charset 的有效值是 UTF-8。 | 取值范围 | 0-4096 | | 缺省值 | CPU 核数的 2 倍 | -## 时间相关 - -### statusInterval - -| 属性 | 说明 | -| -------- | --------------------------- | -| 适用范围 | 仅服务端适用 | -| 含义 | dnode 向 mnode 报告状态间隔 | -| 单位 | 秒 | -| 取值范围 | 1-10 | -| 缺省值 | 1 | - -### shellActivityTimer - -| 属性 | 说明 | -| -------- | --------------------------------- | -| 适用范围 | 服务端和客户端均适用 | -| 含义 | shell 客户端向 mnode 发送心跳间隔 | -| 单位 | 秒 | -| 取值范围 | 1-120 | -| 缺省值 | 3 | +## 时间相关 | ## 性能调优 @@ -367,28 +373,6 @@ charset 的有效值是 UTF-8。 | 含义 | 设置写入线程的最大数量 | | 缺省值 | | -## 压缩相关 - -### compressMsgSize - -| 属性 | 说明 | -| -------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| 适用范围 | 仅服务端适用 | -| 含义 | 客户端与服务器之间进行消息通讯过程中，对通讯的消息进行压缩的阈值。如果要压缩消息，建议设置为 64330 字节，即大于 64330 字节的消息体才进行压缩。 | -| 单位 | bytes | -| 取值范围 | `0 `表示对所有的消息均进行压缩 >0: 超过该值的消息才进行压缩 -1: 不压缩 | -| 缺省值 | -1 | - -### compressColData - -| 属性 | 说明 | -| -------- | --------------------------------------------------------------------------------------- | -| 适用范围 | 仅服务端适用 | -| 含义 | 客户端与服务器之间进行消息通讯过程中，对服务器端查询结果进行列压缩的阈值。 | -| 单位 | bytes | -| 取值范围 | 0: 对所有查询结果均进行压缩 >0: 查询结果中任意列大小超过该值的消息才进行压缩 -1: 不压缩 | -| 缺省值 | -1 | - ## 日志相关 ### logDir @@ -613,7 +597,7 @@ charset 的有效值是 UTF-8。 | 属性 | 说明 | | -------- | ------------------------- | | 适用范围 | 仅客户端适用 | -| 含义 | schemaless 自定义的子表名 | +| 含义 | schemaless 自定义的子表名的 key | | 类型 | 字符串 | | 缺省值 | 无 | @@ -656,12 +640,18 @@ charset 的有效值是 UTF-8。 | 取值范围 | 0: 不启动；1：启动 | | 缺省值 | 1 | -## 2.X 与 3.0 配置参数对比 +## 压缩参数 -:::note -对于 2.x 版本中适用但在 3.0 版本中废弃的参数，其当前行为会有特别说明 +### compressMsgSize -::: +| 属性 | 说明 | +| -------- | ----------------------------- | +| 适用于 | 服务端和客户端均适用 | +| 含义 | 是否对 RPC 消息进行压缩 | +| 取值范围 | -1: 所有消息都不压缩; 0: 所有消息都压缩; N (N>0): 只有大于 N 个字节的消息才压缩 | +| 缺省值 | -1 | + +## 3.0 中有效的配置参数列表 | # | **参数** | **适用于 2.X ** | **适用于 3.0 ** | 3.0 版本的当前行为 | | --- | :---------------------: | --------------- | --------------- | ------------------------------------------------- | @@ -674,159 +664,134 @@ charset 的有效值是 UTF-8。 | 7 | monitorFqdn | 否 | 是 | | | 8 | monitorPort | 否 | 是 | | | 9 | monitorInterval | 是 | 是 | | -| 10 | monitorMaxLogs | 否 | 是 | | -| 11 | monitorComp | 否 | 是 | | -| 12 | telemetryReporting | 是 | 是 | | -| 13 | telemetryInterval | 否 | 是 | | -| 14 | telemetryServer | 否 | 是 | | -| 15 | telemetryPort | 否 | 是 | | -| 16 | queryPolicy | 否 | 是 | | -| 17 | querySmaOptimize | 否 | 是 | | -| 18 | queryRsmaTolerance | 否 | 是 | | -| 19 | queryBufferSize | 是 | 是 | | -| 20 | maxNumOfDistinctRes | 是 | 是 | | -| 21 | minSlidingTime | 是 | 是 | | -| 22 | minIntervalTime | 是 | 是 | | -| 23 | countAlwaysReturnValue | 是 | 是 | | -| 24 | dataDir | 是 | 是 | | -| 25 | minimalDataDirGB | 是 | 是 | | -| 26 | supportVnodes | 否 | 是 | | -| 27 | tempDir | 是 | 是 | | -| 28 | minimalTmpDirGB | 是 | 是 | | -| 29 | compressMsgSize | 是 | 是 | | -| 30 | compressColData | 是 | 是 | | -| 31 | smlChildTableName | 是 | 是 | | -| 32 | smlTagName | 是 | 是 | | -| 33 | smlDataFormat | 否 | 是 | | -| 34 | statusInterval | 是 | 是 | | -| 35 | shellActivityTimer | 是 | 是 | | -| 36 | transPullupInterval | 否 | 是 | | -| 37 | mqRebalanceInterval | 否 | 是 | | -| 38 | ttlUnit | 否 | 是 | | -| 39 | ttlPushInterval | 否 | 是 | | -| 40 | numOfTaskQueueThreads | 否 | 是 | | -| 41 | numOfRpcThreads | 否 | 是 | | -| 42 | numOfCommitThreads | 是 | 是 | | -| 43 | numOfMnodeReadThreads | 否 | 是 | | -| 44 | numOfVnodeQueryThreads | 否 | 是 | | -| 45 | ratioOfVnodeStreamThreads | 否 | 是 | | -| 46 | numOfVnodeFetchThreads | 否 | 是 | | -| 47 | numOfVnodeRsmaThreads | 否 | 是 | | -| 48 | numOfQnodeQueryThreads | 否 | 是 | | -| 49 | numOfQnodeFetchThreads | 否 | 是 | | -| 50 | numOfSnodeSharedThreads | 否 | 是 | | -| 51 | numOfSnodeUniqueThreads | 否 | 是 | | -| 52 | rpcQueueMemoryAllowed | 否 | 是 | | -| 53 | logDir | 是 | 是 | | -| 54 | minimalLogDirGB | 是 | 是 | | -| 55 | numOfLogLines | 是 | 是 | | -| 56 | asyncLog | 是 | 是 | | -| 57 | logKeepDays | 是 | 是 | | -| 60 | debugFlag | 是 | 是 | | -| 61 | tmrDebugFlag | 是 | 是 | | -| 62 | uDebugFlag | 是 | 是 | | -| 63 | rpcDebugFlag | 是 | 是 | | -| 64 | jniDebugFlag | 是 | 是 | | -| 65 | qDebugFlag | 是 | 是 | | -| 66 | cDebugFlag | 是 | 是 | | -| 67 | dDebugFlag | 是 | 是 | | -| 68 | vDebugFlag | 是 | 是 | | -| 69 | mDebugFlag | 是 | 是 | | -| 70 | wDebugFlag | 是 | 是 | | -| 71 | sDebugFlag | 是 | 是 | | -| 72 | tsdbDebugFlag | 是 | 是 | | -| 73 | tqDebugFlag | 否 | 是 | | -| 74 | fsDebugFlag | 是 | 是 | | -| 75 | udfDebugFlag | 否 | 是 | | -| 76 | smaDebugFlag | 否 | 是 | | -| 77 | idxDebugFlag | 否 | 是 | | -| 78 | tdbDebugFlag | 否 | 是 | | -| 79 | metaDebugFlag | 否 | 是 | | -| 80 | timezone | 是 | 是 | | -| 81 | locale | 是 | 是 | | -| 82 | charset | 是 | 是 | | -| 83 | udf | 是 | 是 | | -| 84 | enableCoreFile | 是 | 是 | | -| 85 | arbitrator | 是 | 否 | 通过 RAFT 协议选主 | -| 86 | numOfThreadsPerCore | 是 | 否 | 有其它参数设置多种线程池的大小 | -| 87 | numOfMnodes | 是 | 否 | 通过 create mnode 命令动态创建 mnode | -| 88 | vnodeBak | 是 | 否 | 3.0 行为未知 | -| 89 | balance | 是 | 否 | 负载均衡功能由 split/merge vgroups 实现 | -| 90 | balanceInterval | 是 | 否 | 随着 balance 参数失效 | -| 91 | offlineThreshold | 是 | 否 | 3.0 行为未知 | -| 92 | role | 是 | 否 | 由 supportVnode 决定是否能够创建 | -| 93 | dnodeNopLoop | 是 | 否 | 2.6 文档中未找到此参数 | -| 94 | keepTimeOffset | 是 | 否 | 2.6 文档中未找到此参数 | -| 95 | rpcTimer | 是 | 否 | 3.0 行为未知 | -| 96 | rpcMaxTime | 是 | 否 | 3.0 行为未知 | -| 97 | rpcForceTcp | 是 | 否 | 默认为 TCP | -| 98 | tcpConnTimeout | 是 | 否 | 3.0 行为未知 | -| 99 | syncCheckInterval | 是 | 否 | 3.0 行为未知 | -| 100 | maxTmrCtrl | 是 | 否 | 3.0 行为未知 | -| 101 | monitorReplica | 是 | 否 | 由 RAFT 协议管理多副本 | -| 102 | smlTagNullName | 是 | 否 | 3.0 行为未知 | -| 103 | keepColumnName | 是 | 否 | 3.0 行为未知 | -| 104 | ratioOfQueryCores | 是 | 否 | 由 线程池 相关配置参数决定 | -| 105 | maxStreamCompDelay | 是 | 否 | 3.0 行为未知 | -| 106 | maxFirstStreamCompDelay | 是 | 否 | 3.0 行为未知 | -| 107 | retryStreamCompDelay | 是 | 否 | 3.0 行为未知 | -| 108 | streamCompDelayRatio | 是 | 否 | 3.0 行为未知 | -| 109 | maxVgroupsPerDb | 是 | 否 | 由 create db 的参数 vgroups 指定实际 vgroups 数量 | -| 110 | maxTablesPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | -| 111 | minTablesPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | -| 112 | tableIncStepPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | -| 113 | cache | 是 | 否 | 由 buffer 代替 cache\*blocks | -| 114 | blocks | 是 | 否 | 由 buffer 代替 cache\*blocks | -| 115 | days | 是 | 否 | 由 create db 的参数 duration 取代 | -| 116 | keep | 是 | 否 | 由 create db 的参数 keep 取代 | -| 117 | minRows | 是 | 否 | 由 create db 的参数 minRows 取代 | -| 118 | maxRows | 是 | 否 | 由 create db 的参数 maxRows 取代 | -| 119 | quorum | 是 | 否 | 由 RAFT 协议决定 | -| 120 | comp | 是 | 否 | 由 create db 的参数 comp 取代 | -| 121 | walLevel | 是 | 否 | 由 create db 的参数 wal_level 取代 | -| 122 | fsync | 是 | 否 | 由 create db 的参数 wal_fsync_period 取代 | -| 123 | replica | 是 | 否 | 由 create db 的参数 replica 取代 | -| 124 | partitions | 是 | 否 | 3.0 行为未知 | -| 125 | update | 是 | 否 | 允许更新部分列 | -| 126 | cachelast | 是 | 否 | 由 create db 的参数 cacheModel 取代 | -| 127 | maxSQLLength | 是 | 否 | SQL 上限为 1MB，无需参数控制 | -| 128 | maxWildCardsLength | 是 | 否 | 3.0 行为未知 | -| 129 | maxRegexStringLen | 是 | 否 | 3.0 行为未知 | -| 130 | maxNumOfOrderedRes | 是 | 否 | 3.0 行为未知 | -| 131 | maxConnections | 是 | 否 | 取决于系统配置和系统处理能力，详见后面的 Note | -| 132 | mnodeEqualVnodeNum | 是 | 否 | 3.0 行为未知 | -| 133 | http | 是 | 否 | http 服务由 taosAdapter 提供 | -| 134 | httpEnableRecordSql | 是 | 否 | taosd 不提供 http 服务 | -| 135 | httpMaxThreads | 是 | 否 | taosd 不提供 http 服务 | -| 136 | restfulRowLimit | 是 | 否 | taosd 不提供 http 服务 | -| 137 | httpDbNameMandatory | 是 | 否 | taosd 不提供 http 服务 | -| 138 | httpKeepAlive | 是 | 否 | taosd 不提供 http 服务 | -| 139 | enableRecordSql | 是 | 否 | 3.0 行为未知 | -| 140 | maxBinaryDisplayWidth | 是 | 否 | 3.0 行为未知 | -| 141 | stream | 是 | 否 | 默认启用连续查询 | -| 142 | retrieveBlockingModel | 是 | 否 | 3.0 行为未知 | -| 143 | tsdbMetaCompactRatio | 是 | 否 | 3.0 行为未知 | -| 144 | defaultJSONStrType | 是 | 否 | 3.0 行为未知 | -| 145 | walFlushSize | 是 | 否 | 3.0 行为未知 | -| 146 | keepTimeOffset | 是 | 否 | 3.0 行为未知 | -| 147 | flowctrl | 是 | 否 | 3.0 行为未知 | -| 148 | slaveQuery | 是 | 否 | 3.0 行为未知： slave vnode 是否能够处理查询？ | -| 149 | adjustMaster | 是 | 否 | 3.0 行为未知 | -| 150 | topicBinaryLen | 是 | 否 | 3.0 行为未知 | -| 151 | telegrafUseFieldNum | 是 | 否 | 3.0 行为未知 | -| 152 | deadLockKillQuery | 是 | 否 | 3.0 行为未知 | -| 153 | clientMerge | 是 | 否 | 3.0 行为未知 | -| 154 | sdbDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 155 | odbcDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 156 | httpDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 157 | monDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 158 | cqDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 159 | shortcutFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 160 | probeSeconds | 是 | 否 | 3.0 行为未知 | -| 161 | probeKillSeconds | 是 | 否 | 3.0 行为未知 | -| 162 | probeInterval | 是 | 否 | 3.0 行为未知 | -| 163 | lossyColumns | 是 | 否 | 3.0 行为未知 | -| 164 | fPrecision | 是 | 否 | 3.0 行为未知 | -| 165 | dPrecision | 是 | 否 | 3.0 行为未知 | -| 166 | maxRange | 是 | 否 | 3.0 行为未知 | -| 167 | range | 是 | 否 | 3.0 行为未知 | +| 10 | queryPolicy | 否 | 是 | | +| 11 | querySmaOptimize | 否 | 是 | | +| 12 | maxNumOfDistinctRes | 是 | 是 | | +| 15 | countAlwaysReturnValue | 是 | 是 | | +| 16 | dataDir | 是 | 是 | | +| 17 | minimalDataDirGB | 是 | 是 | | +| 18 | supportVnodes | 否 | 是 | | +| 19 | tempDir | 是 | 是 | | +| 20 | minimalTmpDirGB | 是 | 是 | | +| 21 | smlChildTableName | 是 | 是 | | +| 22 | smlTagName | 是 | 是 | | +| 23 | smlDataFormat | 否 | 是 | | +| 24 | statusInterval | 是 | 是 | | +| 25 | logDir | 是 | 是 | | +| 26 | minimalLogDirGB | 是 | 是 | | +| 27 | numOfLogLines | 是 | 是 | | +| 28 | asyncLog | 是 | 是 | | +| 29 | logKeepDays | 是 | 是 | | +| 30 | debugFlag | 是 | 是 | | +| 31 | tmrDebugFlag | 是 | 是 | | +| 32 | uDebugFlag | 是 | 是 | | +| 33 | rpcDebugFlag | 是 | 是 | | +| 34 | jniDebugFlag | 是 | 是 | | +| 35 | qDebugFlag | 是 | 是 | | +| 36 | cDebugFlag | 是 | 是 | | +| 37 | dDebugFlag | 是 | 是 | | +| 38 | vDebugFlag | 是 | 是 | | +| 39 | mDebugFlag | 是 | 是 | | +| 40 | wDebugFlag | 是 | 是 | | +| 41 | sDebugFlag | 是 | 是 | | +| 42 | tsdbDebugFlag | 是 | 是 | | +| 43 | tqDebugFlag | 否 | 是 | | +| 44 | fsDebugFlag | 是 | 是 | | +| 45 | udfDebugFlag | 否 | 是 | | +| 46 | smaDebugFlag | 否 | 是 | | +| 47 | idxDebugFlag | 否 | 是 | | +| 48 | tdbDebugFlag | 否 | 是 | | +| 49 | metaDebugFlag | 否 | 是 | | +| 50 | timezone | 是 | 是 | | +| 51 | locale | 是 | 是 | | +| 52 | charset | 是 | 是 | | +| 53 | udf | 是 | 是 | | +| 54 | enableCoreFile | 是 | 是 | | + +## 2.x->3.0 的废弃参数 + +| # | **参数** | **适用于 2.X ** | **适用于 3.0 ** | 3.0 版本的当前行为 | +| --- | :---------------------: | --------------- | --------------- | ------------------------------------------------- | +| 1 | arbitrator | 是 | 否 | 通过 RAFT 协议选主 | +| 2 | numOfThreadsPerCore | 是 | 否 | 有其它参数设置多种线程池的大小 | +| 3 | numOfMnodes | 是 | 否 | 通过 create mnode 命令动态创建 mnode | +| 4 | vnodeBak | 是 | 否 | 3.0 行为未知 | +| 5 | balance | 是 | 否 | 负载均衡功能由 split/merge vgroups 实现 （暂不支持） | +| 6 | balanceInterval | 是 | 否 | 随着 balance 参数失效 | +| 7 | offlineThreshold | 是 | 否 | 3.0 行为未知 | +| 8 | role | 是 | 否 | 由 supportVnode 决定是否能够创建 | +| 9 | dnodeNopLoop | 是 | 否 | 2.6 文档中未找到此参数 | +| 10 | keepTimeOffset | 是 | 否 | 2.6 文档中未找到此参数 | +| 11 | rpcTimer | 是 | 否 | 3.0 行为未知 | +| 12 | rpcMaxTime | 是 | 否 | 3.0 行为未知 | +| 13 | rpcForceTcp | 是 | 否 | 默认为 TCP | +| 14 | tcpConnTimeout | 是 | 否 | 3.0 行为未知 | +| 15 | syncCheckInterval | 是 | 否 | 3.0 行为未知 | +| 16 | maxTmrCtrl | 是 | 否 | 3.0 行为未知 | +| 17 | monitorReplica | 是 | 否 | 由 RAFT 协议管理多副本 | +| 18 | smlTagNullName | 是 | 否 | 3.0 行为未知 | +| 19 | keepColumnName | 是 | 否 | 3.0 行为未知 | +| 20 | ratioOfQueryCores | 是 | 否 | 由 线程池 相关配置参数决定 | +| 21 | maxStreamCompDelay | 是 | 否 | 3.0 行为未知 | +| 22 | maxFirstStreamCompDelay | 是 | 否 | 3.0 行为未知 | +| 23 | retryStreamCompDelay | 是 | 否 | 3.0 行为未知 | +| 24 | streamCompDelayRatio | 是 | 否 | 3.0 行为未知 | +| 25 | maxVgroupsPerDb | 是 | 否 | 由 create db 的参数 vgroups 指定实际 vgroups 数量 | +| 26 | maxTablesPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | +| 27 | minTablesPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | +| 28 | tableIncStepPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | +| 29 | cache | 是 | 否 | 由 buffer 代替 cache\*blocks | +| 30 | blocks | 是 | 否 | 由 buffer 代替 cache\*blocks | +| 31 | days | 是 | 否 | 由 create db 的参数 duration 取代 | +| 32 | keep | 是 | 否 | 由 create db 的参数 keep 取代 | +| 33 | minRows | 是 | 否 | 由 create db 的参数 minRows 取代 | +| 34 | maxRows | 是 | 否 | 由 create db 的参数 maxRows 取代 | +| 35 | quorum | 是 | 否 | 由 RAFT 协议决定 | +| 36 | comp | 是 | 否 | 由 create db 的参数 comp 取代 | +| 37 | walLevel | 是 | 否 | 由 create db 的参数 wal_level 取代 | +| 38 | fsync | 是 | 否 | 由 create db 的参数 wal_fsync_period 取代 | +| 39 | replica | 是 | 否 | 由 create db 的参数 replica 取代 | +| 40 | partitions | 是 | 否 | 3.0 行为未知 | +| 41 | update | 是 | 否 | 允许更新部分列 | +| 42 | cachelast | 是 | 否 | 由 create db 的参数 cacheModel 取代 | +| 43 | maxSQLLength | 是 | 否 | SQL 上限为 1MB，无需参数控制 | +| 44 | maxWildCardsLength | 是 | 否 | 3.0 行为未知 | +| 45 | maxRegexStringLen | 是 | 否 | 3.0 行为未知 | +| 46 | maxNumOfOrderedRes | 是 | 否 | 3.0 行为未知 | +| 47 | maxConnections | 是 | 否 | 取决于系统配置和系统处理能力，详见后面的 Note | +| 48 | mnodeEqualVnodeNum | 是 | 否 | 3.0 行为未知 | +| 49 | http | 是 | 否 | http 服务由 taosAdapter 提供 | +| 50 | httpEnableRecordSql | 是 | 否 | taosd 不提供 http 服务 | +| 51 | httpMaxThreads | 是 | 否 | taosd 不提供 http 服务 | +| 52 | restfulRowLimit | 是 | 否 | taosd 不提供 http 服务 | +| 53 | httpDbNameMandatory | 是 | 否 | taosd 不提供 http 服务 | +| 54 | httpKeepAlive | 是 | 否 | taosd 不提供 http 服务 | +| 55 | enableRecordSql | 是 | 否 | 3.0 行为未知 | +| 56 | maxBinaryDisplayWidth | 是 | 否 | 3.0 行为未知 | +| 57 | stream | 是 | 否 | 默认启用连续查询 | +| 58 | retrieveBlockingModel | 是 | 否 | 3.0 行为未知 | +| 59 | tsdbMetaCompactRatio | 是 | 否 | 3.0 行为未知 | +| 60 | defaultJSONStrType | 是 | 否 | 3.0 行为未知 | +| 61 | walFlushSize | 是 | 否 | 3.0 行为未知 | +| 62 | keepTimeOffset | 是 | 否 | 3.0 行为未知 | +| 63 | flowctrl | 是 | 否 | 3.0 行为未知 | +| 64 | slaveQuery | 是 | 否 | 3.0 行为未知： slave vnode 是否能够处理查询？ | +| 65 | adjustMaster | 是 | 否 | 3.0 行为未知 | +| 66 | topicBinaryLen | 是 | 否 | 3.0 行为未知 | +| 67 | telegrafUseFieldNum | 是 | 否 | 3.0 行为未知 | +| 68 | deadLockKillQuery | 是 | 否 | 3.0 行为未知 | +| 69 | clientMerge | 是 | 否 | 3.0 行为未知 | +| 70 | sdbDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 71 | odbcDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 72 | httpDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 73 | monDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 74 | cqDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 75 | shortcutFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 76 | probeSeconds | 是 | 否 | 3.0 行为未知 | +| 77 | probeKillSeconds | 是 | 否 | 3.0 行为未知 | +| 78 | probeInterval | 是 | 否 | 3.0 行为未知 | +| 79 | lossyColumns | 是 | 否 | 3.0 行为未知 | +| 80 | fPrecision | 是 | 否 | 3.0 行为未知 | +| 81 | dPrecision | 是 | 否 | 3.0 行为未知 | +| 82 | maxRange | 是 | 否 | 3.0 行为未知 | +| 83 | range | 是 | 否 | 3.0 行为未知 | diff --git a/docs/zh/27-train-faq/01-faq.md b/docs/zh/27-train-faq/01-faq.md index 595b69b08b87ee33e27937fb89b84adc41c89d08..c6ecbe471a5c0863bac80ed6edfa6abd1e13c010 100644 --- a/docs/zh/27-train-faq/01-faq.md +++ b/docs/zh/27-train-faq/01-faq.md @@ -243,3 +243,8 @@ sudo launchctl load -w /Library/LaunchDaemons/limit.maxfiles.plist ``` launchctl limit maxfiles ``` +### 19 建库时提示Out of dnode +该提示是创建db的vnode数量不够了，需要的vnode不能超过了dnode中vnode的上限。因为系统默认是一个dnode中有cpu核数两倍的vnode，也可以通过配置文件中的参数supportVnodes控制。 +正常调大taos.cfg种这个supportVnodes参数即可。 + + diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index 7ed9e0c5a018401e2028a1e0786459d4e26f27b6..29403f3874970ad32431ae54446ce08350959e5c 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,11 +10,26 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.0.2.3 + + + +## 3.0.2.2 + + + +## 3.0.2.1 + + + +## 3.0.2.0 + + + ## 3.0.1.8 - ## 3.0.1.7 diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md index 67ca3fae67b36e5f08c57440bbaa64ec4f80bf4e..f10d97ebb9e2a316e55963c8c6efa81ecf72e3c2 100644 --- a/docs/zh/28-releases/02-tools.md +++ b/docs/zh/28-releases/02-tools.md @@ -10,6 +10,22 @@ taosTools 各版本安装包下载链接如下： import Release from "/components/ReleaseV3"; +## 2.4.1 + + + +## 2.4.0 + + + +## 2.3.3 + + + +## 2.3.2 + + + ## 2.3.0 diff --git a/include/common/taosdef.h b/include/common/taosdef.h index bf4de9d4ded1d0955bef05b1e3000be0bf34d8aa..d1ca446904594fa57d54031d5b39e529795d70da 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -30,6 +30,11 @@ typedef int64_t tb_uid_t; #define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX)) #define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) +//define show cluster alive and show db.alive +#define SHOW_STATUS_NOT_AVAILABLE 0 +#define SHOW_STATUS_AVAILABLE 1 +#define SHOW_STATUS_HALF_AVAILABLE 2 + typedef enum { TSDB_SUPER_TABLE = 1, // super table TSDB_CHILD_TABLE = 2, // table created from super table diff --git a/include/common/tcommon.h b/include/common/tcommon.h index f74795a250e66298ff08a1a1aa97d37f740a1942..c6e21af644d1054ce91d31b137fac030abcc749a 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -162,6 +162,7 @@ typedef enum EStreamType { STREAM_PULL_DATA, STREAM_PULL_OVER, STREAM_FILL_OVER, + STREAM_CREATE_CHILD_TABLE, } EStreamType; #pragma pack(push, 1) @@ -195,7 +196,7 @@ typedef struct SDataBlockInfo { uint32_t capacity; SBlockID id; int16_t hasVarCol; - int16_t dataLoad; // denote if the data is loaded or not + int16_t dataLoad; // denote if the data is loaded or not // TODO: optimize and remove following int64_t version; // used for stream, and need serialization @@ -204,8 +205,7 @@ typedef struct SDataBlockInfo { STimeWindow calWin; // used for stream, do not serialize TSKEY watermark; // used for stream - char parTbName[TSDB_TABLE_NAME_LEN]; // used for stream partition - STag* pTag; // used for stream partition + char parTbName[TSDB_TABLE_NAME_LEN]; // used for stream partition } SDataBlockInfo; typedef struct SSDataBlock { @@ -239,22 +239,22 @@ typedef struct SVarColAttr { // pBlockAgg->numOfNull == info.rows, all data are null // pBlockAgg->numOfNull == 0, no data are null. typedef struct SColumnInfoData { - char* pData; // the corresponding block data in memory + char* pData; // the corresponding block data in memory union { char* nullbitmap; // bitmap, one bit for each item in the list SVarColAttr varmeta; }; - SColumnInfo info; // column info - bool hasNull; // if current column data has null value. + SColumnInfo info; // column info + bool hasNull; // if current column data has null value. } SColumnInfoData; typedef struct SQueryTableDataCond { uint64_t suid; - int32_t order; // desc|asc order to iterate the data block + int32_t order; // desc|asc order to iterate the data block int32_t numOfCols; SColumnInfo* colList; - int32_t* pSlotList; // the column output destation slot, and it may be null - int32_t type; // data block load type: + int32_t* pSlotList; // the column output destation slot, and it may be null + int32_t type; // data block load type: STimeWindow twindows; int64_t startVersion; int64_t endVersion; @@ -291,6 +291,7 @@ typedef struct STableBlockDistInfo { uint16_t numOfFiles; uint32_t numOfTables; uint32_t numOfBlocks; + uint32_t numOfVgroups; uint64_t totalSize; uint64_t totalRows; int32_t maxRows; @@ -340,7 +341,7 @@ typedef struct SExprInfo { typedef struct { const char* key; - int32_t keyLen; + size_t keyLen; uint8_t type; union { const char* value; @@ -349,7 +350,7 @@ typedef struct { double d; float f; }; - int32_t length; + size_t length; } SSmlKv; #define QUERY_ASC_FORWARD_STEP 1 @@ -377,6 +378,11 @@ typedef struct SSortExecInfo { #define CALCULATE_END_TS_COLUMN_INDEX 5 #define TABLE_NAME_COLUMN_INDEX 6 +// stream create table block column +#define UD_TABLE_NAME_COLUMN_INDEX 0 +#define UD_GROUPID_COLUMN_INDEX 1 +#define UD_TAG_COLUMN_INDEX 2 + #ifdef __cplusplus } #endif diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 9c5b712db6c5daaaa3303c73a73b24ab33e8a9ff..20ffb48ab0173c8f811b2ab620d977e96256425d 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -271,7 +271,7 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag); // for debug char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf); -int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataBlocks, STSchema* pTSchema, int32_t vgId, +int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pDataBlocks, const STSchema* pTSchema, int64_t uid, int32_t vgId, tb_uid_t suid); char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 6855287fb2d354f52998342d368c46b3a852bf89..e0aacbfec9db8804b4003417689f404b78549afb 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -44,18 +44,38 @@ typedef struct SColData SColData; #define HAS_VALUE ((uint8_t)0x4) // bitmap ================================ -const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, - {0b00000000, 0b00000100, 0b00001000, 2}, - {0b00000000, 0b00010000, 0b00100000, 4}, - {0b00000000, 0b01000000, 0b10000000, 6}}; - -#define N1(n) ((((uint8_t)1) << (n)) - 1) -#define BIT1_SIZE(n) ((((n)-1) >> 3) + 1) -#define BIT2_SIZE(n) ((((n)-1) >> 2) + 1) -#define SET_BIT1(p, i, v) ((p)[(i) >> 3] = (p)[(i) >> 3] & N1((i)&7) | (((uint8_t)(v)) << ((i)&7))) -#define GET_BIT1(p, i) (((p)[(i) >> 3] >> ((i)&7)) & ((uint8_t)1)) -#define SET_BIT2(p, i, v) ((p)[(i) >> 2] = (p)[(i) >> 2] & N1(BIT2_MAP[(i)&3][3]) | BIT2_MAP[(i)&3][(v)]) -#define GET_BIT2(p, i) (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3)) +const static uint8_t BIT1_MAP[8] = {0b11111110, 0b11111101, 0b11111011, 0b11110111, + 0b11101111, 0b11011111, 0b10111111, 0b01111111}; + +const static uint8_t BIT2_MAP[4] = {0b11111100, 0b11110011, 0b11001111, 0b00111111}; + +#define ONE ((uint8_t)1) +#define THREE ((uint8_t)3) +#define DIV_8(i) ((i) >> 3) +#define MOD_8(i) ((i)&7) +#define DIV_4(i) ((i) >> 2) +#define MOD_4(i) ((i)&3) +#define MOD_4_TIME_2(i) (MOD_4(i) << 1) +#define BIT1_SIZE(n) (DIV_8((n)-1) + 1) +#define BIT2_SIZE(n) (DIV_4((n)-1) + 1) +#define SET_BIT1(p, i, v) ((p)[DIV_8(i)] = (p)[DIV_8(i)] & BIT1_MAP[MOD_8(i)] | ((v) << MOD_8(i))) +#define SET_BIT1_EX(p, i, v) \ + do { \ + if (MOD_8(i) == 0) { \ + (p)[DIV_8(i)] = 0; \ + } \ + SET_BIT1(p, i, v); \ + } while (0) +#define GET_BIT1(p, i) (((p)[DIV_8(i)] >> MOD_8(i)) & ONE) +#define SET_BIT2(p, i, v) ((p)[DIV_4(i)] = (p)[DIV_4(i)] & BIT2_MAP[MOD_4(i)] | ((v) << MOD_4_TIME_2(i))) +#define SET_BIT2_EX(p, i, v) \ + do { \ + if (MOD_4(i) == 0) { \ + (p)[DIV_4(i)] = 0; \ + } \ + SET_BIT2(p, i, v); \ + } while (0) +#define GET_BIT2(p, i) (((p)[DIV_4(i)] >> MOD_4_TIME_2(i)) & THREE) // SBuffer ================================ struct SBuffer { @@ -70,9 +90,6 @@ int32_t tBufferInit(SBuffer *pBuffer, int64_t size); int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData); int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData); -// STSchema ================================ -void tDestroyTSchema(STSchema *pTSchema); - // SColVal ================================ #define CV_FLAG_VALUE ((int8_t)0x0) #define CV_FLAG_NONE ((int8_t)0x1) @@ -87,8 +104,12 @@ void tDestroyTSchema(STSchema *pTSchema); #define COL_VAL_IS_VALUE(CV) ((CV)->flag == CV_FLAG_VALUE) // SRow ================================ -int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer); +int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow); void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); +void tRowDestroy(SRow *pRow); +void tRowSort(SArray *aRowP); +int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag); +int32_t tRowAppendToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData); // SRowIter ================================ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter); @@ -110,15 +131,28 @@ void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remov int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf); // SColData ================================ +typedef void *(*xMallocFn)(void *, int32_t); void tColDataDestroy(void *ph); void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn); void tColDataClear(SColData *pColData); +void tColDataDeepClear(SColData *pColData); int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal); void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal); uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal); -int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest); +int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMalloc, void *arg); extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull); +// for stmt bind +int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind); +void tColDataSortMerge(SArray *colDataArr); + +//for raw block +int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t bytes, + int32_t nRows, char* lengthOrbitmap, char *data); +// for encode/decode +int32_t tPutColData(uint8_t *pBuf, SColData *pColData); +int32_t tGetColData(uint8_t *pBuf, SColData *pColData); + // STRUCT ================================ struct STColumn { col_id_t colId; @@ -225,23 +259,9 @@ struct STag { memcpy(varDataVal(x), (str), (_size)); \ } while (0); -// ----------------- SCHEMA BUILDER DEFINITION -typedef struct { - int32_t tCols; - int32_t nCols; - schema_ver_t version; - uint16_t flen; - int32_t tlen; - STColumn *columns; -} STSchemaBuilder; - -int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); -void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); -void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); -int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes); -STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); - +// STSchema ================================ STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version); +void tDestroyTSchema(STSchema *pTSchema); #endif diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ad6077db098b18d2b10d95058831d4f8c25d046a..cf57165e5453789cea553a6fc31c4fec53eca725 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -343,7 +343,8 @@ void tFreeSSubmitRsp(SSubmitRsp* pRsp); #define COL_IS_SET(FLG) (((FLG) & (COL_SET_VAL | COL_SET_NULL)) != 0) #define COL_CLR_SET(FLG) ((FLG) &= (~(COL_SET_VAL | COL_SET_NULL))) -#define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON) +#define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON) +#define IS_SET_NULL(s) (((s)->flags & COL_SET_NULL) == COL_SET_NULL) #define SSCHMEA_TYPE(s) ((s)->type) #define SSCHMEA_FLAGS(s) ((s)->flags) @@ -482,8 +483,6 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapperEx(SDecoder* pDecoder, SSchemaW return 0; } -STSchema* tdGetSTSChemaFromSSChema(SSchema* pSchema, int32_t nCols, int32_t sver); - typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igExists; @@ -1734,6 +1733,8 @@ typedef struct { int32_t execId; } STaskDropReq; +int32_t tSerializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); +int32_t tDeserializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); int32_t tSerializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); int32_t tDeserializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); @@ -1751,6 +1752,8 @@ typedef struct { #define STREAM_FILL_HISTORY_ON 1 #define STREAM_FILL_HISTORY_OFF 0 #define STREAM_DEFAULT_FILL_HISTORY STREAM_FILL_HISTORY_OFF +#define STREAM_CREATE_STABLE_TRUE 1 +#define STREAM_CREATE_STABLE_FALSE 0 typedef struct { char name[TSDB_STREAM_FNAME_LEN]; @@ -1768,6 +1771,10 @@ typedef struct { SArray* pTags; // array of SField // 3.0.20 int64_t checkpointFreq; // ms + // 3.0.2.3 + int8_t createStb; + uint64_t targetStbUid; + SArray* fillNullCols; } SCMCreateStreamReq; typedef struct { @@ -2081,10 +2088,15 @@ typedef struct SVCreateTbReq { }; } SVCreateTbReq; -int tEncodeSVCreateTbReq(SEncoder* pCoder, const SVCreateTbReq* pReq); -int tDecodeSVCreateTbReq(SDecoder* pCoder, SVCreateTbReq* pReq); +int tEncodeSVCreateTbReq(SEncoder* pCoder, const SVCreateTbReq* pReq); +int tDecodeSVCreateTbReq(SDecoder* pCoder, SVCreateTbReq* pReq); +void tDestroySVCreateTbReq(SVCreateTbReq* pReq, int32_t flags); static FORCE_INLINE void tdDestroySVCreateTbReq(SVCreateTbReq* req) { + if (NULL == req) { + return; + } + taosMemoryFreeClear(req->name); taosMemoryFreeClear(req->comment); if (req->type == TSDB_CHILD_TABLE) { @@ -3232,6 +3244,57 @@ int32_t tSerializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); int32_t tDeserializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); int32_t tSerializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); int32_t tDeserializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); +int32_t tSerializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); +int32_t tDeserializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); +int32_t tSerializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); +int32_t tDeserializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); + +#define SUBMIT_REQ_AUTO_CREATE_TABLE 0x1 +#define SUBMIT_REQ_COLUMN_DATA_FORMAT 0x2 + +typedef struct { + int32_t flags; + SVCreateTbReq* pCreateTbReq; + int64_t suid; + int64_t uid; + int32_t sver; + union { + SArray* aRowP; + SArray* aCol; + }; +} SSubmitTbData; + +typedef struct { + SArray* aSubmitTbData; // SArray +} SSubmitReq2; + +int32_t tEncodeSSubmitReq2(SEncoder* pCoder, const SSubmitReq2* pReq); +int32_t tDecodeSSubmitReq2(SDecoder* pCoder, SSubmitReq2* pReq); +void tDestroySSubmitTbData(SSubmitTbData* pTbData, int32_t flag); +void tDestroySSubmitReq2(SSubmitReq2* pReq, int32_t flag); + +typedef struct { + int32_t affectedRows; + SArray* aCreateTbRsp; // SArray +} SSubmitRsp2; + +int32_t tEncodeSSubmitRsp2(SEncoder* pCoder, const SSubmitRsp2* pRsp); +int32_t tDecodeSSubmitRsp2(SDecoder* pCoder, SSubmitRsp2* pRsp); +void tDestroySSubmitRsp2(SSubmitRsp2* pRsp, int32_t flag); + +#define TSDB_MSG_FLG_ENCODE 0x1 +#define TSDB_MSG_FLG_DECODE 0x2 + +typedef struct { + union { + struct { + void* msgStr; + int32_t msgLen; + int64_t ver; + }; + void* pDataBlock; + }; +} SPackedData; #pragma pack(pop) diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 0192bbf486270d81de097ff64eb3877b88172e18..597371d9d1a50fce133f8a78c16a72d031119978 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -16,327 +16,331 @@ #ifndef _TD_COMMON_TOKEN_H_ #define _TD_COMMON_TOKEN_H_ -#define TK_OR 1 -#define TK_AND 2 -#define TK_UNION 3 -#define TK_ALL 4 -#define TK_MINUS 5 -#define TK_EXCEPT 6 -#define TK_INTERSECT 7 -#define TK_NK_BITAND 8 -#define TK_NK_BITOR 9 -#define TK_NK_LSHIFT 10 -#define TK_NK_RSHIFT 11 -#define TK_NK_PLUS 12 -#define TK_NK_MINUS 13 -#define TK_NK_STAR 14 -#define TK_NK_SLASH 15 -#define TK_NK_REM 16 -#define TK_NK_CONCAT 17 -#define TK_CREATE 18 -#define TK_ACCOUNT 19 -#define TK_NK_ID 20 -#define TK_PASS 21 -#define TK_NK_STRING 22 -#define TK_ALTER 23 -#define TK_PPS 24 -#define TK_TSERIES 25 -#define TK_STORAGE 26 -#define TK_STREAMS 27 -#define TK_QTIME 28 -#define TK_DBS 29 -#define TK_USERS 30 -#define TK_CONNS 31 -#define TK_STATE 32 -#define TK_USER 33 -#define TK_ENABLE 34 -#define TK_NK_INTEGER 35 -#define TK_SYSINFO 36 -#define TK_DROP 37 -#define TK_GRANT 38 -#define TK_ON 39 -#define TK_TO 40 -#define TK_REVOKE 41 -#define TK_FROM 42 -#define TK_SUBSCRIBE 43 -#define TK_NK_COMMA 44 -#define TK_READ 45 -#define TK_WRITE 46 -#define TK_NK_DOT 47 -#define TK_DNODE 48 -#define TK_PORT 49 -#define TK_DNODES 50 -#define TK_NK_IPTOKEN 51 -#define TK_FORCE 52 -#define TK_LOCAL 53 -#define TK_QNODE 54 -#define TK_BNODE 55 -#define TK_SNODE 56 -#define TK_MNODE 57 -#define TK_DATABASE 58 -#define TK_USE 59 -#define TK_FLUSH 60 -#define TK_TRIM 61 -#define TK_IF 62 -#define TK_NOT 63 -#define TK_EXISTS 64 -#define TK_BUFFER 65 -#define TK_CACHEMODEL 66 -#define TK_CACHESIZE 67 -#define TK_COMP 68 -#define TK_DURATION 69 -#define TK_NK_VARIABLE 70 -#define TK_MAXROWS 71 -#define TK_MINROWS 72 -#define TK_KEEP 73 -#define TK_PAGES 74 -#define TK_PAGESIZE 75 -#define TK_TSDB_PAGESIZE 76 -#define TK_PRECISION 77 -#define TK_REPLICA 78 -#define TK_VGROUPS 79 -#define TK_SINGLE_STABLE 80 -#define TK_RETENTIONS 81 -#define TK_SCHEMALESS 82 -#define TK_WAL_LEVEL 83 -#define TK_WAL_FSYNC_PERIOD 84 -#define TK_WAL_RETENTION_PERIOD 85 -#define TK_WAL_RETENTION_SIZE 86 -#define TK_WAL_ROLL_PERIOD 87 -#define TK_WAL_SEGMENT_SIZE 88 -#define TK_STT_TRIGGER 89 -#define TK_TABLE_PREFIX 90 -#define TK_TABLE_SUFFIX 91 -#define TK_NK_COLON 92 -#define TK_MAX_SPEED 93 -#define TK_TABLE 94 -#define TK_NK_LP 95 -#define TK_NK_RP 96 -#define TK_STABLE 97 -#define TK_ADD 98 -#define TK_COLUMN 99 -#define TK_MODIFY 100 -#define TK_RENAME 101 -#define TK_TAG 102 -#define TK_SET 103 -#define TK_NK_EQ 104 -#define TK_USING 105 -#define TK_TAGS 106 -#define TK_COMMENT 107 -#define TK_BOOL 108 -#define TK_TINYINT 109 -#define TK_SMALLINT 110 -#define TK_INT 111 -#define TK_INTEGER 112 -#define TK_BIGINT 113 -#define TK_FLOAT 114 -#define TK_DOUBLE 115 -#define TK_BINARY 116 -#define TK_TIMESTAMP 117 -#define TK_NCHAR 118 -#define TK_UNSIGNED 119 -#define TK_JSON 120 -#define TK_VARCHAR 121 -#define TK_MEDIUMBLOB 122 -#define TK_BLOB 123 -#define TK_VARBINARY 124 -#define TK_DECIMAL 125 -#define TK_MAX_DELAY 126 -#define TK_WATERMARK 127 -#define TK_ROLLUP 128 -#define TK_TTL 129 -#define TK_SMA 130 -#define TK_DELETE_MARK 131 -#define TK_FIRST 132 -#define TK_LAST 133 -#define TK_SHOW 134 -#define TK_PRIVILEGES 135 -#define TK_DATABASES 136 -#define TK_TABLES 137 -#define TK_STABLES 138 -#define TK_MNODES 139 -#define TK_QNODES 140 -#define TK_FUNCTIONS 141 -#define TK_INDEXES 142 -#define TK_ACCOUNTS 143 -#define TK_APPS 144 -#define TK_CONNECTIONS 145 -#define TK_LICENCES 146 -#define TK_GRANTS 147 -#define TK_QUERIES 148 -#define TK_SCORES 149 -#define TK_TOPICS 150 -#define TK_VARIABLES 151 -#define TK_CLUSTER 152 -#define TK_BNODES 153 -#define TK_SNODES 154 -#define TK_TRANSACTIONS 155 -#define TK_DISTRIBUTED 156 -#define TK_CONSUMERS 157 -#define TK_SUBSCRIPTIONS 158 -#define TK_VNODES 159 -#define TK_LIKE 160 -#define TK_TBNAME 161 -#define TK_QTAGS 162 -#define TK_AS 163 -#define TK_INDEX 164 -#define TK_FUNCTION 165 -#define TK_INTERVAL 166 -#define TK_TOPIC 167 -#define TK_WITH 168 -#define TK_META 169 -#define TK_CONSUMER 170 -#define TK_GROUP 171 -#define TK_DESC 172 -#define TK_DESCRIBE 173 -#define TK_RESET 174 -#define TK_QUERY 175 -#define TK_CACHE 176 -#define TK_EXPLAIN 177 -#define TK_ANALYZE 178 -#define TK_VERBOSE 179 -#define TK_NK_BOOL 180 -#define TK_RATIO 181 -#define TK_NK_FLOAT 182 -#define TK_OUTPUTTYPE 183 -#define TK_AGGREGATE 184 -#define TK_BUFSIZE 185 -#define TK_STREAM 186 -#define TK_INTO 187 -#define TK_TRIGGER 188 -#define TK_AT_ONCE 189 -#define TK_WINDOW_CLOSE 190 -#define TK_IGNORE 191 -#define TK_EXPIRED 192 -#define TK_FILL_HISTORY 193 -#define TK_SUBTABLE 194 -#define TK_KILL 195 -#define TK_CONNECTION 196 -#define TK_TRANSACTION 197 -#define TK_BALANCE 198 -#define TK_VGROUP 199 -#define TK_MERGE 200 -#define TK_REDISTRIBUTE 201 -#define TK_SPLIT 202 -#define TK_DELETE 203 -#define TK_INSERT 204 -#define TK_NULL 205 -#define TK_NK_QUESTION 206 -#define TK_NK_ARROW 207 -#define TK_ROWTS 208 -#define TK_QSTART 209 -#define TK_QEND 210 -#define TK_QDURATION 211 -#define TK_WSTART 212 -#define TK_WEND 213 -#define TK_WDURATION 214 -#define TK_IROWTS 215 -#define TK_CAST 216 -#define TK_NOW 217 -#define TK_TODAY 218 -#define TK_TIMEZONE 219 -#define TK_CLIENT_VERSION 220 -#define TK_SERVER_VERSION 221 -#define TK_SERVER_STATUS 222 -#define TK_CURRENT_USER 223 -#define TK_COUNT 224 -#define TK_LAST_ROW 225 -#define TK_CASE 226 -#define TK_END 227 -#define TK_WHEN 228 -#define TK_THEN 229 -#define TK_ELSE 230 -#define TK_BETWEEN 231 -#define TK_IS 232 -#define TK_NK_LT 233 -#define TK_NK_GT 234 -#define TK_NK_LE 235 -#define TK_NK_GE 236 -#define TK_NK_NE 237 -#define TK_MATCH 238 -#define TK_NMATCH 239 -#define TK_CONTAINS 240 -#define TK_IN 241 -#define TK_JOIN 242 -#define TK_INNER 243 -#define TK_SELECT 244 -#define TK_DISTINCT 245 -#define TK_WHERE 246 -#define TK_PARTITION 247 -#define TK_BY 248 -#define TK_SESSION 249 -#define TK_STATE_WINDOW 250 -#define TK_SLIDING 251 -#define TK_FILL 252 -#define TK_VALUE 253 -#define TK_NONE 254 -#define TK_PREV 255 -#define TK_LINEAR 256 -#define TK_NEXT 257 -#define TK_HAVING 258 -#define TK_RANGE 259 -#define TK_EVERY 260 -#define TK_ORDER 261 -#define TK_SLIMIT 262 -#define TK_SOFFSET 263 -#define TK_LIMIT 264 -#define TK_OFFSET 265 -#define TK_ASC 266 -#define TK_NULLS 267 -#define TK_ABORT 268 -#define TK_AFTER 269 -#define TK_ATTACH 270 -#define TK_BEFORE 271 -#define TK_BEGIN 272 -#define TK_BITAND 273 -#define TK_BITNOT 274 -#define TK_BITOR 275 -#define TK_BLOCKS 276 -#define TK_CHANGE 277 -#define TK_COMMA 278 -#define TK_COMPACT 279 -#define TK_CONCAT 280 -#define TK_CONFLICT 281 -#define TK_COPY 282 -#define TK_DEFERRED 283 -#define TK_DELIMITERS 284 -#define TK_DETACH 285 -#define TK_DIVIDE 286 -#define TK_DOT 287 -#define TK_EACH 288 -#define TK_FAIL 289 -#define TK_FILE 290 -#define TK_FOR 291 -#define TK_GLOB 292 -#define TK_ID 293 -#define TK_IMMEDIATE 294 -#define TK_IMPORT 295 -#define TK_INITIALLY 296 -#define TK_INSTEAD 297 -#define TK_ISNULL 298 -#define TK_KEY 299 -#define TK_MODULES 300 -#define TK_NK_BITNOT 301 -#define TK_NK_SEMI 302 -#define TK_NOTNULL 303 -#define TK_OF 304 -#define TK_PLUS 305 -#define TK_PRIVILEGE 306 -#define TK_RAISE 307 -#define TK_REPLACE 308 -#define TK_RESTRICT 309 -#define TK_ROW 310 -#define TK_SEMI 311 -#define TK_STAR 312 -#define TK_STATEMENT 313 -#define TK_STRICT 314 -#define TK_STRING 315 -#define TK_TIMES 316 -#define TK_UPDATE 317 -#define TK_VALUES 318 -#define TK_VARIABLE 319 -#define TK_VIEW 320 -#define TK_WAL 321 +#define TK_OR 1 +#define TK_AND 2 +#define TK_UNION 3 +#define TK_ALL 4 +#define TK_MINUS 5 +#define TK_EXCEPT 6 +#define TK_INTERSECT 7 +#define TK_NK_BITAND 8 +#define TK_NK_BITOR 9 +#define TK_NK_LSHIFT 10 +#define TK_NK_RSHIFT 11 +#define TK_NK_PLUS 12 +#define TK_NK_MINUS 13 +#define TK_NK_STAR 14 +#define TK_NK_SLASH 15 +#define TK_NK_REM 16 +#define TK_NK_CONCAT 17 +#define TK_CREATE 18 +#define TK_ACCOUNT 19 +#define TK_NK_ID 20 +#define TK_PASS 21 +#define TK_NK_STRING 22 +#define TK_ALTER 23 +#define TK_PPS 24 +#define TK_TSERIES 25 +#define TK_STORAGE 26 +#define TK_STREAMS 27 +#define TK_QTIME 28 +#define TK_DBS 29 +#define TK_USERS 30 +#define TK_CONNS 31 +#define TK_STATE 32 +#define TK_USER 33 +#define TK_ENABLE 34 +#define TK_NK_INTEGER 35 +#define TK_SYSINFO 36 +#define TK_DROP 37 +#define TK_GRANT 38 +#define TK_ON 39 +#define TK_TO 40 +#define TK_REVOKE 41 +#define TK_FROM 42 +#define TK_SUBSCRIBE 43 +#define TK_NK_COMMA 44 +#define TK_READ 45 +#define TK_WRITE 46 +#define TK_NK_DOT 47 +#define TK_DNODE 48 +#define TK_PORT 49 +#define TK_DNODES 50 +#define TK_NK_IPTOKEN 51 +#define TK_FORCE 52 +#define TK_LOCAL 53 +#define TK_QNODE 54 +#define TK_BNODE 55 +#define TK_SNODE 56 +#define TK_MNODE 57 +#define TK_DATABASE 58 +#define TK_USE 59 +#define TK_FLUSH 60 +#define TK_TRIM 61 +#define TK_IF 62 +#define TK_NOT 63 +#define TK_EXISTS 64 +#define TK_BUFFER 65 +#define TK_CACHEMODEL 66 +#define TK_CACHESIZE 67 +#define TK_COMP 68 +#define TK_DURATION 69 +#define TK_NK_VARIABLE 70 +#define TK_MAXROWS 71 +#define TK_MINROWS 72 +#define TK_KEEP 73 +#define TK_PAGES 74 +#define TK_PAGESIZE 75 +#define TK_TSDB_PAGESIZE 76 +#define TK_PRECISION 77 +#define TK_REPLICA 78 +#define TK_VGROUPS 79 +#define TK_SINGLE_STABLE 80 +#define TK_RETENTIONS 81 +#define TK_SCHEMALESS 82 +#define TK_WAL_LEVEL 83 +#define TK_WAL_FSYNC_PERIOD 84 +#define TK_WAL_RETENTION_PERIOD 85 +#define TK_WAL_RETENTION_SIZE 86 +#define TK_WAL_ROLL_PERIOD 87 +#define TK_WAL_SEGMENT_SIZE 88 +#define TK_STT_TRIGGER 89 +#define TK_TABLE_PREFIX 90 +#define TK_TABLE_SUFFIX 91 +#define TK_NK_COLON 92 +#define TK_MAX_SPEED 93 +#define TK_TABLE 94 +#define TK_NK_LP 95 +#define TK_NK_RP 96 +#define TK_STABLE 97 +#define TK_ADD 98 +#define TK_COLUMN 99 +#define TK_MODIFY 100 +#define TK_RENAME 101 +#define TK_TAG 102 +#define TK_SET 103 +#define TK_NK_EQ 104 +#define TK_USING 105 +#define TK_TAGS 106 +#define TK_COMMENT 107 +#define TK_BOOL 108 +#define TK_TINYINT 109 +#define TK_SMALLINT 110 +#define TK_INT 111 +#define TK_INTEGER 112 +#define TK_BIGINT 113 +#define TK_FLOAT 114 +#define TK_DOUBLE 115 +#define TK_BINARY 116 +#define TK_TIMESTAMP 117 +#define TK_NCHAR 118 +#define TK_UNSIGNED 119 +#define TK_JSON 120 +#define TK_VARCHAR 121 +#define TK_MEDIUMBLOB 122 +#define TK_BLOB 123 +#define TK_VARBINARY 124 +#define TK_DECIMAL 125 +#define TK_MAX_DELAY 126 +#define TK_WATERMARK 127 +#define TK_ROLLUP 128 +#define TK_TTL 129 +#define TK_SMA 130 +#define TK_DELETE_MARK 131 +#define TK_FIRST 132 +#define TK_LAST 133 +#define TK_SHOW 134 +#define TK_PRIVILEGES 135 +#define TK_DATABASES 136 +#define TK_TABLES 137 +#define TK_STABLES 138 +#define TK_MNODES 139 +#define TK_QNODES 140 +#define TK_FUNCTIONS 141 +#define TK_INDEXES 142 +#define TK_ACCOUNTS 143 +#define TK_APPS 144 +#define TK_CONNECTIONS 145 +#define TK_LICENCES 146 +#define TK_GRANTS 147 +#define TK_QUERIES 148 +#define TK_SCORES 149 +#define TK_TOPICS 150 +#define TK_VARIABLES 151 +#define TK_CLUSTER 152 +#define TK_BNODES 153 +#define TK_SNODES 154 +#define TK_TRANSACTIONS 155 +#define TK_DISTRIBUTED 156 +#define TK_CONSUMERS 157 +#define TK_SUBSCRIPTIONS 158 +#define TK_VNODES 159 +#define TK_ALIVE 160 +#define TK_LIKE 161 +#define TK_TBNAME 162 +#define TK_QTAGS 163 +#define TK_AS 164 +#define TK_INDEX 165 +#define TK_FUNCTION 166 +#define TK_INTERVAL 167 +#define TK_COUNT 168 +#define TK_LAST_ROW 169 +#define TK_TOPIC 170 +#define TK_WITH 171 +#define TK_META 172 +#define TK_CONSUMER 173 +#define TK_GROUP 174 +#define TK_DESC 175 +#define TK_DESCRIBE 176 +#define TK_RESET 177 +#define TK_QUERY 178 +#define TK_CACHE 179 +#define TK_EXPLAIN 180 +#define TK_ANALYZE 181 +#define TK_VERBOSE 182 +#define TK_NK_BOOL 183 +#define TK_RATIO 184 +#define TK_NK_FLOAT 185 +#define TK_OUTPUTTYPE 186 +#define TK_AGGREGATE 187 +#define TK_BUFSIZE 188 +#define TK_STREAM 189 +#define TK_INTO 190 +#define TK_TRIGGER 191 +#define TK_AT_ONCE 192 +#define TK_WINDOW_CLOSE 193 +#define TK_IGNORE 194 +#define TK_EXPIRED 195 +#define TK_FILL_HISTORY 196 +#define TK_SUBTABLE 197 +#define TK_KILL 198 +#define TK_CONNECTION 199 +#define TK_TRANSACTION 200 +#define TK_BALANCE 201 +#define TK_VGROUP 202 +#define TK_MERGE 203 +#define TK_REDISTRIBUTE 204 +#define TK_SPLIT 205 +#define TK_DELETE 206 +#define TK_INSERT 207 +#define TK_NULL 208 +#define TK_NK_QUESTION 209 +#define TK_NK_ARROW 210 +#define TK_ROWTS 211 +#define TK_QSTART 212 +#define TK_QEND 213 +#define TK_QDURATION 214 +#define TK_WSTART 215 +#define TK_WEND 216 +#define TK_WDURATION 217 +#define TK_IROWTS 218 +#define TK_ISFILLED 219 +#define TK_CAST 220 +#define TK_NOW 221 +#define TK_TODAY 222 +#define TK_TIMEZONE 223 +#define TK_CLIENT_VERSION 224 +#define TK_SERVER_VERSION 225 +#define TK_SERVER_STATUS 226 +#define TK_CURRENT_USER 227 +#define TK_CASE 228 +#define TK_END 229 +#define TK_WHEN 230 +#define TK_THEN 231 +#define TK_ELSE 232 +#define TK_BETWEEN 233 +#define TK_IS 234 +#define TK_NK_LT 235 +#define TK_NK_GT 236 +#define TK_NK_LE 237 +#define TK_NK_GE 238 +#define TK_NK_NE 239 +#define TK_MATCH 240 +#define TK_NMATCH 241 +#define TK_CONTAINS 242 +#define TK_IN 243 +#define TK_JOIN 244 +#define TK_INNER 245 +#define TK_SELECT 246 +#define TK_DISTINCT 247 +#define TK_WHERE 248 +#define TK_PARTITION 249 +#define TK_BY 250 +#define TK_SESSION 251 +#define TK_STATE_WINDOW 252 +#define TK_EVENT_WINDOW 253 +#define TK_START 254 +#define TK_SLIDING 255 +#define TK_FILL 256 +#define TK_VALUE 257 +#define TK_NONE 258 +#define TK_PREV 259 +#define TK_LINEAR 260 +#define TK_NEXT 261 +#define TK_HAVING 262 +#define TK_RANGE 263 +#define TK_EVERY 264 +#define TK_ORDER 265 +#define TK_SLIMIT 266 +#define TK_SOFFSET 267 +#define TK_LIMIT 268 +#define TK_OFFSET 269 +#define TK_ASC 270 +#define TK_NULLS 271 +#define TK_ABORT 272 +#define TK_AFTER 273 +#define TK_ATTACH 274 +#define TK_BEFORE 275 +#define TK_BEGIN 276 +#define TK_BITAND 277 +#define TK_BITNOT 278 +#define TK_BITOR 279 +#define TK_BLOCKS 280 +#define TK_CHANGE 281 +#define TK_COMMA 282 +#define TK_COMPACT 283 +#define TK_CONCAT 284 +#define TK_CONFLICT 285 +#define TK_COPY 286 +#define TK_DEFERRED 287 +#define TK_DELIMITERS 288 +#define TK_DETACH 289 +#define TK_DIVIDE 290 +#define TK_DOT 291 +#define TK_EACH 292 +#define TK_FAIL 293 +#define TK_FILE 294 +#define TK_FOR 295 +#define TK_GLOB 296 +#define TK_ID 297 +#define TK_IMMEDIATE 298 +#define TK_IMPORT 299 +#define TK_INITIALLY 300 +#define TK_INSTEAD 301 +#define TK_ISNULL 302 +#define TK_KEY 303 +#define TK_MODULES 304 +#define TK_NK_BITNOT 305 +#define TK_NK_SEMI 306 +#define TK_NOTNULL 307 +#define TK_OF 308 +#define TK_PLUS 309 +#define TK_PRIVILEGE 310 +#define TK_RAISE 311 +#define TK_REPLACE 312 +#define TK_RESTRICT 313 +#define TK_ROW 314 +#define TK_SEMI 315 +#define TK_STAR 316 +#define TK_STATEMENT 317 +#define TK_STRICT 318 +#define TK_STRING 319 +#define TK_TIMES 320 +#define TK_UPDATE 321 +#define TK_VALUES 322 +#define TK_VARIABLE 323 +#define TK_VIEW 324 +#define TK_WAL 325 #define TK_NK_SPACE 600 #define TK_NK_COMMENT 601 diff --git a/include/common/ttypes.h b/include/common/ttypes.h index b7061300c1760c0b557562924caa329080779bda..97ae151b7a8ccedb84dd113fa9c9560cfe5ffe14 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -266,6 +266,7 @@ typedef struct { #define IS_FLOAT_TYPE(_t) ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE) #define IS_INTEGER_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t))) #define IS_TIMESTAMP_TYPE(_t) ((_t) == TSDB_DATA_TYPE_TIMESTAMP) +#define IS_BOOLEAN_TYPE(_t) ((_t) == TSDB_DATA_TYPE_BOOL) #define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t))) #define IS_MATHABLE_TYPE(_t) \ diff --git a/include/libs/command/command.h b/include/libs/command/command.h index b3339a417ba463212c3abc163b57519194953c10..a8b1a0902acac276080d905554a3773f68e66b1c 100644 --- a/include/libs/command/command.h +++ b/include/libs/command/command.h @@ -22,7 +22,7 @@ typedef struct SExplainCtx SExplainCtx; -int32_t qExecCommand(bool sysInfoUser, SNode *pStmt, SRetrieveTableRsp **pRsp); +int32_t qExecCommand(int64_t* pConnId, bool sysInfoUser, SNode *pStmt, SRetrieveTableRsp **pRsp); int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp); int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int64_t startTs); diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 63e1c556deeae912c010c849218dec514e09be44..095d2f6d10dbe6faa0460f5532c569b543fb4ac0 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -192,7 +192,9 @@ int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts); int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType); -int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq); +// int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq, int64_t ver); +// +int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit); int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset); @@ -216,6 +218,7 @@ int32_t qStreamSourceRecoverStep2(qTaskInfo_t tinfo, int64_t ver); int32_t qStreamRecoverFinish(qTaskInfo_t tinfo); int32_t qStreamRestoreParam(qTaskInfo_t tinfo); bool qStreamRecoverScanFinished(qTaskInfo_t tinfo); +void qStreamCloseTsdbReader(void* task); #ifdef __cplusplus } diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 9ca6a7a9fafe464f1f82682b9e3599d15b6dff79..e98e341f15fdce600e5efa54a824b2ae29b8568e 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -120,6 +120,7 @@ typedef enum EFunctionType { FUNCTION_TYPE_WEND, FUNCTION_TYPE_WDURATION, FUNCTION_TYPE_IROWTS, + FUNCTION_TYPE_ISFILLED, FUNCTION_TYPE_TAGS, // internal function diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index b1054e7b859c8ba9231ee1c942f12d7c1ec6b97c..d77f2b62332ee071747676145b5dcba0b4e48371 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -40,6 +40,7 @@ extern "C" { #define SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE) #define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) +#define SHOW_ALIVE_RESULT_COLS 1 #define PRIVILEGE_TYPE_MASK(n) (1 << n) #define PRIVILEGE_TYPE_ALL PRIVILEGE_TYPE_MASK(0) @@ -171,10 +172,10 @@ typedef struct SCreateSubTableClause { STableOptions* pOptions; } SCreateSubTableClause; -typedef struct SCreateMultiTableStmt { +typedef struct SCreateMultiTablesStmt { ENodeType type; SNodeList* pSubTables; -} SCreateMultiTableStmt; +} SCreateMultiTablesStmt; typedef struct SDropTableClause { ENodeType type; @@ -209,14 +210,14 @@ typedef struct SAlterTableStmt { typedef struct SCreateUserStmt { ENodeType type; - char useName[TSDB_USER_LEN]; + char userName[TSDB_USER_LEN]; char password[TSDB_USET_PASSWORD_LEN]; int8_t sysinfo; } SCreateUserStmt; typedef struct SAlterUserStmt { ENodeType type; - char useName[TSDB_USER_LEN]; + char userName[TSDB_USER_LEN]; int8_t alterType; char password[TSDB_USET_PASSWORD_LEN]; int8_t enable; @@ -225,7 +226,7 @@ typedef struct SAlterUserStmt { typedef struct SDropUserStmt { ENodeType type; - char useName[TSDB_USER_LEN]; + char userName[TSDB_USER_LEN]; } SDropUserStmt; typedef struct SCreateDnodeStmt { @@ -262,6 +263,11 @@ typedef struct SShowCreateDatabaseStmt { void* pCfg; // SDbCfgInfo } SShowCreateDatabaseStmt; +typedef struct SShowAliveStmt { + ENodeType type; + char dbName[TSDB_DB_NAME_LEN]; +} SShowAliveStmt; + typedef struct SShowCreateTableStmt { ENodeType type; char dbName[TSDB_DB_NAME_LEN]; @@ -295,7 +301,7 @@ typedef struct SShowTableTagsStmt { SNodeList* pTags; } SShowTableTagsStmt; -typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT } EIndexType; +typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT, INDEX_TYPE_NORMAL } EIndexType; typedef struct SIndexOptions { ENodeType type; @@ -401,6 +407,7 @@ typedef struct SCreateStreamStmt { SNode* pQuery; SNodeList* pTags; SNode* pSubtable; + SNodeList* pCols; } SCreateStreamStmt; typedef struct SDropStreamStmt { diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 412054b13e3bdceec6234919040c4b99a43d6102..24b436ec994f903cd09c4c6bd918c813b192254d 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -112,11 +112,12 @@ typedef enum ENodeType { QUERY_NODE_COLUMN_REF, QUERY_NODE_WHEN_THEN, QUERY_NODE_CASE_WHEN, + QUERY_NODE_EVENT_WINDOW, // Statement nodes are used in parser and planner module. QUERY_NODE_SET_OPERATOR = 100, QUERY_NODE_SELECT_STMT, - QUERY_NODE_VNODE_MODIF_STMT, + QUERY_NODE_VNODE_MODIFY_STMT, QUERY_NODE_CREATE_DATABASE_STMT, QUERY_NODE_DROP_DATABASE_STMT, QUERY_NODE_ALTER_DATABASE_STMT, @@ -124,7 +125,7 @@ typedef enum ENodeType { QUERY_NODE_TRIM_DATABASE_STMT, QUERY_NODE_CREATE_TABLE_STMT, QUERY_NODE_CREATE_SUBTABLE_CLAUSE, - QUERY_NODE_CREATE_MULTI_TABLE_STMT, + QUERY_NODE_CREATE_MULTI_TABLES_STMT, QUERY_NODE_DROP_TABLE_CLAUSE, QUERY_NODE_DROP_TABLE_STMT, QUERY_NODE_DROP_SUPER_TABLE_STMT, @@ -207,6 +208,8 @@ typedef enum ENodeType { QUERY_NODE_DELETE_STMT, QUERY_NODE_INSERT_STMT, QUERY_NODE_QUERY, + QUERY_NODE_SHOW_DB_ALIVE_STMT, + QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT, // logic plan node QUERY_NODE_LOGIC_PLAN_SCAN = 1000, @@ -265,7 +268,9 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_DELETE, QUERY_NODE_PHYSICAL_SUBPLAN, QUERY_NODE_PHYSICAL_PLAN, - QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN + QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, + QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT, + QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT } ENodeType; /** diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index d8856651174662b6de6fbf3c05543200a00033a1..3602225789f5d2b5586e592646763e8f651f9dcf 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -186,7 +186,12 @@ typedef struct SMergeLogicNode { bool groupSort; } SMergeLogicNode; -typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType; +typedef enum EWindowType { + WINDOW_TYPE_INTERVAL = 1, + WINDOW_TYPE_SESSION, + WINDOW_TYPE_STATE, + WINDOW_TYPE_EVENT +} EWindowType; typedef enum EWindowAlgorithm { INTERVAL_ALGO_HASH = 1, @@ -213,6 +218,8 @@ typedef struct SWindowLogicNode { SNode* pTspk; SNode* pTsEnd; SNode* pStateExpr; + SNode* pStartCond; + SNode* pEndCond; int8_t triggerType; int64_t watermark; int64_t deleteMark; @@ -500,6 +507,14 @@ typedef struct SStateWinodwPhysiNode { typedef SStateWinodwPhysiNode SStreamStateWinodwPhysiNode; +typedef struct SEventWinodwPhysiNode { + SWinodwPhysiNode window; + SNode* pStartCond; + SNode* pEndCond; +} SEventWinodwPhysiNode; + +typedef SEventWinodwPhysiNode SStreamEventWinodwPhysiNode; + typedef struct SSortPhysiNode { SPhysiNode node; SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 48b98fcf3373ad6afec153bfd8845e171c42600d..2d143912474cb9ef769d2db915f24f110d37b1a2 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -223,6 +223,13 @@ typedef struct SIntervalWindowNode { SNode* pFill; } SIntervalWindowNode; +typedef struct SEventWindowNode { + ENodeType type; // QUERY_NODE_EVENT_WINDOW + SNode* pCol; // timestamp primary key + SNode* pStartCond; + SNode* pEndCond; +} SEventWindowNode; + typedef enum EFillMode { FILL_MODE_NONE = 1, FILL_MODE_VALUE, @@ -354,34 +361,34 @@ typedef struct SVgDataBlocks { void* pData; // SSubmitReq + SSubmitBlk + ... } SVgDataBlocks; -typedef void (*FFreeDataBlockHash)(SHashObj*); -typedef void (*FFreeDataBlockArray)(SArray*); - -typedef struct SVnodeModifOpStmt { - ENodeType nodeType; - ENodeType sqlNodeType; - SArray* pDataBlocks; // data block for each vgroup, SArray. - uint32_t insertType; // insert data from [file|sql statement| bound statement] - const char* pSql; // current sql statement position - int32_t totalRowsNum; - int32_t totalTbNum; - SName targetTableName; - SName usingTableName; - const char* pBoundCols; - struct STableMeta* pTableMeta; - SHashObj* pVgroupsHashObj; - SHashObj* pTableBlockHashObj; - SHashObj* pSubTableHashObj; - SHashObj* pTableNameHashObj; - SHashObj* pDbFNameHashObj; - SArray* pVgDataBlocks; - SVCreateTbReq createTblReq; - TdFilePtr fp; - FFreeDataBlockHash freeHashFunc; - FFreeDataBlockArray freeArrayFunc; - bool usingTableProcessing; - bool fileProcessing; -} SVnodeModifOpStmt; +typedef void (*FFreeTableBlockHash)(SHashObj*); +typedef void (*FFreeVgourpBlockArray)(SArray*); + +typedef struct SVnodeModifyOpStmt { + ENodeType nodeType; + ENodeType sqlNodeType; + SArray* pDataBlocks; // data block for each vgroup, SArray. + uint32_t insertType; // insert data from [file|sql statement| bound statement] + const char* pSql; // current sql statement position + int32_t totalRowsNum; + int32_t totalTbNum; + SName targetTableName; + SName usingTableName; + const char* pBoundCols; + struct STableMeta* pTableMeta; + SHashObj* pVgroupsHashObj; + SHashObj* pTableBlockHashObj; // SHashObj + SHashObj* pSubTableHashObj; + SHashObj* pTableNameHashObj; + SHashObj* pDbFNameHashObj; + SArray* pVgDataBlocks; // SArray + SVCreateTbReq* pCreateTblReq; + TdFilePtr fp; + FFreeTableBlockHash freeHashFunc; + FFreeVgourpBlockArray freeArrayFunc; + bool usingTableProcessing; + bool fileProcessing; +} SVnodeModifyOpStmt; typedef struct SExplainOptions { ENodeType type; diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 9be79a539f468d20ad28d2278651fbae169ff4a6..8f227459735f8c43f74b760b16b4b133c883c95b 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -58,7 +58,6 @@ typedef struct SParseContext { bool isSuperUser; bool enableSysInfo; bool async; - int8_t schemalessType; const char* svrVer; bool nodeOffline; SArray* pTableMetaPos; // sql table pos => catalog data pos @@ -85,12 +84,12 @@ int32_t qSetSTableIdForRsma(SNode* pStmt, int64_t uid); void qCleanupKeywordsTable(); int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash); -int32_t qResetStmtDataBlock(void* block, bool keepBuf); -int32_t qCloneStmtDataBlock(void** pDst, void* pSrc); -void qFreeStmtDataBlock(void* pDataBlock); -int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgId); -void qDestroyStmtDataBlock(void* pBlock); -STableMeta* qGetTableMetaInDataBlock(void* pDataBlock); +int32_t qResetStmtDataBlock(STableDataCxt* block, bool keepBuf); +int32_t qCloneStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, bool reset); +int32_t qRebuildStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_t uid, uint64_t suid, int32_t vgId, bool rebuildCreateTb); +void qDestroyStmtDataBlock(STableDataCxt* pBlock); +STableMeta* qGetTableMetaInDataBlock(STableDataCxt* pDataBlock); +int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData **pData); int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx); int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery); @@ -105,11 +104,18 @@ void destroyBoundColumnInfo(void* pBoundInfo); int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf, int32_t msgBufLen); -void* smlInitHandle(SQuery* pQuery); -void smlDestroyHandle(void* pHandle); -int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta, +void qDestroyBoundColInfo(void* pInfo); + +SQuery* smlInitHandle(); +int32_t smlBuildRow(STableDataCxt* pTableCxt); +int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void *kv, int32_t index); +STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta); + +int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int16_t msgBufLen); -int32_t smlBuildOutput(void* handle, SHashObj* pVgHash); +int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash); + +int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD *fields, int numFields); int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray); SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap); diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 5b640dce92052f1aef246f599c3f686a089c2ec7..57ddeb657cb6c583e8a97dc089b735d903f2347e 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -163,6 +163,23 @@ typedef struct STargetInfo { int32_t vgId; } STargetInfo; +typedef struct SBoundColInfo { + int16_t* pColIndex; // bound index => schema index + int32_t numOfCols; + int32_t numOfBound; +} SBoundColInfo; + +typedef struct STableDataCxt { + STableMeta* pMeta; + STSchema* pSchema; + SBoundColInfo boundColsInfo; + SArray* pValues; + SSubmitTbData* pData; + TSKEY lastTs; + bool ordered; + bool duplicateTs; +} STableDataCxt; + typedef int32_t (*__async_send_cb_fn_t)(void* param, SDataBuf* pMsg, int32_t code); typedef int32_t (*__async_exec_fn_t)(void* param); @@ -190,6 +207,12 @@ typedef struct SQueryNodeStat { int32_t tableNum; // vg table number, unit is TSDB_TABLE_NUM_UNIT } SQueryNodeStat; +typedef struct SColLocation { + int16_t slotId; + col_id_t colId; + int8_t type; +} SColLocation; + int32_t initTaskQueue(); int32_t cleanupTaskQueue(); @@ -238,6 +261,7 @@ int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t char* parseTagDatatoJson(void* p); int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst); +int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst); void freeVgInfo(SDBVgInfo* vgInfo); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen, diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 8fdac0da7fe3758a7ba335ee11c0709f2cf56e83..d7bc151eccd2a05610c1ac843859405f97ff8fd6 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -35,6 +35,7 @@ typedef struct STdbState { TTB* pFillStateDb; // todo refactor TTB* pSessionStateDb; TTB* pParNameDb; + TTB* pParTagDb; TXN* txn; } STdbState; @@ -108,6 +109,9 @@ int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname); int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal); +int32_t streamStatePutParTag(SStreamState* pState, int64_t groupId, const void* tag, int32_t tagLen); +int32_t streamStateGetParTag(SStreamState* pState, int64_t groupId, void** tagVal, int32_t* tagLen); + #if 0 char* streamStateSessionDump(SStreamState* pState); #endif diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index c00625c51c86dc96f414cb0f719970f9c1b99ead..0f57da2b804c7c58411ee7ffa2cbb1fc4d68b3e8 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -103,6 +103,7 @@ typedef struct { int8_t type; } SStreamQueueItem; +#if 0 typedef struct { int8_t type; int64_t ver; @@ -116,6 +117,21 @@ typedef struct { SArray* dataRefs; // SArray SArray* reqs; // SArray } SStreamMergedSubmit; +#endif + +typedef struct { + int8_t type; + int64_t ver; + int32_t* dataRef; + SPackedData submit; +} SStreamDataSubmit2; + +typedef struct { + int8_t type; + int64_t ver; + SArray* dataRefs; // SArray + SArray* submits; // SArray +} SStreamMergedSubmit2; typedef struct { int8_t type; @@ -219,11 +235,11 @@ static FORCE_INLINE void* streamQueueNextItem(SStreamQueue* queue) { } } -SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq); +SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit); -void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit); +void streamDataSubmitRefDec(SStreamDataSubmit2* pDataSubmit); -SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit); +SStreamDataSubmit2* streamSubmitRefClone(SStreamDataSubmit2* pSubmit); typedef struct { char* qmsg; @@ -355,14 +371,15 @@ void tFreeSStreamTask(SStreamTask* pTask); static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem* pItem) { if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit*)pItem); + SStreamDataSubmit2* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit2*)pItem); if (pSubmitClone == NULL) { qDebug("task %d %p submit enqueue failed since out of memory", pTask->taskId, pTask); terrno = TSDB_CODE_OUT_OF_MEMORY; atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); return -1; } - qDebug("task %d %p submit enqueue %p %p %p", pTask->taskId, pTask, pItem, pSubmitClone, pSubmitClone->data); + qDebug("task %d %p submit enqueue %p %p %p %d %" PRId64, pTask->taskId, pTask, pItem, pSubmitClone, + pSubmitClone->submit.msgStr, pSubmitClone->submit.msgLen, pSubmitClone->submit.ver); taosWriteQitem(pTask->inputQueue->queue, pSubmitClone); // qStreamInput(pTask->exec.executor, pSubmitClone); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE || diff --git a/include/os/os.h b/include/os/os.h index b27fa84406b843f15ba99d52f7d9fde8580a0899..809b814491e99724561af2356fe1128cfc86dc16 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -116,6 +116,7 @@ extern "C" { #include "osTimer.h" #include "osTimezone.h" #include "taoserror.h" +#include "tlog.h" #ifdef __cplusplus } diff --git a/include/util/tRealloc.h b/include/util/tRealloc.h index f3593d5818cd626c7268ec1b721bd961fba3a845..3229c53039f6dc857dd7e0b2d4774e6875ef8c16 100644 --- a/include/util/tRealloc.h +++ b/include/util/tRealloc.h @@ -52,11 +52,13 @@ _exit: return code; } -static FORCE_INLINE void tFree(uint8_t *pBuf) { - if (pBuf) { - taosMemoryFree(pBuf - sizeof(int64_t)); - } -} +#define tFree(BUF) \ + do { \ + if (BUF) { \ + taosMemoryFree((uint8_t *)(BUF) - sizeof(int64_t)); \ + (BUF) = NULL; \ + } \ + } while (0) #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 51f336c3aa174c0dfd311826d5bd340c319a6b8d..52d8a75ee06b10de97ebd488d3a9a2ee439bacea 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -158,6 +158,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_QUERY_KILLED TAOS_DEF_ERROR_CODE(0, 0X022D) #define TSDB_CODE_TSC_NO_EXEC_NODE TAOS_DEF_ERROR_CODE(0, 0X022E) #define TSDB_CODE_TSC_NOT_STABLE_ERROR TAOS_DEF_ERROR_CODE(0, 0X022F) +#define TSDB_CODE_TSC_STMT_CACHE_ERROR TAOS_DEF_ERROR_CODE(0, 0X0230) +#define TSDB_CODE_TSC_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0X0231) // mnode-common // #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) // 2.x @@ -413,6 +415,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_NO_AVAIL_BUFPOOL TAOS_DEF_ERROR_CODE(0, 0x0528) #define TSDB_CODE_VND_STOPPED TAOS_DEF_ERROR_CODE(0, 0x0529) #define TSDB_CODE_VND_DUP_REQUEST TAOS_DEF_ERROR_CODE(0, 0x0530) +#define TSDB_CODE_VND_QUERY_BUSY TAOS_DEF_ERROR_CODE(0, 0x0531) // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) @@ -699,6 +702,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SML_INVALID_DATA TAOS_DEF_ERROR_CODE(0, 0x3002) #define TSDB_CODE_SML_INVALID_DB_CONF TAOS_DEF_ERROR_CODE(0, 0x3003) #define TSDB_CODE_SML_NOT_SAME_TYPE TAOS_DEF_ERROR_CODE(0, 0x3004) +#define TSDB_CODE_SML_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x3005) //tsma #define TSDB_CODE_TSMA_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x3100) @@ -714,7 +718,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RSMA_INVALID_ENV TAOS_DEF_ERROR_CODE(0, 0x3150) #define TSDB_CODE_RSMA_INVALID_STAT TAOS_DEF_ERROR_CODE(0, 0x3151) #define TSDB_CODE_RSMA_QTASKINFO_CREATE TAOS_DEF_ERROR_CODE(0, 0x3152) -// #define TSDB_CODE_RSMA_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x3153) +#define TSDB_CODE_RSMA_FS_COMMIT TAOS_DEF_ERROR_CODE(0, 0x3153) #define TSDB_CODE_RSMA_REMOVE_EXISTS TAOS_DEF_ERROR_CODE(0, 0x3154) #define TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP TAOS_DEF_ERROR_CODE(0, 0x3155) #define TSDB_CODE_RSMA_EMPTY_INFO TAOS_DEF_ERROR_CODE(0, 0x3156) @@ -722,6 +726,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RSMA_REGEX_MATCH TAOS_DEF_ERROR_CODE(0, 0x3158) #define TSDB_CODE_RSMA_STREAM_STATE_OPEN TAOS_DEF_ERROR_CODE(0, 0x3159) #define TSDB_CODE_RSMA_STREAM_STATE_COMMIT TAOS_DEF_ERROR_CODE(0, 0x3160) +#define TSDB_CODE_RSMA_FS_REF TAOS_DEF_ERROR_CODE(0, 0x3161) +#define TSDB_CODE_RSMA_FS_SYNC TAOS_DEF_ERROR_CODE(0, 0x3162) +#define TSDB_CODE_RSMA_FS_UPDATE TAOS_DEF_ERROR_CODE(0, 0x3163) //index #define TSDB_CODE_INDEX_REBUILDING TAOS_DEF_ERROR_CODE(0, 0x3200) diff --git a/include/util/tarray.h b/include/util/tarray.h index 0e78397ecb7c35d75211e06170ea5cdb99030e34..f8e872ec66aa2caf38bf984d8b55ef4928526016 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -22,19 +22,6 @@ extern "C" { #endif -#if 0 -#define TARRAY(TYPE) \ - struct { \ - int32_t tarray_size_; \ - int32_t tarray_neles_; \ - struct TYPE* td_array_data_; \ - } - -#define TARRAY_SIZE(ARRAY) (ARRAY)->tarray_size_ -#define TARRAY_NELES(ARRAY) (ARRAY)->tarray_neles_ -#define TARRAY_ELE_AT(ARRAY, IDX) ((ARRAY)->td_array_data_ + idx) -#endif - #define TARRAY_MIN_SIZE 8 #define TARRAY_GET_ELEM(array, index) ((void*)((char*)((array)->pData) + (index) * (array)->elemSize)) #define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize) @@ -46,6 +33,9 @@ typedef struct SArray { void* pData; } SArray; +#define TARRAY_SIZE(array) ((array)->size) +#define TARRAY_DATA(array) ((array)->pData) + /** * * @param size @@ -194,6 +184,13 @@ void taosArrayPopTailBatch(SArray* pArray, size_t cnt); */ void taosArrayRemove(SArray* pArray, size_t index); +/** + * remove batch entry from the given index + * @param pArray + * @param index + */ +void taosArrayRemoveBatch(SArray* pArray, size_t index, size_t num, FDelete fp); + /** * copy the whole array from source to destination * @param pDst diff --git a/include/util/tdef.h b/include/util/tdef.h index 9626180b99430c3527d4b4e523db3cbab9cc93fc..9036befc028fde0572018d641495ee521aa688ea 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -190,10 +190,11 @@ typedef enum ELogicConditionType { #define TSDB_MIN_COLUMNS 2 // PRIMARY COLUMN(timestamp) + other columns #define TSDB_NODE_NAME_LEN 64 -#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string -#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string -#define TSDB_CGROUP_LEN 193 // it is a null-terminated string -#define TSDB_STREAM_NAME_LEN 193 // it is a null-terminated string +#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string +#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string +#define TSDB_CGROUP_LEN 193 // it is a null-terminated string +#define TSDB_USER_CGROUP_LEN (TSDB_USER_LEN + TSDB_CGROUP_LEN) // it is a null-terminated string +#define TSDB_STREAM_NAME_LEN 193 // it is a null-terminated string #define TSDB_DB_NAME_LEN 65 #define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) @@ -499,7 +500,7 @@ enum { #define DEFAULT_PAGESIZE 4096 #define VNODE_TIMEOUT_SEC 60 -#define MNODE_TIMEOUT_SEC 10 +#define MNODE_TIMEOUT_SEC 60 #ifdef __cplusplus } diff --git a/include/util/tencode.h b/include/util/tencode.h index a6dd58297e8c1dba644d86eb5145b273406fbf9e..ff97a205073822be2c98e39469066a0470794a51 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -116,6 +116,7 @@ static int32_t tEncodeI64v(SEncoder* pCoder, int64_t val); static int32_t tEncodeFloat(SEncoder* pCoder, float val); static int32_t tEncodeDouble(SEncoder* pCoder, double val); static int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, uint32_t len); +static int32_t tEncodeBinaryEx(SEncoder* pCoder, const uint8_t* val, uint32_t len); static int32_t tEncodeCStrWithLen(SEncoder* pCoder, const char* val, uint32_t len); static int32_t tEncodeCStr(SEncoder* pCoder, const char* val); diff --git a/packaging/release.sh b/packaging/release.sh deleted file mode 100755 index 1dfbf2b1124b0ebdeb1d9062c5d9d659eabe8a78..0000000000000000000000000000000000000000 --- a/packaging/release.sh +++ /dev/null @@ -1,319 +0,0 @@ -#!/bin/bash -# -# Generate the deb package for ubuntu, or rpm package for centos, or tar.gz package for other linux os - -set -e -# set -x - -# release.sh -v [cluster | edge] -# -c [aarch32 | aarch64 | x64 | x86 | mips64 | loongarch64...] -# -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] -# -V [stable | beta] -# -l [full | lite] -# -s [static | dynamic] -# -d [taos | ...] -# -n [2.0.0.3] -# -m [2.0.0.0] -# -H [ false | true] - -# set parameters by default value -verMode=edge # [cluster, edge, cloud] -verType=stable # [stable, beta] -cpuType=x64 # [aarch32 | aarch64 | x64 | x86 | mips64 loongarch64...] -osType=Linux # [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] -pagMode=full # [full | lite] -soMode=dynamic # [static | dynamic] -dbName=taos # [taos | ...] -allocator=glibc # [glibc | jemalloc] -verNumber="" -verNumberComp="3.0.0.0" -httpdBuild=false - -while getopts "hv:V:c:o:l:s:d:a:n:m:H:" arg; do - case $arg in - v) - #echo "verMode=$OPTARG" - verMode=$(echo $OPTARG) - ;; - V) - #echo "verType=$OPTARG" - verType=$(echo $OPTARG) - ;; - c) - #echo "cpuType=$OPTARG" - cpuType=$(echo $OPTARG) - ;; - l) - #echo "pagMode=$OPTARG" - pagMode=$(echo $OPTARG) - ;; - s) - #echo "soMode=$OPTARG" - soMode=$(echo $OPTARG) - ;; - d) - #echo "dbName=$OPTARG" - dbName=$(echo $OPTARG) - ;; - a) - #echo "allocator=$OPTARG" - allocator=$(echo $OPTARG) - ;; - n) - #echo "verNumber=$OPTARG" - verNumber=$(echo $OPTARG) - ;; - m) - #echo "verNumberComp=$OPTARG" - verNumberComp=$(echo $OPTARG) - ;; - o) - #echo "osType=$OPTARG" - osType=$(echo $OPTARG) - ;; - H) - #echo "httpdBuild=$OPTARG" - httpdBuild=$(echo $OPTARG) - ;; - h) - echo "Usage: $(basename $0) -v [cluster | edge] " - echo " -c [aarch32 | aarch64 | x64 | x86 | mips64 | loongarch64 ...] " - echo " -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] " - echo " -V [stable | beta] " - echo " -l [full | lite] " - echo " -a [glibc | jemalloc] " - echo " -s [static | dynamic] " - echo " -d [taos | ...] " - echo " -n [version number] " - echo " -m [compatible version number] " - echo " -H [false | true] " - exit 0 - ;; - ?) #unknow option - echo "unkonw argument" - exit 1 - ;; - esac -done - -osType=$(uname) - -echo "verMode=${verMode} verType=${verType} cpuType=${cpuType} osType=${osType} pagMode=${pagMode} soMode=${soMode} dbName=${dbName} allocator=${allocator} verNumber=${verNumber} verNumberComp=${verNumberComp} httpdBuild=${httpdBuild}" - -curr_dir=$(pwd) - -if [ "$osType" == "Darwin" ]; then - script_dir=$(dirname $0) - cd ${script_dir} - script_dir="$(pwd)" - top_dir=${script_dir}/.. -else - script_dir="$(dirname $(readlink -f $0))" - top_dir="$(readlink -f ${script_dir}/..)" -fi - -csudo="" -#if command -v sudo > /dev/null; then -# csudo="sudo " -#fi - -function is_valid_version() { - [ -z $1 ] && return 1 || : - - rx='^([0-9]+\.){3}(\*|[0-9]+)$' - if [[ $1 =~ $rx ]]; then - return 0 - fi - return 1 -} - -function vercomp() { - if [[ $1 == $2 ]]; then - echo 0 - exit 0 - fi - - local IFS=. - local i ver1=($1) ver2=($2) - - # fill empty fields in ver1 with zeros - for ((i = ${#ver1[@]}; i < ${#ver2[@]}; i++)); do - ver1[i]=0 - done - - for ((i = 0; i < ${#ver1[@]}; i++)); do - if [[ -z ${ver2[i]} ]]; then - # fill empty fields in ver2 with zeros - ver2[i]=0 - fi - if ((10#${ver1[i]} > 10#${ver2[i]})); then - echo 1 - exit 0 - fi - if ((10#${ver1[i]} < 10#${ver2[i]})); then - echo 2 - exit 0 - fi - done - echo 0 -} - -# 1. check version information -if ( (! is_valid_version $verNumber) || (! is_valid_version $verNumberComp) || [[ "$(vercomp $verNumber $verNumberComp)" == '2' ]]); then - echo "please enter correct version" - exit 0 -fi - -echo "=======================new version number: ${verNumber}, compatible version: ${verNumberComp}======================================" - -build_time=$(date +"%F %R") - -# get commint id from git -gitinfo=$(git rev-parse --verify HEAD) - -if [[ "$verMode" == "cluster" ]] || [[ "$verMode" == "cloud" ]]; then - enterprise_dir="${top_dir}/../enterprise" - cd ${enterprise_dir} - gitinfoOfInternal=$(git rev-parse --verify HEAD) -else - gitinfoOfInternal=NULL -fi - -cd "${curr_dir}" - -# 2. cmake executable file -compile_dir="${top_dir}/debug" -if [ -d ${compile_dir} ]; then - rm -rf ${compile_dir} -fi - -mkdir -p ${compile_dir} -cd ${compile_dir} - -if [[ "$allocator" == "jemalloc" ]]; then - allocator_macro="-DJEMALLOC_ENABLED=true" -else - allocator_macro="" -fi - -if [[ "$dbName" != "taos" ]]; then - source ${enterprise_dir}/packaging/oem/sed_$dbName.sh - replace_community_$dbName -fi - -if [[ "$httpdBuild" == "true" ]]; then - BUILD_HTTP=true -else - BUILD_HTTP=false -fi - -if [[ "$verMode" == "cluster" ]] || [[ "$verMode" == "cloud" ]]; then - BUILD_HTTP=internal -fi - -if [[ "$pagMode" == "full" ]]; then - BUILD_TOOLS=true -else - BUILD_TOOLS=false -fi - -# check support cpu type -if [[ "$cpuType" == "x64" ]] || [[ "$cpuType" == "aarch64" ]] || [[ "$cpuType" == "aarch32" ]] || [[ "$cpuType" == "arm64" ]] || [[ "$cpuType" == "arm32" ]] || [[ "$cpuType" == "mips64" ]] || [[ "$cpuType" == "loongarch64" ]] ; then - if [ "$verMode" == "edge" ]; then - # community-version compile - cmake ../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DPAGMODE=${pagMode} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro} - elif [ "$verMode" == "cloud" ]; then - cmake ../../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DBUILD_TAOSX=true -DBUILD_CLOUD=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro} - elif [ "$verMode" == "cluster" ]; then - if [[ "$dbName" != "taos" ]]; then - replace_enterprise_$dbName - fi - cmake ../../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DBUILD_TAOSX=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro} - fi -else - echo "input cpuType=${cpuType} error!!!" - exit 1 -fi - -ostype=`uname` -if [ "${ostype}" == "Darwin" ]; then - CORES=$(sysctl -n hw.ncpu) -else - CORES=$(grep -c ^processor /proc/cpuinfo) -fi - -if [[ "$allocator" == "jemalloc" ]]; then - # jemalloc need compile first, so disable parallel build - make -j ${CORES} && ${csudo}make install -else - make -j ${CORES} && ${csudo}make install -fi - -cd ${curr_dir} - -# 3. Call the corresponding script for packaging -if [ "$osType" != "Darwin" ]; then - if [[ "$verMode" != "cluster" ]] && [[ "$verMode" != "cloud" ]] && [[ "$pagMode" == "full" ]] && [[ "$cpuType" == "x64" ]] && [[ "$dbName" == "taos" ]]; then - ret='0' - command -v dpkg >/dev/null 2>&1 || { ret='1'; } - if [ "$ret" -eq 0 ]; then - echo "====do deb package for the ubuntu system====" - output_dir="${top_dir}/debs" - if [ -d ${output_dir} ]; then - rm -rf ${output_dir} - fi - mkdir -p ${output_dir} - cd ${script_dir}/deb - ${csudo}./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} - - if [[ "$pagMode" == "full" ]]; then - if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then - cd ${top_dir}/tools/taos-tools/packaging/deb - taos_tools_ver=$(git tag |grep -v taos | sort | tail -1) - [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" - - ${csudo}./make-taos-tools-deb.sh ${top_dir} \ - ${compile_dir} ${output_dir} ${taos_tools_ver} ${cpuType} ${osType} ${verMode} ${verType} ${verNumberComp} - fi - fi - else - echo "==========dpkg command not exist, so not release deb package!!!" - fi - ret='0' - command -v rpmbuild >/dev/null 2>&1 || { ret='1'; } - if [ "$ret" -eq 0 ]; then - echo "====do rpm package for the centos system====" - output_dir="${top_dir}/rpms" - if [ -d ${output_dir} ]; then - rm -rf ${output_dir} - fi - mkdir -p ${output_dir} - cd ${script_dir}/rpm - ${csudo}./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} - - if [[ "$pagMode" == "full" ]]; then - if [ -d ${top_dir}/tools/taos-tools/packaging/rpm ]; then - cd ${top_dir}/tools/taos-tools/packaging/rpm - taos_tools_ver=$(git tag |grep -v taos | sort | tail -1) - [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" - - ${csudo}./make-taos-tools-rpm.sh ${top_dir} \ - ${compile_dir} ${output_dir} ${taos_tools_ver} ${cpuType} ${osType} ${verMode} ${verType} ${verNumberComp} - fi - fi - else - echo "==========rpmbuild command not exist, so not release rpm package!!!" - fi - fi - - echo "====do tar.gz package for all systems====" - cd ${script_dir}/tools - - ${csudo}./makepkg.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${verNumberComp} ${dbName} - ${csudo}./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} - -else - cd ${script_dir}/tools - ./makepkg.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${verNumberComp} ${dbName} - ./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} -fi diff --git a/packaging/tools/make_install.bat b/packaging/tools/make_install.bat index c19519f9a1fc4c014c01496ca32925ba1eb19516..bf7418ad7935cb33f4a9d765e6c319d86383db69 100644 --- a/packaging/tools/make_install.bat +++ b/packaging/tools/make_install.bat @@ -14,6 +14,7 @@ set binary_dir=%3 set binary_dir=%binary_dir:/=\\% set osType=%4 set verNumber=%5 +set Enterprise=%6 set target_dir=C:\\TDengine if not exist %target_dir% ( @@ -57,7 +58,33 @@ if exist %binary_dir%\\build\\lib\\taosws.dll ( if exist %binary_dir%\\build\\bin\\taosdump.exe ( copy %binary_dir%\\build\\bin\\taosdump.exe %target_dir% > nul ) - +if %Enterprise% == TRUE ( + if exist %binary_dir%\\build\\bin\\taosx.exe ( + copy %binary_dir%\\build\\bin\\taosx.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\tmq_sim.exe ( + copy %binary_dir%\\build\\bin\\tmq_sim.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\tsim.exe ( + copy %binary_dir%\\build\\bin\\tsim.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\tmq_taosx_ci.exe ( + copy %binary_dir%\\build\\bin\\tmq_taosx_ci.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\tmq_demo.exe ( + copy %binary_dir%\\build\\bin\\tmq_demo.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\dumper.exe ( + copy %binary_dir%\\build\\bin\\dumper.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\runUdf.exe ( + copy %binary_dir%\\build\\bin\\runUdf.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\create_table.exe ( + copy %binary_dir%\\build\\bin\\create_table.exe %target_dir% > nul + ) +) + copy %binary_dir%\\build\\bin\\taosd.exe %target_dir% > nul copy %binary_dir%\\build\\bin\\udfd.exe %target_dir% > nul diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 482345dcd819dce683d3aa68438e6fa9fd5a4a9f..b246e0225f87889a553a6762ba26f83ce9bd2afd 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -109,6 +109,13 @@ function kill_taosadapter() { fi } +function kill_taoskeeper() { + pid=$(ps -ef | grep "taoskeeper" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} + function kill_taosd() { # ${csudo}pkill -f taosd || : pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}') @@ -161,6 +168,7 @@ function install_bin() { ${csudo}rm -f ${bin_link_dir}/udfd || : ${csudo}rm -f ${bin_link_dir}/taosadapter || : ${csudo}rm -f ${bin_link_dir}/taosBenchmark || : + ${csudo}rm -f ${bin_link_dir}/taoskeeper || : ${csudo}rm -f ${bin_link_dir}/taosdemo || : ${csudo}rm -f ${bin_link_dir}/taosdump || : ${csudo}rm -f ${bin_link_dir}/rmtaos || : @@ -179,6 +187,7 @@ function install_bin() { [ -x ${bin_dir}/taosdump ] && ${csudo}ln -s ${bin_dir}/taosdump ${bin_link_dir}/taosdump || : [ -x ${bin_dir}/set_core.sh ] && ${csudo}ln -s ${bin_dir}/set_core.sh ${bin_link_dir}/set_core || : [ -x ${bin_dir}/remove.sh ] && ${csudo}ln -s ${bin_dir}/remove.sh ${bin_link_dir}/rmtaos || : + [ -x ${bin_dir}/taoskeeper ] && ${csudo}ln -sf ${bin_dir}/taoskeeper ${bin_link_dir}/taoskeeper || : } function add_newHostname_to_hosts() { @@ -351,6 +360,22 @@ function install_taosadapter_config() { ${csudo}ln -s ${cfg_install_dir}/taosadapter.toml ${cfg_dir} } +function install_taoskeeper_config() { + if [ ! -f "${cfg_install_dir}/keeper.toml" ]; then + [ ! -d %{cfg_install_dir} ] && + ${csudo}${csudo}mkdir -p ${cfg_install_dir} + [ -f ${cfg_dir}/keeper.toml ] && ${csudo}cp ${cfg_dir}/keeper.toml ${cfg_install_dir} + [ -f ${cfg_install_dir}/keeper.toml ] && + ${csudo}chmod 644 ${cfg_install_dir}/keeper.toml + fi + + [ -f ${cfg_dir}/keeper.toml ] && + ${csudo}mv ${cfg_dir}/keeper.toml ${cfg_dir}/keeper.toml.new + + [ -f ${cfg_install_dir}/keeper.toml ] && + ${csudo}ln -s ${cfg_install_dir}/keeper.toml ${cfg_dir} +} + function install_config() { if [ ! -f "${cfg_install_dir}/taos.cfg" ]; then ${csudo}${csudo}mkdir -p ${cfg_install_dir} @@ -583,6 +608,7 @@ function install_TDengine() { install_bin install_config install_taosadapter_config + install_taoskeeper_config install_taosadapter_service install_service install_app diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 903a6a22cac05c7c921ed6287ae38b2498b064fe..3fc5cc9cecb6d164ce6ec009f67dcd7de2a99564 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -149,7 +149,6 @@ typedef struct STscObj { int32_t numOfReqs; // number of sqlObj bound to this connection SAppInstInfo* pAppInfo; SHashObj* pRequests; - int8_t schemalessType; // todo remove it, this attribute should be move to request } STscObj; typedef struct SResultColumn { diff --git a/source/client/inc/clientLog.h b/source/client/inc/clientLog.h index 0cb36ff61db570f2c6fb488d2964fc38c2cfd7f4..c29f495201d7b9ec0360abf6a5414798739b3da8 100644 --- a/source/client/inc/clientLog.h +++ b/source/client/inc/clientLog.h @@ -30,7 +30,7 @@ extern "C" { #define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0) #define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", DEBUG_TRACE, cDebugFlag, __VA_ARGS__); }} while(0) #define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0) -//#define tscPerf(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", DEBUG_INFO, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscPerf(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", 0, cDebugFlag, __VA_ARGS__); }} while(0) // clang-format on #ifdef __cplusplus diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h new file mode 100644 index 0000000000000000000000000000000000000000..15d91641a470aceb65e5cecf65a28b0cac635b64 --- /dev/null +++ b/source/client/inc/clientSml.h @@ -0,0 +1,246 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TDENGINE_CLIENTSML_H +#define TDENGINE_CLIENTSML_H +#ifdef __cplusplus +extern "C" { +#endif + +#include "catalog.h" +#include "clientInt.h" +#include "osThread.h" +#include "query.h" +#include "taos.h" +#include "taoserror.h" +#include "tcommon.h" +#include "tdef.h" +#include "tglobal.h" +#include "tlog.h" +#include "tmsg.h" +#include "tname.h" +#include "ttime.h" +#include "ttypes.h" +#include "cJSON.h" + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#ifndef likely +#define likely(expr) expect((expr) != 0, 1) +#endif +#ifndef unlikely +#define unlikely(expr) expect((expr) != 0, 0) +#endif + +#define SPACE ' ' +#define COMMA ',' +#define EQUAL '=' +#define QUOTE '"' +#define SLASH '\\' + +#define JUMP_SPACE(sql, sqlEnd) \ + while (sql < sqlEnd) { \ + if (unlikely(*sql == SPACE)) \ + sql++; \ + else \ + break; \ + } + +#define IS_INVALID_COL_LEN(len) ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN) +#define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN) + +#define TS "_ts" +#define TS_LEN 3 +#define VALUE "_value" +#define VALUE_LEN 6 + +#define OTD_JSON_FIELDS_NUM 4 +#define MAX_RETRY_TIMES 5 +typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; + +typedef enum { + SCHEMA_ACTION_NULL, + SCHEMA_ACTION_CREATE_STABLE, + SCHEMA_ACTION_ADD_COLUMN, + SCHEMA_ACTION_ADD_TAG, + SCHEMA_ACTION_CHANGE_COLUMN_SIZE, + SCHEMA_ACTION_CHANGE_TAG_SIZE, +} ESchemaAction; + +typedef struct { + const void *key; + int32_t keyLen; + void *value; + bool used; +}Node; + +typedef struct NodeList{ + Node data; + struct NodeList* next; +}NodeList; + +typedef struct { + char *measure; + char *tags; + char *cols; + char *timestamp; + + int32_t measureLen; + int32_t measureTagsLen; + int32_t tagsLen; + int32_t colsLen; + int32_t timestampLen; + + SArray *colArray; +} SSmlLineInfo; + +typedef struct { + const char *sTableName; // super table name + int32_t sTableNameLen; + char childTableName[TSDB_TABLE_NAME_LEN]; + uint64_t uid; + void *key; // for openTsdb + + SArray *tags; + + // elements are SHashObj for find by key quickly + SArray *cols; + STableDataCxt *tableDataCtx; +} SSmlTableInfo; + +typedef struct { + SArray *tags; // save the origin order to create table + SHashObj *tagHash; // elements are + + SArray *cols; + SHashObj *colHash; + + STableMeta *tableMeta; +} SSmlSTableMeta; + +typedef struct { + int32_t len; + char *buf; +} SSmlMsgBuf; + +typedef struct { + int32_t code; + int32_t lineNum; + + int32_t numOfSTables; + int32_t numOfCTables; + int32_t numOfCreateSTables; + int32_t numOfAlterColSTables; + int32_t numOfAlterTagSTables; + + int64_t parseTime; + int64_t schemaTime; + int64_t insertBindTime; + int64_t insertRpcTime; + int64_t endTime; +} SSmlCostInfo; + +typedef struct { + int64_t id; + + SMLProtocolType protocol; + int8_t precision; + bool reRun; + bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) + bool isRawLine; + int32_t ttl; + int32_t uid; // used for automatic create child table + + NodeList *childTables; + NodeList *superTables; + SHashObj *pVgHash; + + STscObj *taos; + SCatalog *pCatalog; + SRequestObj *pRequest; + SQuery *pQuery; + + SSmlCostInfo cost; + int32_t lineNum; + SSmlMsgBuf msgBuf; + + cJSON *root; // for parse json + int8_t offset[OTD_JSON_FIELDS_NUM]; + SSmlLineInfo *lines; // element is SSmlLineInfo + bool parseJsonByLib; + SArray *tagJsonArray; + + // + SArray *preLineTagKV; + SArray *maxTagKVs; + SArray *preLineColKV; + + SSmlLineInfo preLine; + STableMeta *currSTableMeta; + STableDataCxt *currTableDataCtx; + bool needModifySchema; +} SSmlHandle; + +#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \ +&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0) + +#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \ +&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0) + +#define IS_SAME_KEY (maxKV->keyLen == kv.keyLen && memcmp(maxKV->key, kv.key, kv.keyLen) == 0) + +extern int64_t smlFactorNS[3]; +extern int64_t smlFactorS[3]; + +typedef int32_t (*_equal_fn_sml)(const void *, const void *); + +SSmlHandle *smlBuildSmlInfo(TAOS *taos); +void smlDestroyInfo(SSmlHandle *info); +int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset); +int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset); +//SArray *smlJsonParseTags(char *start, char *end); +bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); +void* nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn); +int nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn); +int nodeListSize(NodeList* list); +bool smlDoubleToInt64OverFlow(double num); +int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); +bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); +int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision); +int8_t smlGetTsTypeByLen(int32_t len); +SSmlTableInfo* smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen); +SSmlSTableMeta* smlBuildSTableMeta(bool isDataFormat); +int32_t smlSetCTableName(SSmlTableInfo *oneTable); +STableMeta* smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen); +int32_t is_same_child_table_telnet(const void *a, const void *b); +int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len); +int32_t smlClearForRerun(SSmlHandle *info); +int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg); +uint8_t smlGetTimestampLen(int64_t num); +void clearColValArray(SArray* pCols); +void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag); + +int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); +int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); +int32_t smlParseJSON(SSmlHandle *info, char *payload); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_CLIENTSML_H diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index e5507ccf816b9fd591e6e27ee2974badcb5734ed..2b42de93e3f45ccb95c085c31825d4e0a117538c 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -21,8 +21,6 @@ extern "C" { #endif #include "catalog.h" -typedef void STableDataBlocks; - typedef enum { STMT_TYPE_INSERT = 1, STMT_TYPE_MULTI_INSERT, @@ -43,8 +41,8 @@ typedef enum { } STMT_STATUS; typedef struct SStmtTableCache { - STableDataBlocks *pDataBlock; - void *boundTags; + STableDataCxt *pDataCtx; + void *boundTags; } SStmtTableCache; typedef struct SStmtQueryResInfo { @@ -71,10 +69,11 @@ typedef struct SStmtBindInfo { } SStmtBindInfo; typedef struct SStmtExecInfo { - int32_t affectedRows; - SRequestObj *pRequest; - SHashObj *pBlockHash; - bool autoCreateTbl; + int32_t affectedRows; + SRequestObj *pRequest; + SHashObj *pBlockHash; + STableDataCxt *pCurrBlock; + SSubmitTbData *pCurrTbData; } SStmtExecInfo; typedef struct SStmtSQLInfo { diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 495c2cca9ae2a43a2629b92d956be51a4bd3480d..28a55156aa1dba2ebe65353c0b8b48e593a7917c 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -65,7 +65,10 @@ static int32_t registerRequest(SRequestObj *pRequest, STscObj *pTscObj) { static void deregisterRequest(SRequestObj *pRequest) { const static int64_t SLOW_QUERY_INTERVAL = 3000000L; // todo configurable - assert(pRequest != NULL); + if(pRequest == NULL){ + tscError("pRequest == NULL"); + return; + } STscObj *pTscObj = pRequest->pTscObj; SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; @@ -79,13 +82,19 @@ static void deregisterRequest(SRequestObj *pRequest) { "current:%d, app current:%d", pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000.0, num, currentInst); - if (QUERY_NODE_VNODE_MODIF_STMT == pRequest->stmtType) { - // tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 - // "us, exec:%" PRId64 "us", - // duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, - // pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - - // pRequest->metric.ctgEnd, pRequest->metric.execEnd - pRequest->metric.semanticEnd); - atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); + tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 + "us, exec:%" PRId64 "us, stmtType:%d", + duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, + pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - pRequest->metric.ctgEnd, + pRequest->metric.execEnd - pRequest->metric.semanticEnd, pRequest->stmtType); + + if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType) { + // tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 + // "us, exec:%" PRId64 "us", + // duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, + // pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - + // pRequest->metric.ctgEnd, pRequest->metric.execEnd - pRequest->metric.semanticEnd); + // atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { // tscPerf("select duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 // "us, planner:%" PRId64 "us, exec:%" PRId64 "us, reqId:0x%" PRIx64, @@ -267,7 +276,6 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c taosThreadMutexInit(&pObj->mutex, NULL); pObj->id = taosAddRef(clientConnRefPool, pObj); - pObj->schemalessType = 1; atomic_add_fetch_64(&pObj->pAppInfo->numOfConns, 1); @@ -560,7 +568,8 @@ void taos_init_imp(void) { initQueryModuleMsgHandle(); if (taosConvInit() != 0) { - ASSERTS(0, "failed to init conv"); + tscError("failed to init conv"); + return; } rpcInit(); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 47ed2cf035a13314b81f2208718d2c466efb5c68..3cb8a2e1bd9e8c6c2c5a68f2e266611d209326a9 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -376,7 +376,6 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { desc.subPlanNum = 0; } desc.subPlanNum = taosArrayGetSize(desc.subDesc); - ASSERT(desc.subPlanNum == taosArrayGetSize(desc.subDesc)); } else { desc.subDesc = NULL; } @@ -813,7 +812,10 @@ static void hbStopThread() { } SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { - hbMgrInit(); + if(hbMgrInit() != 0){ + terrno = TSDB_CODE_TSC_INTERNAL_ERROR; + return NULL; + } SAppHbMgr *pAppHbMgr = taosMemoryMalloc(sizeof(SAppHbMgr)); if (pAppHbMgr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -899,16 +901,28 @@ int hbMgrInit() { TdThreadMutexAttr attr = {0}; int ret = taosThreadMutexAttrInit(&attr); - assert(ret == 0); + if(ret != 0){ + uError("hbMgrInit:taosThreadMutexAttrInit error") + return ret; + } ret = taosThreadMutexAttrSetType(&attr, PTHREAD_MUTEX_RECURSIVE); - assert(ret == 0); + if(ret != 0){ + uError("hbMgrInit:taosThreadMutexAttrSetType error") + return ret; + } ret = taosThreadMutexInit(&clientHbMgr.lock, &attr); - assert(ret == 0); + if(ret != 0){ + uError("hbMgrInit:taosThreadMutexInit error") + return ret; + } ret = taosThreadMutexAttrDestroy(&attr); - assert(ret == 0); + if(ret != 0){ + uError("hbMgrInit:taosThreadMutexAttrDestroy error") + return ret; + } // init handle funcs hbMgrInitHandle(); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 53acafeeaacee03e3016a53df7ee4083e29b833c..aeb73784f20c1d84a9d2d894026c9e4d3c863a5b 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -239,7 +239,6 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC .pTransporter = pTscObj->pAppInfo->pTransporter, .pStmtCb = pStmtCb, .pUser = pTscObj->user, - .schemalessType = pTscObj->schemalessType, .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)), .enableSysInfo = pTscObj->sysInfo, .svrVer = pTscObj->sVer, @@ -273,7 +272,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC int32_t execLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { SRetrieveTableRsp* pRsp = NULL; - int32_t code = qExecCommand(pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp); + int32_t code = qExecCommand(&pRequest->pTscObj->id, pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp); if (TSDB_CODE_SUCCESS == code && NULL != pRsp) { code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, true); } @@ -311,7 +310,7 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { return; } - int32_t code = qExecCommand(pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp); + int32_t code = qExecCommand(&pRequest->pTscObj->id ,pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp); if (TSDB_CODE_SUCCESS == code && NULL != pRsp) { code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, true); } @@ -453,7 +452,10 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra } void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols) { - ASSERT(pSchema != NULL && numOfCols > 0); + if(pResInfo == NULL || pSchema == NULL || numOfCols <= 0){ + tscError("invalid paras, pResInfo == NULL || pSchema == NULL || numOfCols <= 0"); + return; + } pResInfo->numOfCols = numOfCols; if (pResInfo->fields != NULL) { @@ -464,7 +466,10 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t } pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); - ASSERT(numOfCols == pResInfo->numOfCols); + if(numOfCols != pResInfo->numOfCols){ + tscError("numOfCols:%d != pResInfo->numOfCols:%d", numOfCols, pResInfo->numOfCols); + return; + } for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { pResInfo->fields[i].bytes = pSchema[i].bytes; @@ -741,47 +746,21 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList } int32_t handleSubmitExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet* epset) { - int32_t code = 0; - SArray* pArray = NULL; - SSubmitRsp* pRsp = (SSubmitRsp*)res; - if (pRsp->nBlocks <= 0) { - taosMemoryFreeClear(pRsp->pBlocks); + SArray* pArray = NULL; + SSubmitRsp2* pRsp = (SSubmitRsp2*)res; + if (NULL == pRsp->aCreateTbRsp) { return TSDB_CODE_SUCCESS; } - pArray = taosArrayInit(pRsp->nBlocks, sizeof(STbSVersion)); - if (NULL == pArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < pRsp->nBlocks; ++i) { - SSubmitBlkRsp* blk = pRsp->pBlocks + i; - if (blk->pMeta) { - handleCreateTbExecRes(blk->pMeta, pCatalog); - tFreeSTableMetaRsp(blk->pMeta); - taosMemoryFreeClear(blk->pMeta); - } - - if (NULL == blk->tblFName || 0 == blk->tblFName[0]) { - continue; + int32_t tbNum = taosArrayGetSize(pRsp->aCreateTbRsp); + for (int32_t i = 0; i < tbNum; ++i) { + SVCreateTbRsp* pTbRsp = (SVCreateTbRsp*)taosArrayGet(pRsp->aCreateTbRsp, i); + if (pTbRsp->pMeta) { + handleCreateTbExecRes(pTbRsp->pMeta, pCatalog); } - - STbSVersion tbSver = {.tbFName = blk->tblFName, .sver = blk->sver}; - taosArrayPush(pArray, &tbSver); } - SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = *epset}; - - code = catalogChkTbMetaVersion(pCatalog, &conn, pArray); - -_return: - - taosArrayDestroy(pArray); - return code; + return TSDB_CODE_SUCCESS; } int32_t handleQueryExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet* epset) { @@ -882,7 +861,7 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) { } static bool incompletaFileParsing(SNode* pStmt) { - return QUERY_NODE_VNODE_MODIF_STMT != nodeType(pStmt) ? false : ((SVnodeModifOpStmt*)pStmt)->fileProcessing; + return QUERY_NODE_VNODE_MODIFY_STMT != nodeType(pStmt) ? false : ((SVnodeModifyOpStmt*)pStmt)->fileProcessing; } // todo refacto the error code mgmt @@ -961,7 +940,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue if (pQuery->pRoot && !pRequest->inRetry) { STscObj* pTscObj = pRequest->pTscObj; SAppClusterSummary* pActivity = &pTscObj->pAppInfo->summary; - if (QUERY_NODE_VNODE_MODIF_STMT == pQuery->pRoot->type) { + if (QUERY_NODE_VNODE_MODIFY_STMT == pQuery->pRoot->type) { atomic_add_fetch_64((int64_t*)&pActivity->numOfInsertsReq, 1); } else if (QUERY_NODE_SELECT_STMT == pQuery->pRoot->type) { atomic_add_fetch_64((int64_t*)&pActivity->numOfQueryReq, 1); @@ -1066,7 +1045,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat } if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { SArray* pNodeList = NULL; - if (QUERY_NODE_VNODE_MODIF_STMT != nodeType(pQuery->pRoot)) { + if (QUERY_NODE_VNODE_MODIFY_STMT != nodeType(pQuery->pRoot)) { buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); } @@ -1366,7 +1345,10 @@ int32_t doProcessMsgFromServer(void* param) { SEpSet* pEpSet = arg->pEpset; SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle; - assert(pMsg->info.ahandle != NULL); + if(pMsg->info.ahandle == NULL){ + tscError("doProcessMsgFromServer pMsg->info.ahandle == NULL"); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } STscObj* pTscObj = NULL; STraceId* trace = &pMsg->info.traceId; @@ -1379,8 +1361,10 @@ int32_t doProcessMsgFromServer(void* param) { if (pSendInfo->requestObjRefId != 0) { SRequestObj* pRequest = (SRequestObj*)taosAcquireRef(clientReqRefPool, pSendInfo->requestObjRefId); if (pRequest) { - assert(pRequest->self == pSendInfo->requestObjRefId); - + if(pRequest->self != pSendInfo->requestObjRefId){ + tscError("doProcessMsgFromServer pRequest->self:%"PRId64" != pSendInfo->requestObjRefId:%"PRId64, pRequest->self, pSendInfo->requestObjRefId); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } pRequest->metric.rsp = taosGetTimestampUs(); pTscObj = pRequest->pTscObj; /* @@ -1522,7 +1506,9 @@ void doSetOneRowPtr(SReqResultInfo* pResultInfo) { } void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) { - assert(pRequest != NULL); + if(pRequest == NULL){ + return NULL; + } SReqResultInfo* pResultInfo = &pRequest->body.resInfo; if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { @@ -1576,7 +1562,9 @@ static void syncFetchFn(void* param, TAOS_RES* res, int32_t numOfRows) { } void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) { - assert(pRequest != NULL); + if(pRequest == NULL){ + return NULL; + } SReqResultInfo* pResultInfo = &pRequest->body.resInfo; if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { @@ -1640,8 +1628,10 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int char* pStart = pCol->offset[j] + pCol->pData; int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(p)); - ASSERT(len <= bytes); - ASSERT((p + len) < (pResultInfo->convertBuf[i] + colLength[i])); + if(len > bytes || (p + len) >= (pResultInfo->convertBuf[i] + colLength[i])){ + tscError("doConvertUCS4 error, invalid data. len:%d, bytes:%d, (p + len):%p, (pResultInfo->convertBuf[i] + colLength[i]):%p", len, bytes, (p + len), (pResultInfo->convertBuf[i] + colLength[i])); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } varDataSetLen(p, len); pCol->offset[j] = (p - pResultInfo->convertBuf[i]); @@ -1658,9 +1648,6 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int } int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) { - int32_t cols = *(int32_t*)(p + sizeof(int32_t) * 3); - ASSERT(numOfCols == cols); - return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) * 3 + sizeof(uint64_t) + numOfCols * (sizeof(int8_t) + sizeof(int32_t)); } @@ -1670,6 +1657,12 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column // length | + int32_t cols = *(int32_t*)(p + sizeof(int32_t) * 3); + if(ASSERT(numOfCols == cols)){ + tscError("estimateJsonLen error: numOfCols:%d != cols:%d", numOfCols, cols); + return -1; + } + int32_t len = getVersion1BlockMetaSize(p, numOfCols); int32_t* colLength = (int32_t*)(p + len); len += sizeof(int32_t) * numOfCols; @@ -1703,7 +1696,8 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) { len += (VARSTR_HEADER_SIZE + 5); } else { - ASSERT(0); + tscError("estimateJsonLen error: invalid type:%d", jsonInnerType); + return -1; } } } else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { @@ -1737,12 +1731,21 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int char* p = (char*)pResultInfo->pData; int32_t dataLen = estimateJsonLen(pResultInfo, numOfCols, numOfRows); + if(dataLen <= 0){ + return TSDB_CODE_TSC_INTERNAL_ERROR; + } pResultInfo->convertJson = taosMemoryCalloc(1, dataLen); if (pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; char* p1 = pResultInfo->convertJson; int32_t totalLen = 0; + int32_t cols = *(int32_t*)(p + sizeof(int32_t) * 3); + if(ASSERT(numOfCols == cols)){ + tscError("doConvertJson error: numOfCols:%d != cols:%d", numOfCols, cols); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } + int32_t len = getVersion1BlockMetaSize(p, numOfCols); memcpy(p1, p, len); @@ -1763,8 +1766,10 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int for (int32_t i = 0; i < numOfCols; ++i) { int32_t colLen = htonl(colLength[i]); int32_t colLen1 = htonl(colLength1[i]); - ASSERT(colLen < dataLen); - + if(ASSERT(colLen < dataLen)){ + tscError("doConvertJson error: colLen:%d >= dataLen:%d", colLen, dataLen); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) { int32_t* offset = (int32_t*)pStart; int32_t* offset1 = (int32_t*)pStart1; @@ -1809,7 +1814,8 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false"); varDataSetLen(dst, strlen(varDataVal(dst))); } else { - ASSERT(0); + tscError("doConvertJson error: invalid type:%d", jsonInnerType); + return TSDB_CODE_TSC_INTERNAL_ERROR; } offset1[j] = len; @@ -1847,7 +1853,10 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows, bool convertUcs4) { - assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL); + if(ASSERT(numOfCols > 0 && pFields != NULL && pResultInfo != NULL)){ + tscError("setResultDataPtr paras error"); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } if (numOfRows == 0) { return TSDB_CODE_SUCCESS; } @@ -1876,7 +1885,10 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 int32_t cols = *(int32_t*)p; p += sizeof(int32_t); - ASSERT(rows == numOfRows && cols == numOfCols); + if(ASSERT(rows == numOfRows && cols == numOfCols)){ + tscError("setResultDataPtr paras error:rows;%d numOfRows:%d cols:%d numOfCols:%d", rows, numOfRows, cols, numOfCols); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } int32_t hasColumnSeg = *(int32_t*)p; p += sizeof(int32_t); @@ -1903,7 +1915,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 colLength[i] = htonl(colLength[i]); if (colLength[i] >= dataLen) { tscError("invalid colLength %d, dataLen %d", colLength[i], dataLen); - ASSERT(0); + return TSDB_CODE_TSC_INTERNAL_ERROR; } if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { @@ -1941,7 +1953,11 @@ char* getDbOfConnection(STscObj* pObj) { } void setConnectionDB(STscObj* pTscObj, const char* db) { - assert(db != NULL && pTscObj != NULL); + if(db == NULL || pTscObj == NULL){ + tscError("setConnectionDB para is NULL"); + return; + } + taosThreadMutexLock(&pTscObj->mutex); tstrncpy(pTscObj->db, db, tListLen(pTscObj->db)); taosThreadMutexUnlock(&pTscObj->mutex); @@ -1959,7 +1975,10 @@ void resetConnectDB(STscObj* pTscObj) { int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4, bool freeAfterUse) { - assert(pResultInfo != NULL && pRsp != NULL); + if(pResultInfo == NULL || pRsp == NULL){ + tscError("setQueryResultFromRsp paras is null"); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } if (freeAfterUse) taosMemoryFreeClear(pResultInfo->pRspMsg); diff --git a/source/client/src/clientJniConnector.c b/source/client/src/clientJniConnector.c index 859d4ec80ff6e954259415dabaf1761eb3e7129e..750ba684f42c2f35cd6f5180e6d1d722ad3e0e16 100644 --- a/source/client/src/clientJniConnector.c +++ b/source/client/src/clientJniConnector.c @@ -574,7 +574,11 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp(JNI TAOS_RES *tres = (TAOS_RES *)res; int32_t numOfFields = taos_num_fields(tres); - assert(numOfFields > 0); + if(numOfFields <= 0){ + jniError("jobj:%p, conn:%p, query interrupted. taos_num_fields error code:%d, msg:%s", jobj, tscon, numOfFields, + taos_errstr(tres)); + return JNI_RESULT_SET_NULL; + } void *data; int32_t numOfRows; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 15c1d65162a7a9b9b73e4141fb29b6659ed5c779..e1df2a4540b500a53e88147c3eb22affcecb301c 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -293,7 +293,6 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { tscError("invalid result passed to taos_fetch_row"); return NULL; } - return NULL; } int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { @@ -357,9 +356,13 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) case TSDB_DATA_TYPE_NCHAR: { int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); if (fields[i].type == TSDB_DATA_TYPE_BINARY) { - assert(charLen <= fields[i].bytes && charLen >= 0); + if(ASSERT(charLen <= fields[i].bytes && charLen >= 0)){ + tscError("taos_print_row error binary. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes); + } } else { - assert(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0); + if(ASSERT(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0)){ + tscError("taos_print_row error. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes); + } } memcpy(str + len, row[i], charLen); @@ -579,7 +582,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) { (*numOfRows) = pResultInfo->numOfRows; return 0; } else { - ASSERT(0); + tscError("taos_fetch_block_s invalid res type"); return -1; } } @@ -804,7 +807,7 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c tstrerror(code)); if (code == TSDB_CODE_SUCCESS) { - //pWrapper->pCatalogReq->forceUpdate = false; + // pWrapper->pCatalogReq->forceUpdate = false; code = qContinueParseSql(pWrapper->pParseCtx, pWrapper->pCatalogReq, pResultMeta, pQuery); } @@ -833,8 +836,8 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest) tstrerror(code), pWrapper->pRequest->requestId); destorySqlCallbackWrapper(pWrapper); terrno = code; - pWrapper->pRequest->code = code; - pWrapper->pRequest->body.queryFp(pWrapper->pRequest->body.param, pWrapper->pRequest, code); + pRequest->code = code; + pRequest->body.queryFp(pRequest->body.param, pRequest, code); } } @@ -868,7 +871,6 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) { .pTransporter = pTscObj->pAppInfo->pTransporter, .pStmtCb = NULL, .pUser = pTscObj->user, - .schemalessType = pTscObj->schemalessType, .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)), .enableSysInfo = pTscObj->sysInfo, .async = true, @@ -1003,8 +1005,14 @@ static void fetchCallback(void *pResult, void *param, int32_t code) { } void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { - ASSERT(res != NULL && fp != NULL); - ASSERT(TD_RES_QUERY(res)); + if(ASSERT(res != NULL && fp != NULL)){ + tscError("taos_fetch_rows_a invalid paras"); + return; + } + if(ASSERT(TD_RES_QUERY(res))){ + tscError("taos_fetch_rows_a res is NULL"); + return; + } SRequestObj *pRequest = res; pRequest->body.fetchFp = fp; @@ -1047,9 +1055,14 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { } void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { - ASSERT(res != NULL && fp != NULL); - ASSERT(TD_RES_QUERY(res)); - + if(ASSERT(res != NULL && fp != NULL)){ + tscError("taos_fetch_rows_a invalid paras"); + return; + } + if(ASSERT(TD_RES_QUERY(res))){ + tscError("taos_fetch_rows_a res is NULL"); + return; + } SRequestObj *pRequest = res; SReqResultInfo *pResultInfo = &pRequest->body.resInfo; @@ -1061,8 +1074,14 @@ void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { } const void *taos_get_raw_block(TAOS_RES *res) { - ASSERT(res != NULL); - ASSERT(TD_RES_QUERY(res)); + if(ASSERT(res != NULL)){ + tscError("taos_fetch_rows_a invalid paras"); + return NULL; + } + if(ASSERT(TD_RES_QUERY(res))){ + tscError("taos_fetch_rows_a res is NULL"); + return NULL; + } SRequestObj *pRequest = res; return pRequest->body.resInfo.pData; @@ -1382,8 +1401,7 @@ int taos_stmt_get_col_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fiel } // let stmt to reclaim TAOS_FIELD_E that was allocated by `taos_stmt_get_tag_fields`/`taos_stmt_get_col_fields` -void taos_stmt_reclaim_fields(TAOS_STMT *stmt, TAOS_FIELD_E *fields) -{ +void taos_stmt_reclaim_fields(TAOS_STMT *stmt, TAOS_FIELD_E *fields) { (void)stmt; if (!fields) return; taosMemoryFree(fields); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index f414c7e92f8e208a5559dd8bdcf03642b4b7f7ed..2191c54315d6324e819337fa41db3875e1506594 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -150,7 +150,6 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pRequest) { pMsgSendInfo->msgType = pRequest->type; pMsgSendInfo->target.type = TARGET_TYPE_MNODE; - assert(pRequest != NULL); pMsgSendInfo->msgInfo = pRequest->body.requestMsg; pMsgSendInfo->fp = getMsgRspHandle(pRequest->type); return pMsgSendInfo; @@ -274,7 +273,9 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { } int32_t processCreateSTableRsp(void* param, SDataBuf* pMsg, int32_t code) { - assert(pMsg != NULL && param != NULL); + if(pMsg == NULL || param == NULL){ + return TSDB_CODE_TSC_INVALID_INPUT; + } SRequestObj* pRequest = param; if (code != TSDB_CODE_SUCCESS) { @@ -455,7 +456,10 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS); - ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); + if(len != rspSize - sizeof(SRetrieveTableRsp)){ + uError("buildShowVariablesRsp error, len:%d != rspSize - sizeof(SRetrieveTableRsp):%" PRIu64, len, (uint64_t) (rspSize - sizeof(SRetrieveTableRsp))); + return TSDB_CODE_TSC_INVALID_INPUT; + } blockDataDestroy(pBlock); return TSDB_CODE_SUCCESS; diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 8ceb2a3380c486acd86a884282d1d75a94dbb416..3fd6fed4fceb19f52026ab19ca1733b1b0a86a2e 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -373,7 +373,10 @@ static char* processCreateTable(SMqMetaRsp* metaRsp) { } static char* processAutoCreateTable(STaosxRsp* rsp) { - ASSERT(rsp->createTableNum != 0); + if(rsp->createTableNum <= 0){ + uError("WriteRaw:processAutoCreateTable rsp->createTableNum <= 0"); + goto _exit; + } SDecoder* decoder = taosMemoryCalloc(rsp->createTableNum, sizeof(SDecoder)); SVCreateTbReq* pCreateReq = taosMemoryCalloc(rsp->createTableNum, sizeof(SVCreateTbReq)); @@ -389,7 +392,10 @@ static char* processAutoCreateTable(STaosxRsp* rsp) { goto _exit; } - ASSERT(pCreateReq[iReq].type == TSDB_CHILD_TABLE); + if(pCreateReq[iReq].type != TSDB_CHILD_TABLE){ + uError("WriteRaw:processAutoCreateTable pCreateReq[iReq].type != TSDB_CHILD_TABLE"); + goto _exit; + } } string = buildCreateCTableJson(pCreateReq, rsp->createTableNum); @@ -494,7 +500,10 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { char* buf = NULL; if (vAlterTbReq.tagType == TSDB_DATA_TYPE_JSON) { - ASSERT(tTagIsJson(vAlterTbReq.pTagVal) == true); + if(!tTagIsJson(vAlterTbReq.pTagVal)){ + uError("processAlterTable isJson false"); + goto _exit; + } buf = parseTagDatatoJson(vAlterTbReq.pTagVal); } else { buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 1, 1); @@ -1240,21 +1249,12 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { return code; } -typedef struct { - SVgroupInfo vg; - void* data; -} VgData; - -static void destroyVgHash(void* data) { - VgData* vgData = (VgData*)data; - taosMemoryFreeClear(vgData->data); -} - -int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD *fields, int numFields){ +int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD* fields, + int numFields) { int32_t code = TSDB_CODE_SUCCESS; STableMeta* pTableMeta = NULL; SQuery* pQuery = NULL; - SSubmitReq* subReq = NULL; + SHashObj* pVgHash = NULL; SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); if (!pRequest) { @@ -1299,156 +1299,34 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbname); goto end; } - uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); - uint64_t uid = pTableMeta->uid; - int32_t numOfCols = pTableMeta->tableInfo.numOfColumns; - - uint16_t fLen = 0; - int32_t rowSize = 0; - int16_t nVar = 0; - for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - SSchema* schema = pTableMeta->schema + i; - fLen += TYPE_BYTES[schema->type]; - rowSize += schema->bytes; - if (IS_VAR_DATA_TYPE(schema->type)) { - nVar++; - } - } - - fLen -= sizeof(TSKEY); - - int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(numOfCols - 1); - int32_t schemaLen = 0; - int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; - - int32_t totalLen = sizeof(SSubmitReq) + submitLen; - subReq = taosMemoryCalloc(1, totalLen); - SSubmitBlk* blk = POINTER_SHIFT(subReq, sizeof(SSubmitReq)); - void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); - STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTableMeta->sversion); - tdSRowSetTpInfo(&rb, numOfCols, fLen); - int32_t dataLen = 0; - - // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | - char* pStart = pData + getVersion1BlockMetaSize(pData, numFields); - int32_t* colLength = (int32_t*)pStart; - pStart += sizeof(int32_t) * numFields; - - SResultColumn* pCol = taosMemoryCalloc(numFields, sizeof(SResultColumn)); - - for (int32_t i = 0; i < numFields; ++i) { - if (IS_VAR_DATA_TYPE(fields[i].type)) { - pCol[i].offset = (int32_t*)pStart; - pStart += rows * sizeof(int32_t); - } else { - pCol[i].nullbitmap = pStart; - pStart += BitmapLen(rows); - } - - pCol[i].pData = pStart; - pStart += colLength[i]; - } - - SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - for (int i = 0; i < numFields; i++) { - TAOS_FIELD* schema = &fields[i]; - taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); - } - - for (int32_t j = 0; j < rows; j++) { - tdSRowResetBuf(&rb, rowData); - int32_t offset = 0; - for (int32_t k = 0; k < numOfCols; k++) { - const SSchema* pColumn = &pTableMeta->schema[k]; - int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); - if (!index) { // add none - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k); - }else{ - if (IS_VAR_DATA_TYPE(pColumn->type)) { - if (pCol[*index].offset[j] != -1) { - char* data = pCol[*index].pData + pCol[*index].offset[j]; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); - } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } - } else { - if (!colDataIsNull_f(pCol[*index].nullbitmap, j)) { - char* data = pCol[*index].pData + pColumn->bytes * j; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); - } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } - } - } - - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } - } - tdSRowEnd(&rb); - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - dataLen += rowLen; - } - - taosHashCleanup(schemaHash); - taosMemoryFree(pCol); - - blk->uid = htobe64(uid); - blk->suid = htobe64(suid); - blk->sversion = htonl(pTableMeta->sversion); - blk->schemaLen = htonl(schemaLen); - blk->numOfRows = htonl(rows); - blk->dataLen = htonl(dataLen); - subReq->length = sizeof(SSubmitReq) + sizeof(SSubmitBlk) + schemaLen + dataLen; - subReq->numOfBlocks = 1; - - pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == pQuery) { - uError("create SQuery error"); + pQuery = smlInitHandle(); + if (pQuery == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto end; } - pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->haveResultSet = false; - pQuery->msgType = TDMT_VND_SUBMIT; - pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); - if (NULL == pQuery->pRoot) { - uError("create pQuery->pRoot error"); - code = TSDB_CODE_OUT_OF_MEMORY; + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)); + + code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:rawBlockBindData failed"); goto end; } - SVnodeModifOpStmt* nodeStmt = (SVnodeModifOpStmt*)(pQuery->pRoot); - nodeStmt->pDataBlocks = taosArrayInit(1, POINTER_BYTES); - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; + code = smlBuildOutput(pQuery, pVgHash); + if (code != TSDB_CODE_SUCCESS) { + uError("smlBuildOutput failed"); + return code; } - dst->vg = vgData; - dst->numOfTables = subReq->numOfBlocks; - dst->size = subReq->length; - dst->pData = (char*)subReq; - subReq->header.vgId = htonl(dst->vg.vgId); - subReq->version = htonl(1); - subReq->header.contLen = htonl(subReq->length); - subReq->length = htonl(subReq->length); - subReq->numOfBlocks = htonl(subReq->numOfBlocks); - subReq = NULL; // no need free - taosArrayPush(nodeStmt->pDataBlocks, &dst); launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; - end: +end: taosMemoryFreeClear(pTableMeta); qDestroyQuery(pQuery); - taosMemoryFree(subReq); + destroyRequest(pRequest); + taosHashCleanup(pVgHash); return code; } @@ -1456,7 +1334,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) int32_t code = TSDB_CODE_SUCCESS; STableMeta* pTableMeta = NULL; SQuery* pQuery = NULL; - SSubmitReq* subReq = NULL; + SHashObj* pVgHash = NULL; SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); if (!pRequest) { @@ -1501,137 +1379,25 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbname); goto end; } - uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); - uint64_t uid = pTableMeta->uid; - int32_t numOfCols = pTableMeta->tableInfo.numOfColumns; - - uint16_t fLen = 0; - int32_t rowSize = 0; - int16_t nVar = 0; - for (int i = 0; i < numOfCols; i++) { - SSchema* schema = pTableMeta->schema + i; - fLen += TYPE_BYTES[schema->type]; - rowSize += schema->bytes; - if (IS_VAR_DATA_TYPE(schema->type)) { - nVar++; - } - } - fLen -= sizeof(TSKEY); - - int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(numOfCols - 1); - int32_t schemaLen = 0; - int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; - - int32_t totalLen = sizeof(SSubmitReq) + submitLen; - subReq = taosMemoryCalloc(1, totalLen); - SSubmitBlk* blk = POINTER_SHIFT(subReq, sizeof(SSubmitReq)); - void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); - STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTableMeta->sversion); - tdSRowSetTpInfo(&rb, numOfCols, fLen); - int32_t dataLen = 0; - - // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | - char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols); - int32_t* colLength = (int32_t*)pStart; - pStart += sizeof(int32_t) * numOfCols; - - SResultColumn* pCol = taosMemoryCalloc(numOfCols, sizeof(SResultColumn)); - - for (int32_t i = 0; i < numOfCols; ++i) { - if (IS_VAR_DATA_TYPE(pTableMeta->schema[i].type)) { - pCol[i].offset = (int32_t*)pStart; - pStart += rows * sizeof(int32_t); - } else { - pCol[i].nullbitmap = pStart; - pStart += BitmapLen(rows); - } - - pCol[i].pData = pStart; - pStart += colLength[i]; - } - - for (int32_t j = 0; j < rows; j++) { - tdSRowResetBuf(&rb, rowData); - int32_t offset = 0; - for (int32_t k = 0; k < numOfCols; k++) { - const SSchema* pColumn = &pTableMeta->schema[k]; - - if (IS_VAR_DATA_TYPE(pColumn->type)) { - if (pCol[k].offset[j] != -1) { - char* data = pCol[k].pData + pCol[k].offset[j]; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); - } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } - } else { - if (!colDataIsNull_f(pCol[k].nullbitmap, j)) { - char* data = pCol[k].pData + pColumn->bytes * j; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); - } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } - } - - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } - } - tdSRowEnd(&rb); - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - dataLen += rowLen; - } - - taosMemoryFree(pCol); - - blk->uid = htobe64(uid); - blk->suid = htobe64(suid); - blk->sversion = htonl(pTableMeta->sversion); - blk->schemaLen = htonl(schemaLen); - blk->numOfRows = htonl(rows); - blk->dataLen = htonl(dataLen); - subReq->length = sizeof(SSubmitReq) + sizeof(SSubmitBlk) + schemaLen + dataLen; - subReq->numOfBlocks = 1; - - pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == pQuery) { - uError("create SQuery error"); + pQuery = smlInitHandle(); + if (pQuery == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(subReq); goto end; } - pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->haveResultSet = false; - pQuery->msgType = TDMT_VND_SUBMIT; - pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); - if (NULL == pQuery->pRoot) { - uError("create pQuery->pRoot error"); - code = TSDB_CODE_OUT_OF_MEMORY; + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)); + + code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:rawBlockBindData failed"); goto end; } - SVnodeModifOpStmt* nodeStmt = (SVnodeModifOpStmt*)(pQuery->pRoot); - nodeStmt->pDataBlocks = taosArrayInit(1, POINTER_BYTES); - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; + code = smlBuildOutput(pQuery, pVgHash); + if (code != TSDB_CODE_SUCCESS) { + uError("smlBuildOutput failed"); + return code; } - dst->vg = vgData; - dst->numOfTables = subReq->numOfBlocks; - dst->size = subReq->length; - dst->pData = (char*)subReq; - subReq->header.vgId = htonl(dst->vg.vgId); - subReq->version = htonl(1); - subReq->header.contLen = htonl(subReq->length); - subReq->length = htonl(subReq->length); - subReq->numOfBlocks = htonl(subReq->numOfBlocks); - subReq = NULL; // no need free - taosArrayPush(nodeStmt->pDataBlocks, &dst); launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; @@ -1639,7 +1405,8 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) end: taosMemoryFreeClear(pTableMeta); qDestroyQuery(pQuery); - taosMemoryFree(subReq); + destroyRequest(pRequest); + taosHashCleanup(pVgHash); return code; } @@ -1676,8 +1443,6 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { goto end; } - pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - taosHashSetFreeFp(pVgHash, destroyVgHash); struct SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { @@ -1691,6 +1456,12 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { conn.requestObjRefId = pRequest->self; conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + pQuery = smlInitHandle(); + if (pQuery == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); uDebug("raw data block num:%d\n", rspObj.rsp.blockNum); while (++rspObj.resIter < rspObj.rsp.blockNum) { SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); @@ -1698,14 +1469,6 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { uError("WriteRaw:no schema, iter:%d", rspObj.resIter); goto end; } - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter); - setResSchemaInfo(&rspObj.resInfo, pSW->pSchema, pSW->nCols); - - code = setQueryResultFromRsp(&rspObj.resInfo, pRetrieve, false, false); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw: setQueryResultFromRsp error"); - goto end; - } const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.blockTbName, rspObj.resIter); if (!tbName) { @@ -1719,13 +1482,6 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { strcpy(pName.dbname, pRequest->pDb); strcpy(pName.tname, tbName); - VgData vgData = {0}; - code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &(vgData.vg)); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName); - goto end; - } - code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { uError("WriteRaw:catalogGetTableMeta table not exist. table name: %s", tbName); @@ -1737,164 +1493,29 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { goto end; } - uint16_t fLen = 0; - int32_t rowSize = 0; - int16_t nVar = 0; - for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - SSchema* schema = &pTableMeta->schema[i]; - fLen += TYPE_BYTES[schema->type]; - rowSize += schema->bytes; - if (IS_VAR_DATA_TYPE(schema->type)) { - nVar++; - } - } - fLen -= sizeof(TSKEY); - - int32_t rows = rspObj.resInfo.numOfRows; - int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(pTableMeta->tableInfo.numOfColumns - 1); - int32_t schemaLen = 0; - int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; - - SSubmitReq* subReq = NULL; - SSubmitBlk* blk = NULL; - void* hData = taosHashGet(pVgHash, &vgData.vg.vgId, sizeof(vgData.vg.vgId)); - if (hData) { - vgData = *(VgData*)hData; - - int32_t totalLen = ((SSubmitReq*)(vgData.data))->length + submitLen; - void* tmp = taosMemoryRealloc(vgData.data, totalLen); - if (tmp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - vgData.data = tmp; - ((VgData*)hData)->data = tmp; - subReq = (SSubmitReq*)(vgData.data); - blk = POINTER_SHIFT(vgData.data, subReq->length); - } else { - int32_t totalLen = sizeof(SSubmitReq) + submitLen; - void* tmp = taosMemoryCalloc(1, totalLen); - if (tmp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - vgData.data = tmp; - taosHashPut(pVgHash, (const char*)&vgData.vg.vgId, sizeof(vgData.vg.vgId), (char*)&vgData, sizeof(vgData)); - subReq = (SSubmitReq*)(vgData.data); - subReq->length = sizeof(SSubmitReq); - subReq->numOfBlocks = 0; - - blk = POINTER_SHIFT(vgData.data, sizeof(SSubmitReq)); + SVgroupInfo vg; + code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName); + goto end; } - // pSW->pSchema should be same as pTableMeta->schema - // ASSERT(pSW->nCols == pTableMeta->tableInfo.numOfColumns); - uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); - uint64_t uid = pTableMeta->uid; - int16_t sver = pTableMeta->sversion; - - void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); - STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - - SRowBuilder rb = {0}; - tdSRowInit(&rb, sver); - tdSRowSetTpInfo(&rb, pTableMeta->tableInfo.numOfColumns, fLen); - int32_t totalLen = 0; - - SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - for (int i = 0; i < pSW->nCols; i++) { - SSchema* schema = &pSW->pSchema[i]; - taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); + void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId)); + if (hData == NULL) { + taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); } - for (int32_t j = 0; j < rows; j++) { - tdSRowResetBuf(&rb, rowData); - - doSetOneRowPtr(&rspObj.resInfo); - rspObj.resInfo.current += 1; - - int32_t offset = 0; - for (int32_t k = 0; k < pTableMeta->tableInfo.numOfColumns; k++) { - const SSchema* pColumn = &pTableMeta->schema[k]; - int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); - if (!index) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k); - } else { - char* colData = rspObj.resInfo.row[*index]; - if (!colData) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } else { - if (IS_VAR_DATA_TYPE(pColumn->type)) { - colData -= VARSTR_HEADER_SIZE; - } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); - } - } - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } - } - tdSRowEnd(&rb); - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - totalLen += rowLen; + code = rawBlockBindData(pQuery, pTableMeta, pRetrieve->data, NULL, NULL, 0); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:rawBlockBindData failed"); + goto end; } - - taosHashCleanup(schemaHash); - blk->uid = htobe64(uid); - blk->suid = htobe64(suid); - blk->sversion = htonl(sver); - blk->schemaLen = htonl(schemaLen); - blk->numOfRows = htonl(rows); - blk->dataLen = htonl(totalLen); - subReq->length += sizeof(SSubmitBlk) + schemaLen + totalLen; - subReq->numOfBlocks++; - taosMemoryFreeClear(pTableMeta); - rspObj.resInfo.pRspMsg = NULL; - doFreeReqResultInfo(&rspObj.resInfo); } - pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == pQuery) { - uError("create SQuery error"); - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->haveResultSet = false; - pQuery->msgType = TDMT_VND_SUBMIT; - pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); - if (NULL == pQuery->pRoot) { - uError("create pQuery->pRoot error"); - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - SVnodeModifOpStmt* nodeStmt = (SVnodeModifOpStmt*)(pQuery->pRoot); - - int32_t numOfVg = taosHashGetSize(pVgHash); - nodeStmt->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); - - VgData* vData = (VgData*)taosHashIterate(pVgHash, NULL); - while (vData) { - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - dst->vg = vData->vg; - SSubmitReq* subReq = (SSubmitReq*)(vData->data); - dst->numOfTables = subReq->numOfBlocks; - dst->size = subReq->length; - dst->pData = (char*)subReq; - vData->data = NULL; // no need free - subReq->header.vgId = htonl(dst->vg.vgId); - subReq->version = htonl(1); - subReq->header.contLen = htonl(subReq->length); - subReq->length = htonl(subReq->length); - subReq->numOfBlocks = htonl(subReq->numOfBlocks); - taosArrayPush(nodeStmt->pDataBlocks, &dst); - vData = (VgData*)taosHashIterate(pVgHash, vData); + code = smlBuildOutput(pQuery, pVgHash); + if (code != TSDB_CODE_SUCCESS) { + uError("smlBuildOutput failed"); + return code; } launchQueryImpl(pRequest, pQuery, true, NULL); @@ -1902,8 +1523,6 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { end: tDeleteSMqDataRsp(&rspObj.rsp); - rspObj.resInfo.pRspMsg = NULL; - doFreeReqResultInfo(&rspObj.resInfo); tDecoderClear(&decoder); qDestroyQuery(pQuery); destroyRequest(pRequest); @@ -1912,7 +1531,6 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { return code; } - static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) { int32_t code = TSDB_CODE_SUCCESS; SHashObj* pVgHash = NULL; @@ -1946,8 +1564,6 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) goto end; } - pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - taosHashSetFreeFp(pVgHash, destroyVgHash); struct SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { @@ -1961,6 +1577,13 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) conn.requestObjRefId = pRequest->self; conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + pQuery = smlInitHandle(); + if (pQuery == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + uDebug("raw data block num:%d\n", rspObj.rsp.blockNum); while (++rspObj.resIter < rspObj.rsp.blockNum) { SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); @@ -1968,14 +1591,6 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) uError("WriteRaw:no schema, iter:%d", rspObj.resIter); goto end; } - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter); - setResSchemaInfo(&rspObj.resInfo, pSW->pSchema, pSW->nCols); - - code = setQueryResultFromRsp(&rspObj.resInfo, pRetrieve, false, false); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw: setQueryResultFromRsp error"); - goto end; - } const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.blockTbName, rspObj.resIter); if (!tbName) { @@ -1989,44 +1604,32 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) strcpy(pName.dbname, pRequest->pDb); strcpy(pName.tname, tbName); - VgData vgData = {0}; - code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &(vgData.vg)); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName); - goto end; - } - // find schema data info - int32_t schemaLen = 0; - void* schemaData = NULL; + SVCreateTbReq pCreateReq = {0}; + for (int j = 0; j < rspObj.rsp.createTableNum; j++) { void** dataTmp = taosArrayGet(rspObj.rsp.createTableReq, j); int32_t* lenTmp = taosArrayGet(rspObj.rsp.createTableLen, j); - SDecoder decoderTmp = {0}; - SVCreateTbReq pCreateReq = {0}; - + SDecoder decoderTmp = {0}; tDecoderInit(&decoderTmp, *dataTmp, *lenTmp); + memset(&pCreateReq, 0, sizeof(SVCreateTbReq)); if (tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq) < 0) { tDecoderClear(&decoderTmp); - taosMemoryFreeClear(pCreateReq.comment); - taosArrayDestroy(pCreateReq.ctb.tagName); goto end; } - ASSERT(pCreateReq.type == TSDB_CHILD_TABLE); + if(pCreateReq.type != TSDB_CHILD_TABLE){ + uError("WriteRaw:pCreateReq.type != TSDB_CHILD_TABLE. table name: %s", tbName); + code = TSDB_CODE_TSC_INVALID_VALUE; + goto end; + } if (strcmp(tbName, pCreateReq.name) == 0) { - schemaLen = *lenTmp; - schemaData = *dataTmp; strcpy(pName.tname, pCreateReq.ctb.stbName); tDecoderClear(&decoderTmp); - taosMemoryFreeClear(pCreateReq.comment); - taosArrayDestroy(pCreateReq.ctb.tagName); break; } tDecoderClear(&decoderTmp); - taosMemoryFreeClear(pCreateReq.comment); - taosArrayDestroy(pCreateReq.ctb.tagName); } code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); @@ -2040,176 +1643,30 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) goto end; } - uint16_t fLen = 0; - int32_t rowSize = 0; - int16_t nVar = 0; - for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - SSchema* schema = &pTableMeta->schema[i]; - fLen += TYPE_BYTES[schema->type]; - rowSize += schema->bytes; - if (IS_VAR_DATA_TYPE(schema->type)) { - nVar++; - } - } - fLen -= sizeof(TSKEY); - - int32_t rows = rspObj.resInfo.numOfRows; - int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(pTableMeta->tableInfo.numOfColumns - 1); - - int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; - - SSubmitReq* subReq = NULL; - SSubmitBlk* blk = NULL; - void* hData = taosHashGet(pVgHash, &vgData.vg.vgId, sizeof(vgData.vg.vgId)); - if (hData) { - vgData = *(VgData*)hData; - - int32_t totalLen = ((SSubmitReq*)(vgData.data))->length + submitLen; - void* tmp = taosMemoryRealloc(vgData.data, totalLen); - if (tmp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - vgData.data = tmp; - ((VgData*)hData)->data = tmp; - subReq = (SSubmitReq*)(vgData.data); - blk = POINTER_SHIFT(vgData.data, subReq->length); - } else { - int32_t totalLen = sizeof(SSubmitReq) + submitLen; - void* tmp = taosMemoryCalloc(1, totalLen); - if (tmp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - vgData.data = tmp; - taosHashPut(pVgHash, (const char*)&vgData.vg.vgId, sizeof(vgData.vg.vgId), (char*)&vgData, sizeof(vgData)); - subReq = (SSubmitReq*)(vgData.data); - subReq->length = sizeof(SSubmitReq); - subReq->numOfBlocks = 0; - - blk = POINTER_SHIFT(vgData.data, sizeof(SSubmitReq)); - } - - // pSW->pSchema should be same as pTableMeta->schema - // ASSERT(pSW->nCols == pTableMeta->tableInfo.numOfColumns); - uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); - uint64_t uid = pTableMeta->uid; - int16_t sver = pTableMeta->sversion; - - void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); - if (schemaData) { - memcpy(blkSchema, schemaData, schemaLen); - } - STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - - SRowBuilder rb = {0}; - tdSRowInit(&rb, sver); - tdSRowSetTpInfo(&rb, pTableMeta->tableInfo.numOfColumns, fLen); - int32_t totalLen = 0; - - SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - for (int i = 0; i < pSW->nCols; i++) { - SSchema* schema = &pSW->pSchema[i]; - taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); + SVgroupInfo vg; + code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName); + goto end; } - for (int32_t j = 0; j < rows; j++) { - tdSRowResetBuf(&rb, rowData); - - doSetOneRowPtr(&rspObj.resInfo); - rspObj.resInfo.current += 1; - - int32_t offset = 0; - for (int32_t k = 0; k < pTableMeta->tableInfo.numOfColumns; k++) { - const SSchema* pColumn = &pTableMeta->schema[k]; - int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); - if (!index) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k); - } else { - char* colData = rspObj.resInfo.row[*index]; - if (!colData) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } else { - if (IS_VAR_DATA_TYPE(pColumn->type)) { - colData -= VARSTR_HEADER_SIZE; - } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); - } - } - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } - } - tdSRowEnd(&rb); - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - totalLen += rowLen; + void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId)); + if (hData == NULL) { + taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); } - taosHashCleanup(schemaHash); - blk->uid = htobe64(uid); - blk->suid = htobe64(suid); - blk->sversion = htonl(sver); - blk->schemaLen = htonl(schemaLen); - blk->numOfRows = htonl(rows); - blk->dataLen = htonl(totalLen); - subReq->length += sizeof(SSubmitBlk) + schemaLen + totalLen; - subReq->numOfBlocks++; - taosMemoryFreeClear(pTableMeta); - rspObj.resInfo.pRspMsg = NULL; - doFreeReqResultInfo(&rspObj.resInfo); - } - - pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == pQuery) { - uError("create SQuery error"); - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->haveResultSet = false; - pQuery->msgType = TDMT_VND_SUBMIT; - pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); - if (NULL == pQuery->pRoot) { - uError("create pQuery->pRoot error"); - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - SVnodeModifOpStmt* nodeStmt = (SVnodeModifOpStmt*)(pQuery->pRoot); - - int32_t numOfVg = taosHashGetSize(pVgHash); - nodeStmt->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); - - VgData* vData = (VgData*)taosHashIterate(pVgHash, NULL); - while (vData) { - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = rawBlockBindData(pQuery, pTableMeta, pRetrieve->data, &pCreateReq, NULL, 0); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:rawBlockBindData failed"); goto end; } - dst->vg = vData->vg; - SSubmitReq* subReq = (SSubmitReq*)(vData->data); - dst->numOfTables = subReq->numOfBlocks; - dst->size = subReq->length; - dst->pData = (char*)subReq; - vData->data = NULL; // no need free - subReq->header.vgId = htonl(dst->vg.vgId); - subReq->version = htonl(1); - subReq->header.contLen = htonl(subReq->length); - subReq->length = htonl(subReq->length); - subReq->numOfBlocks = htonl(subReq->numOfBlocks); - taosArrayPush(nodeStmt->pDataBlocks, &dst); - vData = (VgData*)taosHashIterate(pVgHash, vData); } launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; - end: +end: tDeleteSTaosxRsp(&rspObj.rsp); - rspObj.resInfo.pRspMsg = NULL; - doFreeReqResultInfo(&rspObj.resInfo); tDecoderClear(&decoder); qDestroyQuery(pQuery); destroyRequest(pRequest); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index c24aa536c217e34027a3407bef43cba82911fa7a..f25456999b44f1d44db55c9071563b3bc52baecd 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1,220 +1,101 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #include #include #include #include -#include "cJSON.h" -#include "catalog.h" -#include "clientInt.h" -#include "osSemaphore.h" -#include "osThread.h" -#include "query.h" -#include "taos.h" -#include "taoserror.h" -#include "tcommon.h" -#include "tdef.h" -#include "tglobal.h" -#include "tlog.h" -#include "tmsg.h" -#include "tname.h" -#include "ttime.h" -#include "ttypes.h" - -//================================================================================================= - -#define SPACE ' ' -#define COMMA ',' -#define EQUAL '=' -#define QUOTE '"' -#define SLASH '\\' - -#define JUMP_SPACE(sql, sqlEnd) \ - while (sql < sqlEnd) { \ - if (*sql == SPACE) \ - sql++; \ - else \ - break; \ - } -// comma , -#define IS_SLASH_COMMA(sql) (*(sql) == COMMA && *((sql)-1) == SLASH) -#define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH) -// space -#define IS_SLASH_SPACE(sql) (*(sql) == SPACE && *((sql)-1) == SLASH) -#define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH) -// equal = -#define IS_SLASH_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) == SLASH) -#define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH) -// quote " -#define IS_SLASH_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) == SLASH) -#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH) -// SLASH -#define IS_SLASH_SLASH(sql) (*(sql) == SLASH && *((sql)-1) == SLASH) - -#define IS_SLASH_LETTER(sql) \ - (IS_SLASH_COMMA(sql) || IS_SLASH_SPACE(sql) || IS_SLASH_EQUAL(sql) || IS_SLASH_QUOTE(sql) || IS_SLASH_SLASH(sql)) - -#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) - -#define PROCESS_SLASH(key, keyLen) \ - for (int i = 1; i < keyLen; ++i) { \ - if (IS_SLASH_LETTER(key + i)) { \ - MOVE_FORWARD_ONE(key + i, keyLen - i); \ - i--; \ - keyLen--; \ - } \ - } - -#define IS_INVALID_COL_LEN(len) ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN) -#define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN) - -#define OTD_JSON_SUB_FIELDS_NUM 2 -#define OTD_JSON_FIELDS_NUM 4 - -#define TS "_ts" -#define TS_LEN 3 -#define VALUE "_value" -#define VALUE_LEN 6 - -#define BINARY_ADD_LEN 2 // "binary" 2 means " " -#define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" " - -#define MAX_RETRY_TIMES 5 -//================================================================================================= -typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; - -typedef enum { - SCHEMA_ACTION_NULL, - SCHEMA_ACTION_CREATE_STABLE, - SCHEMA_ACTION_ADD_COLUMN, - SCHEMA_ACTION_ADD_TAG, - SCHEMA_ACTION_CHANGE_COLUMN_SIZE, - SCHEMA_ACTION_CHANGE_TAG_SIZE, -} ESchemaAction; - -typedef struct { - const char *measure; - const char *tags; - const char *cols; - const char *timestamp; - - int32_t measureLen; - int32_t measureTagsLen; - int32_t tagsLen; - int32_t colsLen; - int32_t timestampLen; -} SSmlLineInfo; - -typedef struct { - const char *sTableName; // super table name - int32_t sTableNameLen; - char childTableName[TSDB_TABLE_NAME_LEN]; - uint64_t uid; - - SArray *tags; - - // if info->formatData is true, elements are SArray. - // if info->formatData is false, elements are SHashObj for find by key quickly - SArray *cols; -} SSmlTableInfo; - -typedef struct { - SArray *tags; // save the origin order to create table - SHashObj *tagHash; // elements are - - SArray *cols; - SHashObj *colHash; - - STableMeta *tableMeta; -} SSmlSTableMeta; - -typedef struct { - int32_t len; - char *buf; -} SSmlMsgBuf; - -typedef struct { - int32_t code; - int32_t lineNum; - - int32_t numOfSTables; - int32_t numOfCTables; - int32_t numOfCreateSTables; - int32_t numOfAlterColSTables; - int32_t numOfAlterTagSTables; - - int64_t parseTime; - int64_t schemaTime; - int64_t insertBindTime; - int64_t insertRpcTime; - int64_t endTime; -} SSmlCostInfo; - -typedef struct { - SRequestObj *request; - tsem_t sem; - int32_t cnt; - int32_t total; - TdThreadSpinlock lock; -} Params; - -typedef struct { - int64_t id; - Params *params; - - SMLProtocolType protocol; - int8_t precision; - bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) - bool isRawLine; - int32_t ttl; - - SHashObj *childTables; - SHashObj *superTables; - SHashObj *pVgHash; - void *exec; - - STscObj *taos; - SCatalog *pCatalog; - SRequestObj *pRequest; - SQuery *pQuery; - - SSmlCostInfo cost; - int32_t affectedRows; - SSmlMsgBuf msgBuf; - SHashObj *dumplicateKey; // for dumplicate key - SArray *colsContainer; // for cols parse, if dataFormat == false - int32_t uid; // used for automatic create child table - - cJSON *root; // for parse json -} SSmlHandle; -//================================================================================================= - -//================================================================================================= -static volatile int64_t linesSmlHandleId = 0; -static int64_t smlGenId() { - int64_t id; - - do { - id = atomic_add_fetch_64(&linesSmlHandleId, 1); - } while (id == 0); +#include "clientSml.h" + +int64_t smlToMilli[3] = {3600000LL, 60000LL, 1000LL}; +int64_t smlFactorNS[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; +int64_t smlFactorS[3] = {1000LL, 1000000LL, 1000000000LL}; + +void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) { + NodeList *tmp = list; + while (tmp) { + if (fn == NULL) { + if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { + return tmp->data.value; + } + } else { + if (tmp->data.used && fn(tmp->data.key, key) == 0) { + return tmp->data.value; + } + } - return id; + tmp = tmp->next; + } + return NULL; } -static inline bool smlDoubleToInt64OverFlow(double num) { - if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; - return false; +int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) { + NodeList *tmp = *list; + while (tmp) { + if (!tmp->data.used) break; + if (fn == NULL) { + if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { + return -1; + } + } else { + if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) { + return -1; + } + } + + tmp = tmp->next; + } + if (tmp) { + tmp->data.key = key; + tmp->data.keyLen = len; + tmp->data.value = value; + tmp->data.used = true; + } else { + NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList)); + if (newNode == NULL) { + return -1; + } + newNode->data.key = key; + newNode->data.keyLen = len; + newNode->data.value = value; + newNode->data.used = true; + newNode->next = *list; + *list = newNode; + } + return 0; } -static inline bool smlCheckDuplicateKey(const char *key, int32_t keyLen, SHashObj *pHash) { - void *val = taosHashGet(pHash, key, keyLen); - if (val) { - return true; +int nodeListSize(NodeList *list) { + int cnt = 0; + while (list) { + if (list->data.used) + cnt++; + else + break; + list = list->next; } - taosHashPut(pHash, key, keyLen, key, 1); + return cnt; +} + +inline bool smlDoubleToInt64OverFlow(double num) { + if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; return false; } -static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { +int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { if (pBuf->buf) { memset(pBuf->buf, 0, pBuf->len); if (msg1) strncat(pBuf->buf, msg1, pBuf->len); @@ -227,391 +108,307 @@ static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const return TSDB_CODE_SML_INVALID_DATA; } -static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSmlKv *kv, bool isTag, - ESchemaAction *action, SSmlHandle *info) { - uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL; - if (index) { - if (colField[*index].type != kv->type) { - uError("SML:0x%" PRIx64 " point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, - kv->key, colField[*index].type, kv->type); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if ((colField[*index].type == TSDB_DATA_TYPE_VARCHAR && - (colField[*index].bytes - VARSTR_HEADER_SIZE) < kv->length) || - (colField[*index].type == TSDB_DATA_TYPE_NCHAR && - ((colField[*index].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE < kv->length))) { - if (isTag) { - *action = SCHEMA_ACTION_CHANGE_TAG_SIZE; - } else { - *action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE; - } - } - } else { - if (isTag) { - *action = SCHEMA_ACTION_ADD_TAG; - } else { - *action = SCHEMA_ACTION_ADD_COLUMN; - } +int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision) { + char *endPtr = NULL; + int64_t tsInt64 = taosStr2Int64(value, &endPtr, 10); + if (unlikely(value + len != endPtr)) { + return -1; } - return 0; -} -static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { - int32_t result = 1; - while (result <= length) { - result *= 2; - } - if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { - result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE; - } else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - result = (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + if (unlikely(fromPrecision >= TSDB_TIME_PRECISION_HOURS)) { + int64_t unit = smlToMilli[fromPrecision - TSDB_TIME_PRECISION_HOURS]; + if (unit > INT64_MAX / tsInt64) { + return -1; + } + tsInt64 *= unit; + fromPrecision = TSDB_TIME_PRECISION_MILLI; } - if (type == TSDB_DATA_TYPE_NCHAR) { - result = result * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; - } else if (type == TSDB_DATA_TYPE_BINARY) { - result = result + VARSTR_HEADER_SIZE; - } - return result; + return convertTimePrecision(tsInt64, fromPrecision, toPrecision); } -static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, - ESchemaAction *action, bool isTag) { - int32_t code = TSDB_CODE_SUCCESS; - for (int j = 0; j < taosArrayGetSize(cols); ++j) { - if (j == 0 && !isTag) continue; - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j); - code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info); - if (code != TSDB_CODE_SUCCESS) { - return code; - } +int8_t smlGetTsTypeByLen(int32_t len) { + if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { + return TSDB_TIME_PRECISION_SECONDS; + } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { + return TSDB_TIME_PRECISION_MILLI; + } else { + return -1; } - return TSDB_CODE_SUCCESS; } -static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { - SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - int32_t i = 0; - for (; i < length; i++) { - taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); +SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen) { + SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); + if (!tag) { + return NULL; } - if (isTag) { - i = 0; - } else { - i = 1; - } - for (; i < taosArrayGetSize(cols); i++) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); - if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { - taosHashCleanup(hashTmp); - return -1; - } - } - taosHashCleanup(hashTmp); - return 0; -} + tag->sTableName = measure; + tag->sTableNameLen = measureLen; -static int32_t getBytes(uint8_t type, int32_t length) { - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - return smlFindNearestPowerOf2(length, type); - } else { - return tDataTypes[type].bytes; + tag->cols = taosArrayInit(numRows, POINTER_BYTES); + if (tag->cols == NULL) { + uError("SML:smlBuildTableInfo failed to allocate memory"); + goto cleanup; } + + // tag->tags = taosArrayInit(16, sizeof(SSmlKv)); + // if (tag->tags == NULL) { + // uError("SML:smlBuildTableInfo failed to allocate memory"); + // goto cleanup; + // } + return tag; + +cleanup: + taosMemoryFree(tag); + return NULL; } -static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, - SArray *results, int32_t numOfCols, bool isTag) { - for (int j = 0; j < taosArrayGetSize(cols); ++j) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j); - ESchemaAction action = SCHEMA_ACTION_NULL; - smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); - if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) { - SField field = {0}; - field.type = kv->type; - field.bytes = getBytes(kv->type, kv->length); - memcpy(field.name, kv->key, kv->keyLen); - taosArrayPush(results, &field); - } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { - uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); - uint16_t newIndex = *index; - if (isTag) newIndex -= numOfCols; - SField *field = (SField *)taosArrayGet(results, newIndex); - field->bytes = getBytes(kv->type, kv->length); +static int32_t smlParseTableName(SArray *tags, char *childTableName) { + size_t childTableNameLen = strlen(tsSmlChildTableName); + if (childTableNameLen <= 0) return TSDB_CODE_SUCCESS; + + for (int i = 0; i < taosArrayGetSize(tags); i++) { + SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); + // handle child table name + if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) { + memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); + break; } } + return TSDB_CODE_SUCCESS; } -// static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData, -// int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){ -static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta, - ESchemaAction action) { - SRequestObj *pRequest = NULL; - SMCreateStbReq pReq = {0}; - int32_t code = TSDB_CODE_SUCCESS; - SCmdMsgInfo pCmdMsg = {0}; +int32_t smlSetCTableName(SSmlTableInfo *oneTable) { + smlParseTableName(oneTable->tags, oneTable->childTableName); - // put front for free - pReq.numOfColumns = taosArrayGetSize(pColumns); - pReq.pColumns = pColumns; - pReq.numOfTags = taosArrayGetSize(pTags); - pReq.pTags = pTags; + if (strlen(oneTable->childTableName) == 0) { + SArray *dst = taosArrayDup(oneTable->tags, NULL); + RandTableName rName = {dst, oneTable->sTableName, (uint8_t)oneTable->sTableNameLen, oneTable->childTableName}; - code = buildRequest(info->taos->id, "", 0, NULL, false, &pRequest, 0); - if (code != TSDB_CODE_SUCCESS) { - goto end; + buildChildTableName(&rName); + taosArrayDestroy(dst); } + return TSDB_CODE_SUCCESS; +} - pRequest->syncQuery = true; - if (!pRequest->pDb) { - code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; - goto end; +SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { + SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); + if (!meta) { + return NULL; } - if (action == SCHEMA_ACTION_CREATE_STABLE) { - pReq.colVer = 1; - pReq.tagVer = 1; - pReq.suid = 0; - pReq.source = TD_REQ_FROM_APP; - } else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { - pReq.colVer = pTableMeta->sversion; - pReq.tagVer = pTableMeta->tversion + 1; - pReq.suid = pTableMeta->uid; - pReq.source = TD_REQ_FROM_TAOX; - } else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE) { - pReq.colVer = pTableMeta->sversion + 1; - pReq.tagVer = pTableMeta->tversion; - pReq.suid = pTableMeta->uid; - pReq.source = TD_REQ_FROM_TAOX; - } + if (unlikely(!isDataFormat)) { + meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (meta->tagHash == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } - if (pReq.numOfTags == 0) { - pReq.numOfTags = 1; - SField field = {0}; - field.type = TSDB_DATA_TYPE_NCHAR; - field.bytes = 1; - strcpy(field.name, tsSmlTagName); - taosArrayPush(pReq.pTags, &field); + meta->colHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (meta->colHash == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } } - pReq.commentLen = -1; - pReq.igExists = true; - tNameExtractFullName(pName, pReq.name); - - pCmdMsg.epSet = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - pCmdMsg.msgType = TDMT_MND_CREATE_STB; - pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq); - pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); - if (NULL == pCmdMsg.pMsg) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; + meta->tags = taosArrayInit(32, sizeof(SSmlKv)); + if (meta->tags == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; } - tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); - - SQuery pQuery; - memset(&pQuery, 0, sizeof(pQuery)); - pQuery.execMode = QUERY_EXEC_MODE_RPC; - pQuery.pCmdMsg = &pCmdMsg; - pQuery.msgType = pQuery.pCmdMsg->msgType; - pQuery.stableQuery = true; - - launchQueryImpl(pRequest, &pQuery, true, NULL); - if (pRequest->code == TSDB_CODE_SUCCESS) { - catalogRemoveTableMeta(info->pCatalog, pName); + meta->cols = taosArrayInit(32, sizeof(SSmlKv)); + if (meta->cols == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; } - code = pRequest->code; - taosMemoryFree(pCmdMsg.pMsg); + return meta; -end: - destroyRequest(pRequest); - tFreeSMCreateStbReq(&pReq); - return code; +cleanup: + taosMemoryFree(meta); + return NULL; } -static int32_t smlModifyDBSchemas(SSmlHandle *info) { - int32_t code = 0; - SHashObj *hashTmp = NULL; - STableMeta *pTableMeta = NULL; +// uint16_t smlCalTypeSum(char* endptr, int32_t left){ +// uint16_t sum = 0; +// for(int i = 0; i < left; i++){ +// sum += endptr[i]; +// } +// return sum; +// } - SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; - tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); +#define RETURN_FALSE \ + smlBuildInvalidDataMsg(msg, "invalid data", pVal); \ + return false; - SRequestConnInfo conn = {0}; - conn.pTrans = info->taos->pAppInfo->pTransporter; - conn.requestId = info->pRequest->requestId; - conn.requestObjRefId = info->pRequest->self; - conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - - SSmlSTableMeta **tableMetaSml = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); - while (tableMetaSml) { - SSmlSTableMeta *sTableData = *tableMetaSml; - bool needCheckMeta = false; // for multi thread - - size_t superTableLen = 0; - void *superTable = taosHashGetKey(tableMetaSml, &superTableLen); - memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - memcpy(pName.tname, superTable, superTableLen); - - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - - if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) { - SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); - SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); - smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true); - smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false); - - code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); - goto end; - } - info->cost.numOfCreateSTables++; - taosMemoryFreeClear(pTableMeta); - - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); - goto end; - } - } else if (code == TSDB_CODE_SUCCESS) { - hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, - HASH_NO_LOCK); - for (uint16_t i = pTableMeta->tableInfo.numOfColumns; - i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); - } - - ESchemaAction action = SCHEMA_ACTION_NULL; - code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->tags, &action, true); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - if (action != SCHEMA_ACTION_NULL) { - SArray *pColumns = - taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); - SArray *pTags = - taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - SField field = {0}; - field.type = pTableMeta->schema[i].type; - field.bytes = pTableMeta->schema[i].bytes; - strcpy(field.name, pTableMeta->schema[i].name); - if (i < pTableMeta->tableInfo.numOfColumns) { - taosArrayPush(pColumns, &field); - } else { - taosArrayPush(pTags, &field); - } - } - smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, - pTableMeta->tableInfo.numOfColumns, true); - - code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); - goto end; - } - - info->cost.numOfAlterTagSTables++; - taosMemoryFreeClear(pTableMeta); - code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - } +#define SET_DOUBLE \ + kvVal->type = TSDB_DATA_TYPE_DOUBLE; \ + kvVal->d = result; + +#define SET_FLOAT \ + if (!IS_VALID_FLOAT(result)) { \ + smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_FLOAT; \ + kvVal->f = (float)result; + +#define SET_BIGINT \ + if (smlDoubleToInt64OverFlow(result)) { \ + errno = 0; \ + int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \ + if (errno == ERANGE) { \ + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_BIGINT; \ + kvVal->i = tmp; \ + return true; \ + } \ + kvVal->type = TSDB_DATA_TYPE_BIGINT; \ + kvVal->i = (int64_t)result; + +#define SET_INT \ + if (!IS_VALID_INT(result)) { \ + smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_INT; \ + kvVal->i = result; + +#define SET_SMALL_INT \ + if (!IS_VALID_SMALLINT(result)) { \ + smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_SMALLINT; \ + kvVal->i = result; + +#define SET_UBIGINT \ + if (result >= (double)UINT64_MAX || result < 0) { \ + errno = 0; \ + uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \ + if (errno == ERANGE || result < 0) { \ + smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ + kvVal->u = tmp; \ + return true; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ + kvVal->u = result; + +#define SET_UINT \ + if (!IS_VALID_UINT(result)) { \ + smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UINT; \ + kvVal->u = result; + +#define SET_USMALL_INT \ + if (!IS_VALID_USMALLINT(result)) { \ + smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_USMALLINT; \ + kvVal->u = result; + +#define SET_TINYINT \ + if (!IS_VALID_TINYINT(result)) { \ + smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_TINYINT; \ + kvVal->i = result; + +#define SET_UTINYINT \ + if (!IS_VALID_UTINYINT(result)) { \ + smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UTINYINT; \ + kvVal->u = result; + +bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->length; + char *endptr = NULL; + double result = taosStr2Double(pVal, &endptr); + if (pVal == endptr) { + RETURN_FALSE + } - taosHashClear(hashTmp); - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + int32_t left = len - (endptr - pVal); + if (left == 0) { + SET_DOUBLE + } else if (left == 3) { + if (endptr[0] == 'f' || endptr[0] == 'F') { + if (endptr[1] == '6' && endptr[2] == '4') { + SET_DOUBLE + } else if (endptr[1] == '3' && endptr[2] == '2') { + SET_FLOAT + } else { + RETURN_FALSE } - action = SCHEMA_ACTION_NULL; - code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, &action, false); - if (code != TSDB_CODE_SUCCESS) { - goto end; + } else if (endptr[0] == 'i' || endptr[0] == 'I') { + if (endptr[1] == '6' && endptr[2] == '4') { + SET_BIGINT + } else if (endptr[1] == '3' && endptr[2] == '2') { + SET_INT + } else if (endptr[1] == '1' && endptr[2] == '6') { + SET_SMALL_INT + } else { + RETURN_FALSE } - if (action != SCHEMA_ACTION_NULL) { - SArray *pColumns = - taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); - SArray *pTags = - taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - SField field = {0}; - field.type = pTableMeta->schema[i].type; - field.bytes = pTableMeta->schema[i].bytes; - strcpy(field.name, pTableMeta->schema[i].name); - if (i < pTableMeta->tableInfo.numOfColumns) { - taosArrayPush(pColumns, &field); - } else { - taosArrayPush(pTags, &field); - } - } - - smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns, - pTableMeta->tableInfo.numOfColumns, false); - - code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); - goto end; - } - - info->cost.numOfAlterColSTables++; - taosMemoryFreeClear(pTableMeta); - code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); - goto end; - } + } else if (endptr[0] == 'u' || endptr[0] == 'U') { + if (endptr[1] == '6' && endptr[2] == '4') { + SET_UBIGINT + } else if (endptr[1] == '3' && endptr[2] == '2') { + SET_UINT + } else if (endptr[1] == '1' && endptr[2] == '6') { + SET_USMALL_INT + } else { + RETURN_FALSE } - - needCheckMeta = true; - taosHashCleanup(hashTmp); - hashTmp = NULL; } else { - uError("SML:0x%" PRIx64 " load table meta error: %s", info->id, tstrerror(code)); - goto end; + RETURN_FALSE } - - if (needCheckMeta) { - code = smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags, - sTableData->tags, true); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " check tag failed. super table name %s", info->id, pName.tname); - goto end; + } else if (left == 2) { + if (endptr[0] == 'i' || endptr[0] == 'I') { + if (endptr[1] == '8') { + SET_TINYINT + } else { + RETURN_FALSE } - code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols, false); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " check cols failed. super table name %s", info->id, pName.tname); - goto end; + } else if (endptr[0] == 'u' || endptr[0] == 'U') { + if (endptr[1] == '8') { + SET_UTINYINT + } else { + RETURN_FALSE } + } else { + RETURN_FALSE } - - sTableData->tableMeta = pTableMeta; - - tableMetaSml = (SSmlSTableMeta **)taosHashIterate(info->superTables, tableMetaSml); + } else if (left == 1) { + if (endptr[0] == 'i' || endptr[0] == 'I') { + SET_BIGINT + } else if (endptr[0] == 'u' || endptr[0] == 'U') { + SET_UBIGINT + } else { + RETURN_FALSE + } + } else { + RETURN_FALSE; } - return 0; - -end: - taosHashCleanup(hashTmp); - taosMemoryFreeClear(pTableMeta); -// catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); - return code; + return true; } -static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { +bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->length; char *endptr = NULL; @@ -709,764 +506,462 @@ static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { return true; } -static bool smlParseBool(SSmlKv *kvVal) { - const char *pVal = kvVal->value; - int32_t len = kvVal->length; - if ((len == 1) && (pVal[0] == 't' || pVal[0] == 'T')) { - kvVal->i = true; - return true; - } +STableMeta *smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen) { + STableMeta *pTableMeta = NULL; - if ((len == 1) && (pVal[0] == 'f' || pVal[0] == 'F')) { - kvVal->i = false; - return true; - } + SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; + tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); - if ((len == 4) && !strncasecmp(pVal, "true", len)) { - kvVal->i = true; - return true; - } - if ((len == 5) && !strncasecmp(pVal, "false", len)) { - kvVal->i = false; - return true; - } - return false; -} + SRequestConnInfo conn = {0}; + conn.pTrans = info->taos->pAppInfo->pTransporter; + conn.requestId = info->pRequest->requestId; + conn.requestObjRefId = info->pRequest->self; + conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + memcpy(pName.tname, measure, measureLen); -static bool smlIsBinary(const char *pVal, uint16_t len) { - // binary: "abc" - if (len < 2) { - return false; - } - if (pVal[0] == '"' && pVal[len - 1] == '"') { - return true; + int32_t code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + return NULL; } - return false; + return pTableMeta; } -static bool smlIsNchar(const char *pVal, uint16_t len) { - // nchar: L"abc" - if (len < 3) { - return false; - } - if ((pVal[0] == 'l' || pVal[0] == 'L') && pVal[1] == '"' && pVal[len - 1] == '"') { - return true; - } - return false; -} +static int64_t smlGenId() { + static volatile int64_t linesSmlHandleId = 0; -static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { - char *endPtr = NULL; - int64_t tsInt64 = taosStr2Int64(value, &endPtr, 10); - if (value + len != endPtr) { - return -1; - } - double ts = tsInt64; - switch (type) { - case TSDB_TIME_PRECISION_HOURS: - ts *= NANOSECOND_PER_HOUR; - tsInt64 *= NANOSECOND_PER_HOUR; - break; - case TSDB_TIME_PRECISION_MINUTES: - ts *= NANOSECOND_PER_MINUTE; - tsInt64 *= NANOSECOND_PER_MINUTE; - break; - case TSDB_TIME_PRECISION_SECONDS: - ts *= NANOSECOND_PER_SEC; - tsInt64 *= NANOSECOND_PER_SEC; - break; - case TSDB_TIME_PRECISION_MILLI: - ts *= NANOSECOND_PER_MSEC; - tsInt64 *= NANOSECOND_PER_MSEC; - break; - case TSDB_TIME_PRECISION_MICRO: - ts *= NANOSECOND_PER_USEC; - tsInt64 *= NANOSECOND_PER_USEC; - break; - case TSDB_TIME_PRECISION_NANO: - break; - default: - ASSERT(0); - } - if (ts >= (double)INT64_MAX || ts < 0) { - return -1; - } + int64_t id = 0; + do { + id = atomic_add_fetch_64(&linesSmlHandleId, 1); + } while (id == 0); - return tsInt64; + return id; } -static int8_t smlGetTsTypeByLen(int32_t len) { - if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { - return TSDB_TIME_PRECISION_SECONDS; - } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { - return TSDB_TIME_PRECISION_MILLI; +static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSmlKv *kv, bool isTag, + ESchemaAction *action, SSmlHandle *info) { + uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL; + if (index) { + if (colField[*index].type != kv->type) { + uError("SML:0x%" PRIx64 " point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, + kv->key, colField[*index].type, kv->type); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if ((colField[*index].type == TSDB_DATA_TYPE_VARCHAR && + (colField[*index].bytes - VARSTR_HEADER_SIZE) < kv->length) || + (colField[*index].type == TSDB_DATA_TYPE_NCHAR && + ((colField[*index].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE < kv->length))) { + if (isTag) { + *action = SCHEMA_ACTION_CHANGE_TAG_SIZE; + } else { + *action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE; + } + } } else { - return -1; + if (isTag) { + *action = SCHEMA_ACTION_ADD_TAG; + } else { + *action = SCHEMA_ACTION_ADD_COLUMN; + } } + return 0; } -static int8_t smlGetTsTypeByPrecision(int8_t precision) { - switch (precision) { - case TSDB_SML_TIMESTAMP_HOURS: - return TSDB_TIME_PRECISION_HOURS; - case TSDB_SML_TIMESTAMP_MILLI_SECONDS: - return TSDB_TIME_PRECISION_MILLI; - case TSDB_SML_TIMESTAMP_NANO_SECONDS: - case TSDB_SML_TIMESTAMP_NOT_CONFIGURED: - return TSDB_TIME_PRECISION_NANO; - case TSDB_SML_TIMESTAMP_MICRO_SECONDS: - return TSDB_TIME_PRECISION_MICRO; - case TSDB_SML_TIMESTAMP_SECONDS: - return TSDB_TIME_PRECISION_SECONDS; - case TSDB_SML_TIMESTAMP_MINUTES: - return TSDB_TIME_PRECISION_MINUTES; - default: - return -1; +static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { + int32_t result = 1; + while (result <= length) { + result *= 2; } -} - -static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t len) { - if (len == 0 || (len == 1 && data[0] == '0')) { - return taosGetTimestampNs(); + if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { + result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE; + } else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + result = (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; } - int8_t tsType = smlGetTsTypeByPrecision(info->precision); - if (tsType == -1) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp precision", NULL); - return -1; + if (type == TSDB_DATA_TYPE_NCHAR) { + result = result * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; + } else if (type == TSDB_DATA_TYPE_BINARY) { + result = result + VARSTR_HEADER_SIZE; } - - int64_t ts = smlGetTimeValue(data, len, tsType); - if (ts == -1) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); - return -1; - } - return ts; -} - -static int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len) { - if (!data) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL); - return -1; - } - if (len == 1 && data[0] == '0') { - return taosGetTimestampNs(); - } - int8_t tsType = smlGetTsTypeByLen(len); - if (tsType == -1) { - smlBuildInvalidDataMsg(&info->msgBuf, - "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data); - return -1; - } - int64_t ts = smlGetTimeValue(data, len, tsType); - if (ts == -1) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); - return -1; - } - return ts; -} - -static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArray *cols) { - int64_t ts = 0; - if (info->protocol == TSDB_SML_LINE_PROTOCOL) { - // uError("SML:data:%s,len:%d", data, len); - ts = smlParseInfluxTime(info, data, len); - } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - ts = smlParseOpenTsdbTime(info, data, len); - } else { - ASSERT(0); - } - uDebug("SML:0x%" PRIx64 " smlParseTS:%" PRId64, info->id, ts); - - if (ts <= 0) { - uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); - return TSDB_CODE_INVALID_TIMESTAMP; - } - - // add ts to - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - kv->key = TS; - kv->keyLen = TS_LEN; - kv->i = ts; - kv->type = TSDB_DATA_TYPE_TIMESTAMP; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - taosArrayPush(cols, &kv); - - return TSDB_CODE_SUCCESS; + return result; } -static int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { - // binary - if (smlIsBinary(pVal->value, pVal->length)) { - pVal->type = TSDB_DATA_TYPE_BINARY; - pVal->length -= BINARY_ADD_LEN; - if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - pVal->value += (BINARY_ADD_LEN - 1); - return TSDB_CODE_SUCCESS; - } - // nchar - if (smlIsNchar(pVal->value, pVal->length)) { - pVal->type = TSDB_DATA_TYPE_NCHAR; - pVal->length -= NCHAR_ADD_LEN; - if (pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; +static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, + ESchemaAction *action, bool isTag) { + int32_t code = TSDB_CODE_SUCCESS; + for (int j = 0; j < taosArrayGetSize(cols); ++j) { + if (j == 0 && !isTag) continue; + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); + code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info); + if (code != TSDB_CODE_SUCCESS) { + return code; } - pVal->value += (NCHAR_ADD_LEN - 1); - return TSDB_CODE_SUCCESS; - } - - // bool - if (smlParseBool(pVal)) { - pVal->type = TSDB_DATA_TYPE_BOOL; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - return TSDB_CODE_SUCCESS; - } - // number - if (smlParseNumber(pVal, msg)) { - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - return TSDB_CODE_SUCCESS; } - - return TSDB_CODE_TSC_INVALID_VALUE; + return TSDB_CODE_SUCCESS; } -static int32_t smlParseInfluxString(const char *sql, const char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) { - if (!sql) return TSDB_CODE_SML_INVALID_DATA; - JUMP_SPACE(sql, sqlEnd) - if (*sql == COMMA) return TSDB_CODE_SML_INVALID_DATA; - elements->measure = sql; - - // parse measure - while (sql < sqlEnd) { - if ((sql != elements->measure) && IS_SLASH_LETTER(sql)) { - MOVE_FORWARD_ONE(sql, sqlEnd - sql); - sqlEnd--; - continue; - } - if (IS_COMMA(sql)) { - break; - } - - if (IS_SPACE(sql)) { - break; - } - sql++; - } - elements->measureLen = sql - elements->measure; - if (IS_INVALID_TABLE_LEN(elements->measureLen)) { - smlBuildInvalidDataMsg(msg, "measure is empty or too large than 192", NULL); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; +static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { + SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + int32_t i = 0; + for (; i < length; i++) { + taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); } - // parse tag - if (*sql == SPACE) { - elements->tagsLen = 0; + if (isTag) { + i = 0; } else { - if (*sql == COMMA) sql++; - elements->tags = sql; - while (sql < sqlEnd) { - if (IS_SPACE(sql)) { - break; - } - sql++; - } - elements->tagsLen = sql - elements->tags; - } - elements->measureTagsLen = sql - elements->measure; - - // parse cols - JUMP_SPACE(sql, sqlEnd) - elements->cols = sql; - bool isInQuote = false; - while (sql < sqlEnd) { - if (IS_QUOTE(sql)) { - isInQuote = !isInQuote; - } - if (!isInQuote && IS_SPACE(sql)) { - break; - } - sql++; - } - if (isInQuote) { - smlBuildInvalidDataMsg(msg, "only one quote", elements->cols); - return TSDB_CODE_SML_INVALID_DATA; - } - elements->colsLen = sql - elements->cols; - if (elements->colsLen == 0) { - smlBuildInvalidDataMsg(msg, "cols is empty", NULL); - return TSDB_CODE_SML_INVALID_DATA; + i = 1; } - - // parse timestamp - JUMP_SPACE(sql, sqlEnd) - elements->timestamp = sql; - while (sql < sqlEnd) { - if (isspace(*sql)) { - break; + for (; i < taosArrayGetSize(cols); i++) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { + taosHashCleanup(hashTmp); + return -1; } - sql++; } - elements->timestampLen = sql - elements->timestamp; - - return TSDB_CODE_SUCCESS; + taosHashCleanup(hashTmp); + return 0; } -static void smlParseTelnetElement(const char **sql, const char *sqlEnd, const char **data, int32_t *len) { - while (*sql < sqlEnd) { - if (**sql != SPACE && !(*data)) { - *data = *sql; - } else if (**sql == SPACE && *data) { - *len = *sql - *data; - break; - } - (*sql)++; +static int32_t getBytes(uint8_t type, int32_t length) { + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + return smlFindNearestPowerOf2(length, type); + } else { + return tDataTypes[type].bytes; } } -static int32_t smlParseTelnetTags(const char *data, const char *sqlEnd, SArray *cols, char *childTableName, - SHashObj *dumplicateKey, SSmlMsgBuf *msg) { - if (!cols) return TSDB_CODE_OUT_OF_MEMORY; - const char *sql = data; - size_t childTableNameLen = strlen(tsSmlChildTableName); - while (sql < sqlEnd) { - JUMP_SPACE(sql, sqlEnd) - if (*sql == '\0') break; - - const char *key = sql; - int32_t keyLen = 0; - - // parse key - while (sql < sqlEnd) { - if (*sql == SPACE) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; - } - if (*sql == EQUAL) { - keyLen = sql - key; - sql++; - break; - } - sql++; - } - - if (IS_INVALID_COL_LEN(keyLen)) { - smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { - smlBuildInvalidDataMsg(msg, "dumplicate key", key); - return TSDB_CODE_TSC_DUP_NAMES; - } - - // parse value - const char *value = sql; - int32_t valueLen = 0; - while (sql < sqlEnd) { - // parse value - if (*sql == SPACE) { - break; - } - if (*sql == EQUAL) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; - } - sql++; - } - valueLen = sql - value; - - if (valueLen == 0) { - smlBuildInvalidDataMsg(msg, "invalid value", value); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - // handle child table name - if (childTableNameLen != 0 && strncmp(key, tsSmlChildTableName, keyLen) == 0) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, value, (valueLen < TSDB_TABLE_NAME_LEN ? valueLen : TSDB_TABLE_NAME_LEN)); - continue; - } - - if (valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; +static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, + SArray *results, int32_t numOfCols, bool isTag) { + for (int j = 0; j < taosArrayGetSize(cols); ++j) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); + ESchemaAction action = SCHEMA_ACTION_NULL; + smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); + if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) { + SField field = {0}; + field.type = kv->type; + field.bytes = getBytes(kv->type, kv->length); + memcpy(field.name, kv->key, kv->keyLen); + taosArrayPush(results, &field); + } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { + uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); + uint16_t newIndex = *index; + if (isTag) newIndex -= numOfCols; + SField *field = (SField *)taosArrayGet(results, newIndex); + field->bytes = getBytes(kv->type, kv->length); } - - // add kv to SSmlKv - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - kv->key = key; - kv->keyLen = keyLen; - kv->value = value; - kv->length = valueLen; - kv->type = TSDB_DATA_TYPE_NCHAR; - - taosArrayPush(cols, &kv); } - return TSDB_CODE_SUCCESS; } -// format: =[ =] -static int32_t smlParseTelnetString(SSmlHandle *info, const char *sql, const char *sqlEnd, SSmlTableInfo *tinfo, - SArray *cols) { - if (!sql) return TSDB_CODE_SML_INVALID_DATA; +// static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData, +// int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){ +static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta, + ESchemaAction action) { + SRequestObj *pRequest = NULL; + SMCreateStbReq pReq = {0}; + int32_t code = TSDB_CODE_SUCCESS; + SCmdMsgInfo pCmdMsg = {0}; + + // put front for free + pReq.numOfColumns = taosArrayGetSize(pColumns); + pReq.pColumns = pColumns; + pReq.numOfTags = taosArrayGetSize(pTags); + pReq.pTags = pTags; - // parse metric - smlParseTelnetElement(&sql, sqlEnd, &tinfo->sTableName, &tinfo->sTableNameLen); - if (!(tinfo->sTableName) || IS_INVALID_TABLE_LEN(tinfo->sTableNameLen)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + code = buildRequest(info->taos->id, "", 0, NULL, false, &pRequest, 0); + if (code != TSDB_CODE_SUCCESS) { + goto end; } - // parse timestamp - const char *timestamp = NULL; - int32_t tLen = 0; - smlParseTelnetElement(&sql, sqlEnd, ×tamp, &tLen); - if (!timestamp || tLen == 0) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); - return TSDB_CODE_SML_INVALID_DATA; + pRequest->syncQuery = true; + if (!pRequest->pDb) { + code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + goto end; } - int32_t ret = smlParseTS(info, timestamp, tLen, cols); - if (ret != TSDB_CODE_SUCCESS) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); - return ret; + if (action == SCHEMA_ACTION_CREATE_STABLE) { + pReq.colVer = 1; + pReq.tagVer = 1; + pReq.suid = 0; + pReq.source = TD_REQ_FROM_APP; + } else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { + pReq.colVer = pTableMeta->sversion; + pReq.tagVer = pTableMeta->tversion + 1; + pReq.suid = pTableMeta->uid; + pReq.source = TD_REQ_FROM_TAOX; + } else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE) { + pReq.colVer = pTableMeta->sversion + 1; + pReq.tagVer = pTableMeta->tversion; + pReq.suid = pTableMeta->uid; + pReq.source = TD_REQ_FROM_TAOX; } - // parse value - const char *value = NULL; - int32_t valueLen = 0; - smlParseTelnetElement(&sql, sqlEnd, &value, &valueLen); - if (!value || valueLen == 0) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); - return TSDB_CODE_TSC_INVALID_VALUE; + if (pReq.numOfTags == 0) { + pReq.numOfTags = 1; + SField field = {0}; + field.type = TSDB_DATA_TYPE_NCHAR; + field.bytes = 1; + strcpy(field.name, tsSmlTagName); + taosArrayPush(pReq.pTags, &field); } - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - taosArrayPush(cols, &kv); - kv->key = VALUE; - kv->keyLen = VALUE_LEN; - kv->value = value; - kv->length = valueLen; - if ((ret = smlParseValue(kv, &info->msgBuf)) != TSDB_CODE_SUCCESS) { - return ret; + pReq.commentLen = -1; + pReq.igExists = true; + tNameExtractFullName(pName, pReq.name); + + pCmdMsg.epSet = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + pCmdMsg.msgType = TDMT_MND_CREATE_STB; + pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq); + pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); + if (NULL == pCmdMsg.pMsg) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; } + tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); + + SQuery pQuery; + memset(&pQuery, 0, sizeof(pQuery)); + pQuery.execMode = QUERY_EXEC_MODE_RPC; + pQuery.pCmdMsg = &pCmdMsg; + pQuery.msgType = pQuery.pCmdMsg->msgType; + pQuery.stableQuery = true; + + launchQueryImpl(pRequest, &pQuery, true, NULL); - // parse tags - ret = smlParseTelnetTags(sql, sqlEnd, tinfo->tags, tinfo->childTableName, info->dumplicateKey, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); - return ret; + if (pRequest->code == TSDB_CODE_SUCCESS) { + catalogRemoveTableMeta(info->pCatalog, pName); } + code = pRequest->code; + taosMemoryFree(pCmdMsg.pMsg); - return TSDB_CODE_SUCCESS; +end: + destroyRequest(pRequest); + tFreeSMCreateStbReq(&pReq); + return code; } -static int32_t smlParseCols(const char *data, int32_t len, SArray *cols, char *childTableName, bool isTag, - SHashObj *dumplicateKey, SSmlMsgBuf *msg) { - if (len == 0) { +static int32_t smlModifyDBSchemas(SSmlHandle *info) { + if (info->dataFormat && !info->needModifySchema) { return TSDB_CODE_SUCCESS; } + int32_t code = 0; + SHashObj *hashTmp = NULL; + STableMeta *pTableMeta = NULL; - size_t childTableNameLen = strlen(tsSmlChildTableName); - const char *sql = data; - while (sql < data + len) { - const char *key = sql; - int32_t keyLen = 0; - - while (sql < data + len) { - // parse key - if (IS_COMMA(sql)) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; - } - if (IS_EQUAL(sql)) { - keyLen = sql - key; - sql++; - break; - } - sql++; - } + SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; + tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); - if (IS_INVALID_COL_LEN(keyLen)) { - smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { - smlBuildInvalidDataMsg(msg, "dumplicate key", key); - return TSDB_CODE_TSC_DUP_NAMES; - } + SRequestConnInfo conn = {0}; + conn.pTrans = info->taos->pAppInfo->pTransporter; + conn.requestId = info->pRequest->requestId; + conn.requestObjRefId = info->pRequest->self; + conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - // parse value - const char *value = sql; - int32_t valueLen = 0; - bool isInQuote = false; - while (sql < data + len) { - // parse value - if (!isTag && IS_QUOTE(sql)) { - isInQuote = !isInQuote; - sql++; - continue; - } - if (!isInQuote && IS_COMMA(sql)) { - break; - } - if (!isInQuote && IS_EQUAL(sql)) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; - } - sql++; - } - valueLen = sql - value; - sql++; + NodeList *tmp = info->superTables; + while (tmp) { + SSmlSTableMeta *sTableData = (SSmlSTableMeta *)tmp->data.value; + bool needCheckMeta = false; // for multi thread - if (isInQuote) { - smlBuildInvalidDataMsg(msg, "only one quote", value); - return TSDB_CODE_SML_INVALID_DATA; - } - if (valueLen == 0) { - smlBuildInvalidDataMsg(msg, "invalid value", value); - return TSDB_CODE_SML_INVALID_DATA; - } - PROCESS_SLASH(key, keyLen) - PROCESS_SLASH(value, valueLen) + size_t superTableLen = (size_t)tmp->data.keyLen; + const void *superTable = tmp->data.key; + memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + memcpy(pName.tname, superTable, superTableLen); - // handle child table name - if (childTableName && childTableNameLen != 0 && strncmp(key, tsSmlChildTableName, keyLen) == 0) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, value, (valueLen < TSDB_TABLE_NAME_LEN ? valueLen : TSDB_TABLE_NAME_LEN)); - continue; - } + code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - // add kv to SSmlKv - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - if (cols) taosArrayPush(cols, &kv); + if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) { + SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); + SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); + smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true); + smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false); - kv->key = key; - kv->keyLen = keyLen; - kv->value = value; - kv->length = valueLen; - if (isTag) { - if (valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - kv->type = TSDB_DATA_TYPE_NCHAR; - } else { - int32_t ret = smlParseValue(kv, msg); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); + goto end; } - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, SSmlMsgBuf *msg) { - for (int i = 0; i < taosArrayGetSize(cols); ++i) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); + info->cost.numOfCreateSTables++; + taosMemoryFreeClear(pTableMeta); - int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); - if (index) { - SSmlKv **value = (SSmlKv **)taosArrayGet(metaArray, *index); - if (kv->type != (*value)->type) { - smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); - return TSDB_CODE_SML_NOT_SAME_TYPE; - } else { - if (IS_VAR_DATA_TYPE(kv->type)) { // update string len, if bigger - if (kv->length > (*value)->length) { - *value = kv; + code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); + goto end; + } + } else if (code == TSDB_CODE_SUCCESS) { + hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, + HASH_NO_LOCK); + for (uint16_t i = pTableMeta->tableInfo.numOfColumns; + i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { + taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + } + + ESchemaAction action = SCHEMA_ACTION_NULL; + code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->tags, &action, true); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + if (action != SCHEMA_ACTION_NULL) { + SArray *pColumns = + taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); + SArray *pTags = + taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); + + for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { + SField field = {0}; + field.type = pTableMeta->schema[i].type; + field.bytes = pTableMeta->schema[i].bytes; + strcpy(field.name, pTableMeta->schema[i].name); + if (i < pTableMeta->tableInfo.numOfColumns) { + taosArrayPush(pColumns, &field); + } else { + taosArrayPush(pTags, &field); } } - } - } else { - size_t tmp = taosArrayGetSize(metaArray); - ASSERT(tmp <= INT16_MAX); - int16_t size = tmp; - taosArrayPush(metaArray, &kv); - taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); - } - } + smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, + pTableMeta->tableInfo.numOfColumns, true); - return TSDB_CODE_SUCCESS; -} + code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); + goto end; + } -static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) { - for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); - taosArrayPush(metaArray, &kv); - taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); - } -} + info->cost.numOfAlterTagSTables++; + taosMemoryFreeClear(pTableMeta); + code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + } -static SSmlTableInfo *smlBuildTableInfo() { - SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); - if (!tag) { - return NULL; - } + taosHashClear(hashTmp); + for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { + taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + } + action = SCHEMA_ACTION_NULL; + code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, &action, false); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + if (action != SCHEMA_ACTION_NULL) { + SArray *pColumns = + taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); + SArray *pTags = + taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - tag->cols = taosArrayInit(16, POINTER_BYTES); - if (tag->cols == NULL) { - uError("SML:smlBuildTableInfo failed to allocate memory"); - goto cleanup; - } + for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { + SField field = {0}; + field.type = pTableMeta->schema[i].type; + field.bytes = pTableMeta->schema[i].bytes; + strcpy(field.name, pTableMeta->schema[i].name); + if (i < pTableMeta->tableInfo.numOfColumns) { + taosArrayPush(pColumns, &field); + } else { + taosArrayPush(pTags, &field); + } + } - tag->tags = taosArrayInit(16, POINTER_BYTES); - if (tag->tags == NULL) { - uError("SML:smlBuildTableInfo failed to allocate memory"); - goto cleanup; - } - return tag; + smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns, + pTableMeta->tableInfo.numOfColumns, false); -cleanup: - taosMemoryFree(tag); - return NULL; -} + code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); + goto end; + } -static void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) { - if (info->dataFormat) { - for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { - SArray *kvArray = (SArray *)taosArrayGetP(tag->cols, i); - for (int j = 0; j < taosArrayGetSize(kvArray); ++j) { - SSmlKv *p = (SSmlKv *)taosArrayGetP(kvArray, j); - taosMemoryFree(p); + info->cost.numOfAlterColSTables++; + taosMemoryFreeClear(pTableMeta); + code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); + goto end; + } } - taosArrayDestroy(kvArray); + + needCheckMeta = true; + taosHashCleanup(hashTmp); + hashTmp = NULL; + } else { + uError("SML:0x%" PRIx64 " load table meta error: %s", info->id, tstrerror(code)); + goto end; } - } else { - for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { - SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i); - void **p1 = (void **)taosHashIterate(kvHash, NULL); - while (p1) { - taosMemoryFree(*p1); - p1 = (void **)taosHashIterate(kvHash, p1); + + if (needCheckMeta) { + code = smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags, + sTableData->tags, true); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " check tag failed. super table name %s", info->id, pName.tname); + goto end; + } + code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols, false); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " check cols failed. super table name %s", info->id, pName.tname); + goto end; } - taosHashCleanup(kvHash); } - } - for (size_t i = 0; i < taosArrayGetSize(tag->tags); i++) { - SSmlKv *p = (SSmlKv *)taosArrayGetP(tag->tags, i); - taosMemoryFree(p); - } - taosArrayDestroy(tag->cols); - taosArrayDestroy(tag->tags); - taosMemoryFree(tag); -} -static int32_t smlKvTimeArrayCompare(const void *key1, const void *key2) { - SArray *s1 = *(SArray **)key1; - SArray *s2 = *(SArray **)key2; - SSmlKv *kv1 = (SSmlKv *)taosArrayGetP(s1, 0); - SSmlKv *kv2 = (SSmlKv *)taosArrayGetP(s2, 0); - ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP); - if (kv1->i < kv2->i) { - return -1; - } else if (kv1->i > kv2->i) { - return 1; - } else { - return 0; - } -} + sTableData->tableMeta = pTableMeta; -static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) { - SHashObj *s1 = *(SHashObj **)key1; - SHashObj *s2 = *(SHashObj **)key2; - SSmlKv **kv1pp = (SSmlKv **)taosHashGet(s1, TS, TS_LEN); - SSmlKv **kv2pp = (SSmlKv **)taosHashGet(s2, TS, TS_LEN); - if (!kv1pp || !kv2pp) { - uError("smlKvTimeHashCompare kv is null"); - return -1; - } - SSmlKv *kv1 = *kv1pp; - SSmlKv *kv2 = *kv2pp; - if (!kv1 || kv1->type != TSDB_DATA_TYPE_TIMESTAMP) { - uError("smlKvTimeHashCompare kv1"); - return -1; - } - if (!kv2 || kv2->type != TSDB_DATA_TYPE_TIMESTAMP) { - uError("smlKvTimeHashCompare kv2"); - return -1; - } - if (kv1->i < kv2->i) { - return -1; - } else if (kv1->i > kv2->i) { - return 1; - } else { - return 0; + tmp = tmp->next; } + return 0; + +end: + taosHashCleanup(hashTmp); + taosMemoryFreeClear(pTableMeta); + // catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); + return code; } -static int32_t smlDealCols(SSmlTableInfo *oneTable, bool dataFormat, SArray *cols) { - if (dataFormat) { - void *p = taosArraySearch(oneTable->cols, &cols, smlKvTimeArrayCompare, TD_GT); - if (p == NULL) { - taosArrayPush(oneTable->cols, &cols); - } else { - taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols); +/* +static int32_t smlCheckDupUnit(SHashObj *dumplicateKey, SArray *tags, SSmlMsgBuf *msg){ + for(int i = 0; i < taosArrayGetSize(tags); i++) { + SSmlKv *tag = taosArrayGet(tags, i); + if (smlCheckDuplicateKey(tag->key, tag->keyLen, dumplicateKey)) { + smlBuildInvalidDataMsg(msg, "dumplicate key", tag->key); + return TSDB_CODE_TSC_DUP_NAMES; } - return TSDB_CODE_SUCCESS; - } - - SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (!kvHash) { - uError("SML:smlDealCols failed to allocate memory"); - return TSDB_CODE_OUT_OF_MEMORY; - } - for (size_t i = 0; i < taosArrayGetSize(cols); i++) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); - taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); - } - - void *p = taosArraySearch(oneTable->cols, &kvHash, smlKvTimeHashCompare, TD_GT); - if (p == NULL) { - taosArrayPush(oneTable->cols, &kvHash); - } else { - taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &kvHash); } return TSDB_CODE_SUCCESS; } -static SSmlSTableMeta *smlBuildSTableMeta() { - SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); - if (!meta) { - return NULL; - } - meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (meta->tagHash == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; +static int32_t smlJudgeDupColName(SArray *cols, SArray *tags, SSmlMsgBuf *msg) { + SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + int ret = smlCheckDupUnit(dumplicateKey, cols, msg); + if(ret != TSDB_CODE_SUCCESS){ + goto end; } - - meta->colHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (meta->colHash == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; + ret = smlCheckDupUnit(dumplicateKey, tags, msg); + if(ret != TSDB_CODE_SUCCESS){ + goto end; } - meta->tags = taosArrayInit(32, POINTER_BYTES); - if (meta->tags == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; - } + end: + taosHashCleanup(dumplicateKey); + return ret; +} +*/ - meta->cols = taosArrayInit(32, POINTER_BYTES); - if (meta->cols == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; +static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) { + for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); + if (ret == 0) { + taosArrayPush(metaArray, kv); + } } - return meta; - -cleanup: - taosMemoryFree(meta); - return NULL; } static void smlDestroySTableMeta(SSmlSTableMeta *meta) { @@ -1478,835 +973,244 @@ static void smlDestroySTableMeta(SSmlSTableMeta *meta) { taosMemoryFree(meta); } -static void smlDestroyCols(SArray *cols) { - if (!cols) return; +static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg) { for (int i = 0; i < taosArrayGetSize(cols); ++i) { - void *kv = taosArrayGetP(cols, i); - taosMemoryFree(kv); - } -} - -static void smlDestroyInfo(SSmlHandle *info) { - if (!info) return; - qDestroyQuery(info->pQuery); - smlDestroyHandle(info->exec); - - // destroy info->childTables - void **p1 = (void **)taosHashIterate(info->childTables, NULL); - while (p1) { - smlDestroyTableInfo(info, (SSmlTableInfo *)(*p1)); - p1 = (void **)taosHashIterate(info->childTables, p1); - } - taosHashCleanup(info->childTables); - - // destroy info->superTables - p1 = (void **)taosHashIterate(info->superTables, NULL); - while (p1) { - smlDestroySTableMeta((SSmlSTableMeta *)(*p1)); - p1 = (void **)taosHashIterate(info->superTables, p1); - } - taosHashCleanup(info->superTables); - - // destroy info->pVgHash - taosHashCleanup(info->pVgHash); - taosHashCleanup(info->dumplicateKey); - if (!info->dataFormat) { - taosArrayDestroy(info->colsContainer); - } - destroyRequest(info->pRequest); + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - cJSON_Delete(info->root); - taosMemoryFreeClear(info); -} - -static SSmlHandle *smlBuildSmlInfo(STscObj *pTscObj, SRequestObj *request, SMLProtocolType protocol, int8_t precision) { - int32_t code = TSDB_CODE_SUCCESS; - SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle)); - if (NULL == info) { - return NULL; - } - info->id = smlGenId(); - - info->pQuery = (SQuery *)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == info->pQuery) { - uError("SML:0x%" PRIx64 " create info->pQuery error", info->id); - goto cleanup; - } - info->pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - info->pQuery->haveResultSet = false; - info->pQuery->msgType = TDMT_VND_SUBMIT; - info->pQuery->pRoot = (SNode *)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); - if (NULL == info->pQuery->pRoot) { - uError("SML:0x%" PRIx64 " create info->pQuery->pRoot error", info->id); - goto cleanup; - } - - if (pTscObj) { - info->taos = pTscObj; - code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); - goto cleanup; - } - } - - info->precision = precision; - info->protocol = protocol; - if (protocol == TSDB_SML_LINE_PROTOCOL) { - info->dataFormat = tsSmlDataFormat; - } else { - info->dataFormat = true; - } - - if (request) { - info->pRequest = request; - info->msgBuf.buf = info->pRequest->msgBuf; - info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; - info->pRequest->stmtType = info->pQuery->pRoot->type; - } - - info->exec = smlInitHandle(info->pQuery); - info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - - info->dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (!info->dataFormat) { - info->colsContainer = taosArrayInit(32, POINTER_BYTES); - if (NULL == info->colsContainer) { - uError("SML:0x%" PRIx64 " create info failed", info->id); - goto cleanup; - } - } - if (NULL == info->exec || NULL == info->childTables || NULL == info->superTables || NULL == info->pVgHash || - NULL == info->dumplicateKey) { - uError("SML:0x%" PRIx64 " create info failed", info->id); - goto cleanup; - } - - return info; -cleanup: - smlDestroyInfo(info); - return NULL; -} - -/************* TSDB_SML_JSON_PROTOCOL function start **************/ -static int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *root, SSmlTableInfo *tinfo) { - cJSON *metric = cJSON_GetObjectItem(root, "metric"); - if (!cJSON_IsString(metric)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - tinfo->sTableNameLen = strlen(metric->valuestring); - if (IS_INVALID_TABLE_LEN(tinfo->sTableNameLen)) { - uError("OTD:0x%" PRIx64 " Metric lenght is 0 or large than 192", info->id); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - - tinfo->sTableName = metric->valuestring; - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsVal) { - int32_t size = cJSON_GetArraySize(root); - if (size != OTD_JSON_SUB_FIELDS_NUM) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *value = cJSON_GetObjectItem(root, "value"); - if (!cJSON_IsNumber(value)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *type = cJSON_GetObjectItem(root, "type"); - if (!cJSON_IsString(type)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - double timeDouble = value->valuedouble; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - - if (timeDouble == 0) { - *tsVal = taosGetTimestampNs(); - return TSDB_CODE_SUCCESS; - } - - if (timeDouble < 0) { - return TSDB_CODE_INVALID_TIMESTAMP; - } - - *tsVal = timeDouble; - size_t typeLen = strlen(type->valuestring); - if (typeLen == 1 && (type->valuestring[0] == 's' || type->valuestring[0] == 'S')) { - // seconds - *tsVal = *tsVal * NANOSECOND_PER_SEC; - timeDouble = timeDouble * NANOSECOND_PER_SEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - } else if (typeLen == 2 && (type->valuestring[1] == 's' || type->valuestring[1] == 'S')) { - switch (type->valuestring[0]) { - case 'm': - case 'M': - // milliseconds - *tsVal = *tsVal * NANOSECOND_PER_MSEC; - timeDouble = timeDouble * NANOSECOND_PER_MSEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - break; - case 'u': - case 'U': - // microseconds - *tsVal = *tsVal * NANOSECOND_PER_USEC; - timeDouble = timeDouble * NANOSECOND_PER_USEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; + int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); + if (index) { + SSmlKv *value = (SSmlKv *)taosArrayGet(metaArray, *index); + if (isTag) { + if (kv->length > value->length) { + value->length = kv->length; } - break; - case 'n': - case 'N': - break; - default: - return TSDB_CODE_TSC_INVALID_JSON; - } - } else { - return TSDB_CODE_TSC_INVALID_JSON; - } - - return TSDB_CODE_SUCCESS; -} - -static uint8_t smlGetTimestampLen(int64_t num) { - uint8_t len = 0; - while ((num /= 10) != 0) { - len++; - } - len++; - return len; -} - -static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) { - // Timestamp must be the first KV to parse - int64_t tsVal = 0; - - cJSON *timestamp = cJSON_GetObjectItem(root, "timestamp"); - if (cJSON_IsNumber(timestamp)) { - // timestamp value 0 indicates current system time - double timeDouble = timestamp->valuedouble; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - - if (timeDouble < 0) { - return TSDB_CODE_INVALID_TIMESTAMP; - } - - uint8_t tsLen = smlGetTimestampLen((int64_t)timeDouble); - tsVal = (int64_t)timeDouble; - if (tsLen == TSDB_TIME_PRECISION_SEC_DIGITS) { - tsVal = tsVal * NANOSECOND_PER_SEC; - timeDouble = timeDouble * NANOSECOND_PER_SEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - } else if (tsLen == TSDB_TIME_PRECISION_MILLI_DIGITS) { - tsVal = tsVal * NANOSECOND_PER_MSEC; - timeDouble = timeDouble * NANOSECOND_PER_MSEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - } else if (timeDouble == 0) { - tsVal = taosGetTimestampNs(); - } else { - return TSDB_CODE_INVALID_TIMESTAMP; - } - } else if (cJSON_IsObject(timestamp)) { - int32_t ret = smlParseTSFromJSONObj(info, timestamp, &tsVal); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " Failed to parse timestamp from JSON Obj", info->id); - return ret; - } - } else { - return TSDB_CODE_TSC_INVALID_JSON; - } - - // add ts to - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) { - return TSDB_CODE_OUT_OF_MEMORY; - } - kv->key = TS; - kv->keyLen = TS_LEN; - kv->i = tsVal; - kv->type = TSDB_DATA_TYPE_TIMESTAMP; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - taosArrayPush(cols, &kv); - return TSDB_CODE_SUCCESS; -} - -static int32_t smlConvertJSONBool(SSmlKv *pVal, char *typeStr, cJSON *value) { - if (strcasecmp(typeStr, "bool") != 0) { - uError("OTD:invalid type(%s) for JSON Bool", typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - pVal->type = TSDB_DATA_TYPE_BOOL; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valueint; - - return TSDB_CODE_SUCCESS; -} - -static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { - // tinyint - if (strcasecmp(typeStr, "i8") == 0 || strcasecmp(typeStr, "tinyint") == 0) { - if (!IS_VALID_TINYINT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(tinyint)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_TINYINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // smallint - if (strcasecmp(typeStr, "i16") == 0 || strcasecmp(typeStr, "smallint") == 0) { - if (!IS_VALID_SMALLINT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(smallint)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_SMALLINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // int - if (strcasecmp(typeStr, "i32") == 0 || strcasecmp(typeStr, "int") == 0) { - if (!IS_VALID_INT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(int)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_INT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // bigint - if (strcasecmp(typeStr, "i64") == 0 || strcasecmp(typeStr, "bigint") == 0) { - pVal->type = TSDB_DATA_TYPE_BIGINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - if (value->valuedouble >= (double)INT64_MAX) { - pVal->i = INT64_MAX; - } else if (value->valuedouble <= (double)INT64_MIN) { - pVal->i = INT64_MIN; - } else { - pVal->i = value->valuedouble; - } - return TSDB_CODE_SUCCESS; - } - // float - if (strcasecmp(typeStr, "f32") == 0 || strcasecmp(typeStr, "float") == 0) { - if (!IS_VALID_FLOAT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(float)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_FLOAT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->f = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // double - if (strcasecmp(typeStr, "f64") == 0 || strcasecmp(typeStr, "double") == 0) { - pVal->type = TSDB_DATA_TYPE_DOUBLE; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->d = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - - // if reach here means type is unsupported - uError("OTD:invalid type(%s) for JSON Number", typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; -} - -static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) { - if (strcasecmp(typeStr, "binary") == 0) { - pVal->type = TSDB_DATA_TYPE_BINARY; - } else if (strcasecmp(typeStr, "nchar") == 0) { - pVal->type = TSDB_DATA_TYPE_NCHAR; - } else { - uError("OTD:invalid type(%s) for JSON String", typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - pVal->length = (int16_t)strlen(value->valuestring); - - if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - if (pVal->type == TSDB_DATA_TYPE_NCHAR && - pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - - pVal->value = value->valuestring; - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseValueFromJSONObj(cJSON *root, SSmlKv *kv) { - int32_t ret = TSDB_CODE_SUCCESS; - int32_t size = cJSON_GetArraySize(root); - - if (size != OTD_JSON_SUB_FIELDS_NUM) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *value = cJSON_GetObjectItem(root, "value"); - if (value == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *type = cJSON_GetObjectItem(root, "type"); - if (!cJSON_IsString(type)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - switch (value->type) { - case cJSON_True: - case cJSON_False: { - ret = smlConvertJSONBool(kv, type->valuestring, value); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - break; - } - case cJSON_Number: { - ret = smlConvertJSONNumber(kv, type->valuestring, value); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - break; - } - case cJSON_String: { - ret = smlConvertJSONString(kv, type->valuestring, value); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + continue; + } + if (kv->type != value->type) { + smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); + return TSDB_CODE_SML_NOT_SAME_TYPE; } - break; - } - default: - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { - switch (root->type) { - case cJSON_True: - case cJSON_False: { - kv->type = TSDB_DATA_TYPE_BOOL; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - kv->i = root->valueint; - break; - } - case cJSON_Number: { - kv->type = TSDB_DATA_TYPE_DOUBLE; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - kv->d = root->valuedouble; - break; - } - case cJSON_String: { - /* set default JSON type to binary/nchar according to - * user configured parameter tsDefaultJSONStrType - */ - char *tsDefaultJSONStrType = "nchar"; // todo - smlConvertJSONString(kv, tsDefaultJSONStrType, root); - break; - } - case cJSON_Object: { - int32_t ret = smlParseValueFromJSONObj(root, kv); - if (ret != TSDB_CODE_SUCCESS) { - uError("OTD:Failed to parse value from JSON Obj"); - return ret; + if (IS_VAR_DATA_TYPE(kv->type) && (kv->length > value->length)) { // update string len, if bigger + value->length = kv->length; + } + } else { + size_t tmp = taosArrayGetSize(metaArray); + if (tmp > INT16_MAX) { + uError("too many cols or tags"); + return -1; + } + int16_t size = tmp; + int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); + if (ret == 0) { + taosArrayPush(metaArray, kv); } - break; } - default: - return TSDB_CODE_TSC_INVALID_JSON; } return TSDB_CODE_SUCCESS; } -static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { - if (!cols) return TSDB_CODE_OUT_OF_MEMORY; - cJSON *metricVal = cJSON_GetObjectItem(root, "value"); - if (metricVal == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; +void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) { + for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { + SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i); + taosHashCleanup(kvHash); } - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) { - return TSDB_CODE_OUT_OF_MEMORY; + if (info->parseJsonByLib) { + SSmlLineInfo *key = (SSmlLineInfo *)(tag->key); + if (key != NULL) taosMemoryFree(key->tags); } - taosArrayPush(cols, &kv); + taosMemoryFree(tag->key); + taosArrayDestroy(tag->cols); + taosArrayDestroy(tag->tags); + taosMemoryFree(tag); +} - kv->key = VALUE; - kv->keyLen = VALUE_LEN; - int32_t ret = smlParseValueFromJSON(metricVal, kv); - if (ret != TSDB_CODE_SUCCESS) { - return ret; +void clearColValArray(SArray *pCols) { + int32_t num = taosArrayGetSize(pCols); + for (int32_t i = 0; i < num; ++i) { + SColVal *pCol = taosArrayGet(pCols, i); + if (TSDB_DATA_TYPE_NCHAR == pCol->type) { + taosMemoryFreeClear(pCol->value.pData); + } } - return TSDB_CODE_SUCCESS; } -static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableName, SHashObj *dumplicateKey, - SSmlMsgBuf *msg) { - int32_t ret = TSDB_CODE_SUCCESS; - if (!pKVs) { - return TSDB_CODE_OUT_OF_MEMORY; - } - cJSON *tags = cJSON_GetObjectItem(root, "tags"); - if (tags == NULL || tags->type != cJSON_Object) { - return TSDB_CODE_TSC_INVALID_JSON; - } +void smlDestroyInfo(SSmlHandle *info) { + if (!info) return; + qDestroyQuery(info->pQuery); - size_t childTableNameLen = strlen(tsSmlChildTableName); - int32_t tagNum = cJSON_GetArraySize(tags); - for (int32_t i = 0; i < tagNum; ++i) { - cJSON *tag = cJSON_GetArrayItem(tags, i); - if (tag == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; - } - size_t keyLen = strlen(tag->string); - if (IS_INVALID_COL_LEN(keyLen)) { - uError("OTD:Tag key length is 0 or too large than 64"); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - // check duplicate keys - if (smlCheckDuplicateKey(tag->string, keyLen, dumplicateKey)) { - return TSDB_CODE_TSC_DUP_NAMES; + // destroy info->childTables + NodeList *tmp = info->childTables; + while (tmp) { + if (tmp->data.used) { + smlDestroyTableInfo(info, (SSmlTableInfo *)tmp->data.value); } + NodeList *t = tmp->next; + taosMemoryFree(tmp); + tmp = t; + } - // handle child table name - if (childTableNameLen != 0 && strcmp(tag->string, tsSmlChildTableName) == 0) { - if (!cJSON_IsString(tag)) { - uError("OTD:ID must be JSON string"); - return TSDB_CODE_TSC_INVALID_JSON; - } - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - tstrncpy(childTableName, tag->valuestring, TSDB_TABLE_NAME_LEN); - continue; + // destroy info->superTables + tmp = info->superTables; + while (tmp) { + if (tmp->data.used) { + smlDestroySTableMeta((SSmlSTableMeta *)tmp->data.value); } + NodeList *t = tmp->next; + taosMemoryFree(tmp); + tmp = t; + } + + // destroy info->pVgHash + taosHashCleanup(info->pVgHash); - // add kv to SSmlKv - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - taosArrayPush(pKVs, &kv); + for (int i = 0; i < taosArrayGetSize(info->tagJsonArray); i++) { + cJSON *tags = (cJSON *)taosArrayGetP(info->tagJsonArray, i); + cJSON_Delete(tags); + } + taosArrayDestroy(info->tagJsonArray); - // key - kv->keyLen = keyLen; - kv->key = tag->string; + taosArrayDestroy(info->preLineTagKV); + taosArrayDestroy(info->maxTagKVs); + taosArrayDestroy(info->preLineColKV); - // value - ret = smlParseValueFromJSON(tag, kv); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + if (!info->dataFormat) { + for (int i = 0; i < info->lineNum; i++) { + taosArrayDestroy(info->lines[i].colArray); + if (info->parseJsonByLib) { + taosMemoryFree(info->lines[i].tags); + } } + taosMemoryFree(info->lines); } - return ret; + cJSON_Delete(info->root); + taosMemoryFreeClear(info); } -static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo *tinfo, SArray *cols) { - int32_t ret = TSDB_CODE_SUCCESS; - - if (!cJSON_IsObject(root)) { - uError("OTD:0x%" PRIx64 " data point needs to be JSON object", info->id); - return TSDB_CODE_TSC_INVALID_JSON; +SSmlHandle *smlBuildSmlInfo(TAOS *taos) { + int32_t code = TSDB_CODE_SUCCESS; + SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle)); + if (NULL == info) { + return NULL; } - - int32_t size = cJSON_GetArraySize(root); - // outmost json fields has to be exactly 4 - if (size != OTD_JSON_FIELDS_NUM) { - uError("OTD:0x%" PRIx64 " Invalid number of JSON fields in data point %d", info->id, size); - return TSDB_CODE_TSC_INVALID_JSON; + if (taos != NULL) { + info->taos = acquireTscObj(*(int64_t *)taos); + code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); + goto cleanup; + } } - // Parse metric - ret = smlParseMetricFromJSON(info, root, tinfo); - if (ret != TSDB_CODE_SUCCESS) { - uError("OTD:0x%" PRIx64 " Unable to parse metric from JSON payload", info->id); - return ret; - } - uDebug("OTD:0x%" PRIx64 " Parse metric from JSON payload finished", info->id); + info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + info->id = smlGenId(); + info->pQuery = smlInitHandle(); + info->dataFormat = true; - // Parse timestamp - ret = smlParseTSFromJSON(info, root, cols); - if (ret) { - uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); - return ret; - } - uDebug("OTD:0x%" PRIx64 " Parse timestamp from JSON payload finished", info->id); + info->tagJsonArray = taosArrayInit(8, POINTER_BYTES); + info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); + info->maxTagKVs = taosArrayInit(8, sizeof(SSmlKv)); + info->preLineColKV = taosArrayInit(8, sizeof(SSmlKv)); - // Parse metric value - ret = smlParseColsFromJSON(root, cols); - if (ret) { - uError("OTD:0x%" PRIx64 " Unable to parse metric value from JSON payload", info->id); - return ret; + if (NULL == info->pVgHash) { + uError("create SSmlHandle failed"); + goto cleanup; } - uDebug("OTD:0x%" PRIx64 " Parse metric value from JSON payload finished", info->id); - // Parse tags - ret = smlParseTagsFromJSON(root, tinfo->tags, tinfo->childTableName, info->dumplicateKey, &info->msgBuf); - if (ret) { - uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); - return ret; + return info; + +cleanup: + smlDestroyInfo(info); + return NULL; +} + +static int32_t smlPushCols(SArray *colsArray, SArray *cols) { + SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (!kvHash) { + uError("SML:smlDealCols failed to allocate memory"); + return TSDB_CODE_OUT_OF_MEMORY; + } + for (size_t i = 0; i < taosArrayGetSize(cols); i++) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); } - uDebug("OTD:0x%" PRIx64 " Parse tags from JSON payload finished", info->id); + taosArrayPush(colsArray, &kvHash); return TSDB_CODE_SUCCESS; } -/************* TSDB_SML_JSON_PROTOCOL function end **************/ - -static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql, const int len) { - SSmlLineInfo elements = {0}; - uDebug("SML:0x%" PRIx64 " smlParseInfluxLine raw:%d, len:%d, sql:%s", info->id, info->isRawLine, len, (info->isRawLine ? "rawdata" : sql)); - int ret = smlParseInfluxString(sql, sql + len, &elements, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseInfluxLine failed", info->id); - return ret; - } +static int32_t smlParseLineBottom(SSmlHandle *info) { + if (info->dataFormat) return TSDB_CODE_SUCCESS; - SArray *cols = NULL; - if (info->dataFormat) { // if dataFormat, cols need new memory to save data - cols = taosArrayInit(16, POINTER_BYTES); - if (cols == NULL) { - uError("SML:0x%" PRIx64 " smlParseInfluxLine failed to allocate memory", info->id); - return TSDB_CODE_OUT_OF_MEMORY; - } - } else { // if dataFormat is false, cols do not need to save data, there is another new memory to save data - cols = info->colsContainer; - } - - ret = smlParseTS(info, elements.timestamp, elements.timestampLen, cols); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseTS failed", info->id); - if (info->dataFormat) taosArrayDestroy(cols); - return ret; - } - ret = smlParseCols(elements.cols, elements.colsLen, cols, NULL, false, info->dumplicateKey, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseCols parse cloums fields failed", info->id); - smlDestroyCols(cols); - if (info->dataFormat) taosArrayDestroy(cols); - return ret; - } - - bool hasTable = true; - SSmlTableInfo *tinfo = NULL; - SSmlTableInfo **oneTable = - (SSmlTableInfo **)taosHashGet(info->childTables, elements.measure, elements.measureTagsLen); - if (!oneTable) { - tinfo = smlBuildTableInfo(); - if (!tinfo) { - smlDestroyCols(cols); - if (info->dataFormat) taosArrayDestroy(cols); - return TSDB_CODE_OUT_OF_MEMORY; + for (int32_t i = 0; i < info->lineNum; i++) { + SSmlLineInfo *elements = info->lines + i; + SSmlTableInfo *tinfo = NULL; + if (info->protocol == TSDB_SML_LINE_PROTOCOL) { + tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements->measure, elements->measureTagsLen, NULL); + } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { + tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); + } else { + tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); } - taosHashPut(info->childTables, elements.measure, elements.measureTagsLen, &tinfo, POINTER_BYTES); - oneTable = &tinfo; - hasTable = false; - } - - ret = smlDealCols(*oneTable, info->dataFormat, cols); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - if (!hasTable) { - ret = smlParseCols(elements.tags, elements.tagsLen, (*oneTable)->tags, (*oneTable)->childTableName, true, - info->dumplicateKey, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseCols parse tag fields failed", info->id); - return ret; + if (tinfo == NULL) { + uError("SML:0x%" PRIx64 "get oneTable failed, line num:%d", info->id, i); + smlBuildInvalidDataMsg(&info->msgBuf, "get oneTable failed", elements->measure); + return TSDB_CODE_SML_INVALID_DATA; } - if (taosArrayGetSize((*oneTable)->tags) > TSDB_MAX_TAGS) { + if (taosArrayGetSize(tinfo->tags) > TSDB_MAX_TAGS) { smlBuildInvalidDataMsg(&info->msgBuf, "too many tags than 128", NULL); return TSDB_CODE_PAR_INVALID_TAGS_NUM; } - if (taosArrayGetSize(cols) + taosArrayGetSize((*oneTable)->tags) > TSDB_MAX_COLUMNS) { + if (taosArrayGetSize(elements->colArray) + taosArrayGetSize(tinfo->tags) > TSDB_MAX_COLUMNS) { smlBuildInvalidDataMsg(&info->msgBuf, "too many columns than 4096", NULL); return TSDB_CODE_PAR_TOO_MANY_COLUMNS; } - (*oneTable)->sTableName = elements.measure; - (*oneTable)->sTableNameLen = elements.measureLen; - if (strlen((*oneTable)->childTableName) == 0) { - RandTableName rName = {(*oneTable)->tags, (*oneTable)->sTableName, (uint8_t)(*oneTable)->sTableNameLen, - (*oneTable)->childTableName}; - - buildChildTableName(&rName); - } - (*oneTable)->uid = info->uid++; - } - - SSmlSTableMeta **tableMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, elements.measure, elements.measureLen); - if (tableMeta) { // update meta - ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, cols, &info->msgBuf); - if (!hasTable && ret == TSDB_CODE_SUCCESS) { - ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, (*oneTable)->tags, &info->msgBuf); - } + int ret = smlPushCols(tinfo->cols, elements->colArray); if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); return ret; } - } else { - SSmlSTableMeta *meta = smlBuildSTableMeta(); - smlInsertMeta(meta->tagHash, meta->tags, (*oneTable)->tags); - smlInsertMeta(meta->colHash, meta->cols, cols); - taosHashPut(info->superTables, elements.measure, elements.measureLen, &meta, POINTER_BYTES); - } - - if (!info->dataFormat) { - taosArrayClear(info->colsContainer); - } - taosHashClear(info->dumplicateKey); - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseTelnetLine(SSmlHandle *info, void *data, const int len) { - int ret = TSDB_CODE_SUCCESS; - SSmlTableInfo *tinfo = smlBuildTableInfo(); - if (!tinfo) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - SArray *cols = taosArrayInit(16, POINTER_BYTES); - if (cols == NULL) { - uError("SML:0x%" PRIx64 " smlParseTelnetLine failed to allocate memory", info->id); - return TSDB_CODE_OUT_OF_MEMORY; - } - - if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - ret = smlParseTelnetString(info, (const char *)data, (char *)data + len, tinfo, cols); - } else if (info->protocol == TSDB_SML_JSON_PROTOCOL) { - ret = smlParseJSONString(info, (cJSON *)data, tinfo, cols); - } else { - ASSERT(0); - } - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseTelnetLine failed", info->id); - smlDestroyTableInfo(info, tinfo); - smlDestroyCols(cols); - taosArrayDestroy(cols); - return ret; - } - - if (taosArrayGetSize(tinfo->tags) <= 0 || taosArrayGetSize(tinfo->tags) > TSDB_MAX_TAGS) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalidate tags length:[1,128]", NULL); - smlDestroyTableInfo(info, tinfo); - smlDestroyCols(cols); - taosArrayDestroy(cols); - return TSDB_CODE_PAR_INVALID_TAGS_NUM; - } - taosHashClear(info->dumplicateKey); - - if (strlen(tinfo->childTableName) == 0) { - RandTableName rName = {tinfo->tags, tinfo->sTableName, (uint8_t)tinfo->sTableNameLen, tinfo->childTableName}; - buildChildTableName(&rName); - } - bool hasTable = true; - SSmlTableInfo **oneTable = - (SSmlTableInfo **)taosHashGet(info->childTables, tinfo->childTableName, strlen(tinfo->childTableName)); - if (!oneTable) { - taosHashPut(info->childTables, tinfo->childTableName, strlen(tinfo->childTableName), &tinfo, POINTER_BYTES); - oneTable = &tinfo; - tinfo->uid = info->uid++; - hasTable = false; - } else { - smlDestroyTableInfo(info, tinfo); - } + SSmlSTableMeta *tableMeta = + (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + if (tableMeta) { // update meta + ret = smlUpdateMeta(tableMeta->colHash, tableMeta->cols, elements->colArray, false, &info->msgBuf); + if (ret == TSDB_CODE_SUCCESS) { + ret = smlUpdateMeta(tableMeta->tagHash, tableMeta->tags, tinfo->tags, true, &info->msgBuf); + } + if (ret != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); + return ret; + } + } else { + // ret = smlJudgeDupColName(elements->colArray, tinfo->tags, &info->msgBuf); + // if (ret != TSDB_CODE_SUCCESS) { + // uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); + // return ret; + // } - taosArrayPush((*oneTable)->cols, &cols); - SSmlSTableMeta **tableMeta = - (SSmlSTableMeta **)taosHashGet(info->superTables, (*oneTable)->sTableName, (*oneTable)->sTableNameLen); - if (tableMeta) { // update meta - ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, cols, &info->msgBuf); - if (!hasTable && ret == TSDB_CODE_SUCCESS) { - ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, (*oneTable)->tags, &info->msgBuf); - } - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); - return ret; + SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); + smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); + smlInsertMeta(meta->colHash, meta->cols, elements->colArray); + nodeListSet(&info->superTables, elements->measure, elements->measureLen, meta, NULL); } - } else { - SSmlSTableMeta *meta = smlBuildSTableMeta(); - smlInsertMeta(meta->tagHash, meta->tags, (*oneTable)->tags); - smlInsertMeta(meta->colHash, meta->cols, cols); - taosHashPut(info->superTables, (*oneTable)->sTableName, (*oneTable)->sTableNameLen, &meta, POINTER_BYTES); } return TSDB_CODE_SUCCESS; } -static int32_t smlParseJSON(SSmlHandle *info, char *payload) { - int32_t payloadNum = 0; - int32_t ret = TSDB_CODE_SUCCESS; - - if (payload == NULL) { - uError("SML:0x%" PRIx64 " empty JSON Payload", info->id); - return TSDB_CODE_TSC_INVALID_JSON; - } - - info->root = cJSON_Parse(payload); - if (info->root == NULL) { - uError("SML:0x%" PRIx64 " parse json failed:%s", info->id, payload); - return TSDB_CODE_TSC_INVALID_JSON; - } - // multiple data points must be sent in JSON array - if (cJSON_IsObject(info->root)) { - payloadNum = 1; - } else if (cJSON_IsArray(info->root)) { - payloadNum = cJSON_GetArraySize(info->root); - } else { - uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); - ret = TSDB_CODE_TSC_INVALID_JSON; - goto end; - } - - for (int32_t i = 0; i < payloadNum; ++i) { - cJSON *dataPoint = (payloadNum == 1 && cJSON_IsObject(info->root)) ? info->root : cJSON_GetArrayItem(info->root, i); - ret = smlParseTelnetLine(info, dataPoint, -1); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); - goto end; - } - } - -end: - return ret; -} - static int32_t smlInsertData(SSmlHandle *info) { int32_t code = TSDB_CODE_SUCCESS; - SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); - while (oneTable) { - SSmlTableInfo *tableData = *oneTable; + NodeList *tmp = info->childTables; + while (tmp) { + SSmlTableInfo *tableData = (SSmlTableInfo *)tmp->data.value; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); @@ -2326,57 +1230,92 @@ static int32_t smlInsertData(SSmlHandle *info) { } taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg)); - SSmlSTableMeta **pMeta = - (SSmlSTableMeta **)taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); - ASSERT(NULL != pMeta && NULL != *pMeta); + SSmlSTableMeta *pMeta = + (SSmlSTableMeta *)nodeListGet(info->superTables, tableData->sTableName, tableData->sTableNameLen, NULL); + if (unlikely(NULL == pMeta || NULL == pMeta->tableMeta)) { + uError("SML:0x%" PRIx64 " NULL == pMeta. table name: %s", info->id, tableData->childTableName); + return TSDB_CODE_SML_INTERNAL_ERROR; + } // use tablemeta of stable to save vgid and uid of child table - (*pMeta)->tableMeta->vgId = vg.vgId; - (*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid + pMeta->tableMeta->vgId = vg.vgId; + pMeta->tableMeta->uid = tableData->uid; // one table merge data block together according uid - code = smlBindData(info->exec, tableData->tags, (*pMeta)->cols, tableData->cols, info->dataFormat, - (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, - info->ttl, info->msgBuf.buf, info->msgBuf.len); + code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, pMeta->cols, tableData->cols, pMeta->tableMeta, + tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, info->ttl, + info->msgBuf.buf, info->msgBuf.len); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBindData failed", info->id); return code; } - oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); + tmp = tmp->next; } - code = smlBuildOutput(info->exec, info->pVgHash); + code = smlBuildOutput(info->pQuery, info->pVgHash); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildOutput failed", info->id); return code; } info->cost.insertRpcTime = taosGetTimestampUs(); - // launchQueryImpl(info->pRequest, info->pQuery, false, NULL); - // info->affectedRows = taos_affected_rows(info->pRequest); - // return info->pRequest->code; - SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary; atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); - SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper)); - if (pWrapper == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pWrapper->pRequest = info->pRequest; - launchAsyncQuery(info->pRequest, info->pQuery, NULL, pWrapper); - return TSDB_CODE_SUCCESS; + launchQueryImpl(info->pRequest, info->pQuery, true, NULL); + return info->pRequest->code; } static void smlPrintStatisticInfo(SSmlHandle *info) { - uError("SML:0x%" PRIx64 - " smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \ + uError( + "SML:0x%" PRIx64 + " smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \ parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64 - "", - info->id, info->cost.code, info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables, - info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, info->cost.numOfAlterColSTables, - info->cost.schemaTime - info->cost.parseTime, - info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime, - info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime); + "", + info->id, info->cost.code, info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables, + info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, info->cost.numOfAlterColSTables, + info->cost.schemaTime - info->cost.parseTime, info->cost.insertBindTime - info->cost.schemaTime, + info->cost.insertRpcTime - info->cost.insertBindTime, info->cost.endTime - info->cost.insertRpcTime, + info->cost.endTime - info->cost.parseTime); +} + +int32_t smlClearForRerun(SSmlHandle *info) { + info->reRun = false; + // clear info->childTables + NodeList *pList = info->childTables; + while (pList) { + if (pList->data.used) { + smlDestroyTableInfo(info, (SSmlTableInfo *)pList->data.value); + pList->data.used = false; + } + pList = pList->next; + } + + // clear info->superTables + pList = info->superTables; + while (pList) { + if (pList->data.used) { + smlDestroySTableMeta((SSmlSTableMeta *)pList->data.value); + pList->data.used = false; + } + pList = pList->next; + } + + if (!info->dataFormat) { + if (unlikely(info->lines != NULL)) { + uError("SML:0x%" PRIx64 " info->lines != NULL", info->id); + return TSDB_CODE_SML_INVALID_DATA; + } + info->lines = (SSmlLineInfo *)taosMemoryCalloc(info->lineNum, sizeof(SSmlLineInfo)); + } + + memset(&info->preLine, 0, sizeof(SSmlLineInfo)); + info->currSTableMeta = NULL; + info->currTableDataCtx = NULL; + + SVnodeModifyOpStmt *stmt = (SVnodeModifyOpStmt *)(info->pQuery->pRoot); + stmt->freeHashFunc(stmt->pTableBlockHashObj); + stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + return TSDB_CODE_SUCCESS; } static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { @@ -2394,7 +1333,9 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char return code; } - for (int32_t i = 0; i < numLines; ++i) { + char *oldRaw = rawLine; + int32_t i = 0; + while (i < numLines) { char *tmp = NULL; int len = 0; if (lines) { @@ -2413,18 +1354,42 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char } } + uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, len:%d, sql:%s", info->id, info->isRawLine, len, + (info->isRawLine ? "rawdata" : tmp)); + if (info->protocol == TSDB_SML_LINE_PROTOCOL) { - code = smlParseInfluxLine(info, tmp, len); + if (info->dataFormat) { + SSmlLineInfo element = {0}; + code = smlParseInfluxString(info, tmp, tmp + len, &element); + } else { + code = smlParseInfluxString(info, tmp, tmp + len, info->lines + i); + } } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - code = smlParseTelnetLine(info, tmp, len); + if (info->dataFormat) { + SSmlLineInfo element = {0}; + code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, &element); + } else { + code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, info->lines + i); + } } else { - ASSERT(0); + code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; } if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); return code; } + if (info->reRun) { + i = 0; + rawLine = oldRaw; + code = smlClearForRerun(info); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + continue; + } + i++; } + return code; } @@ -2439,17 +1404,22 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL uError("SML:0x%" PRIx64 " smlParseLine error : %s", info->id, tstrerror(code)); return code; } + code = smlParseLineBottom(info); + if (code != 0) { + uError("SML:0x%" PRIx64 " smlParseLineBottom error : %s", info->id, tstrerror(code)); + return code; + } - info->cost.lineNum = numLines; - info->cost.numOfSTables = taosHashGetSize(info->superTables); - info->cost.numOfCTables = taosHashGetSize(info->childTables); + info->cost.lineNum = info->lineNum; + info->cost.numOfSTables = nodeListSize(info->superTables); + info->cost.numOfCTables = nodeListSize(info->childTables); info->cost.schemaTime = taosGetTimestampUs(); do { code = smlModifyDBSchemas(info); if (code == 0) break; - } while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES); + } while (retryNum++ < nodeListSize(info->superTables) * MAX_RETRY_TIMES); if (code != 0) { uError("SML:0x%" PRIx64 " smlModifyDBSchemas error : %s", info->id, tstrerror(code)); @@ -2466,92 +1436,42 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL return code; } -static int32_t isSchemalessDb(STscObj *taos, SRequestObj *request) { - // SCatalog *catalog = NULL; - // int32_t code = catalogGetHandle(((STscObj *)taos)->pAppInfo->clusterId, &catalog); - // if (code != TSDB_CODE_SUCCESS) { - // uError("SML get catalog error %d", code); - // return code; - // } - // - // SName name; - // tNameSetDbName(&name, taos->acctId, taos->db, strlen(taos->db)); - // char dbFname[TSDB_DB_FNAME_LEN] = {0}; - // tNameGetFullDbName(&name, dbFname); - // SDbCfgInfo pInfo = {0}; - // - // SRequestConnInfo conn = {0}; - // conn.pTrans = taos->pAppInfo->pTransporter; - // conn.requestId = request->requestId; - // conn.requestObjRefId = request->self; - // conn.mgmtEps = getEpSet_s(&taos->pAppInfo->mgmtEp); - // - // code = catalogGetDBCfg(catalog, &conn, dbFname, &pInfo); - // if (code != TSDB_CODE_SUCCESS) { - // return code; - // } - // taosArrayDestroy(pInfo.pRetensions); - // - // if (!pInfo.schemaless) { - // return TSDB_CODE_SML_INVALID_DB_CONF; - // } - return TSDB_CODE_SUCCESS; -} - -static void smlInsertCallback(void *param, void *res, int32_t code) { - SRequestObj *pRequest = (SRequestObj *)res; - SSmlHandle *info = (SSmlHandle *)param; - int32_t rows = taos_affected_rows(pRequest); - - uDebug("SML:0x%" PRIx64 " result. code:%d, msg:%s", info->id, pRequest->code, pRequest->msgBuf); - Params *pParam = info->params; - // lock - taosThreadSpinLock(&pParam->lock); - pParam->cnt++; - if (code != TSDB_CODE_SUCCESS) { - pParam->request->code = code; - pParam->request->body.resInfo.numOfRows += rows; - } else { - pParam->request->body.resInfo.numOfRows += info->affectedRows; +TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, char *rawLineEnd, int numLines, + int protocol, int precision, int32_t ttl, int64_t reqid) { + int32_t code = TSDB_CODE_SUCCESS; + if (NULL == taos) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return NULL; } - // unlock - taosThreadSpinUnlock(&pParam->lock); - if (pParam->cnt == pParam->total) { - tsem_post(&pParam->sem); + SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); + if (request == NULL) { + uError("SML:taos_schemaless_insert error request is null"); + return NULL; } - uDebug("SML:0x%" PRIx64 " insert finished, code: %d, rows: %d, total: %d", info->id, code, rows, info->affectedRows); - info->cost.endTime = taosGetTimestampUs(); - info->cost.code = code; - smlPrintStatisticInfo(info); - smlDestroyInfo(info); -} -TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char *rawLine, char *rawLineEnd, - int numLines, int protocol, int precision, int32_t ttl) { - int batchs = 0; - STscObj *pTscObj = request->pTscObj; + SSmlHandle *info = smlBuildSmlInfo(taos); + if (info == NULL) { + request->code = TSDB_CODE_OUT_OF_MEMORY; + uError("SML:taos_schemaless_insert error SSmlHandle is null"); + return (TAOS_RES *)request; + } + info->pRequest = request; + info->isRawLine = rawLine != NULL; + info->ttl = ttl; + info->precision = precision; + info->protocol = (TSDB_SML_PROTOCOL_TYPE)protocol; + info->msgBuf.buf = info->pRequest->msgBuf; + info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; + info->lineNum = numLines; - pTscObj->schemalessType = 1; SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - - Params params = {0}; - params.request = request; - tsem_init(¶ms.sem, 0, 0); - taosThreadSpinInit(&(params.lock), 0); - if (request->pDb == NULL) { request->code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; smlBuildInvalidDataMsg(&msg, "Database not specified", NULL); goto end; } - if (isSchemalessDb(pTscObj, request) != TSDB_CODE_SUCCESS) { - request->code = TSDB_CODE_SML_INVALID_DB_CONF; - smlBuildInvalidDataMsg(&msg, "Cannot write data to a non schemaless database", NULL); - goto end; - } - if (protocol < TSDB_SML_LINE_PROTOCOL || protocol > TSDB_SML_JSON_PROTOCOL) { request->code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; smlBuildInvalidDataMsg(&msg, "protocol invalidate", NULL); @@ -2573,65 +1493,14 @@ TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char goto end; } - batchs = ceil(((double)numLines) / tsSmlBatchSize); - params.total = batchs; - for (int i = 0; i < batchs; ++i) { - SRequestObj *req = (SRequestObj *)createRequest(pTscObj->id, TSDB_SQL_INSERT, 0); - if (!req) { - request->code = TSDB_CODE_OUT_OF_MEMORY; - uError("SML:taos_schemaless_insert error request is null"); - goto end; - } - SSmlHandle *info = smlBuildSmlInfo(pTscObj, req, (SMLProtocolType)protocol, precision); - if (!info) { - request->code = TSDB_CODE_OUT_OF_MEMORY; - uError("SML:taos_schemaless_insert error SSmlHandle is null"); - goto end; - } - - info->isRawLine = (rawLine == NULL); - info->ttl = ttl; - - int32_t perBatch = tsSmlBatchSize; - - if (numLines > perBatch) { - numLines -= perBatch; - } else { - perBatch = numLines; - numLines = 0; - } - - info->params = ¶ms; - info->affectedRows = perBatch; - info->pRequest->body.queryFp = smlInsertCallback; - info->pRequest->body.param = info; - int32_t code = smlProcess(info, lines, rawLine, rawLineEnd, perBatch); - if (lines) { - lines += perBatch; - } - if (rawLine) { - int num = 0; - while (rawLine < rawLineEnd) { - if (*(rawLine++) == '\n') { - num++; - } - if (num == perBatch) { - break; - } - } - } - if (code != TSDB_CODE_SUCCESS) { - info->pRequest->body.queryFp(info, req, code); - } - } - tsem_wait(¶ms.sem); + code = smlProcess(info, lines, rawLine, rawLineEnd, numLines); + request->code = code; + info->cost.endTime = taosGetTimestampUs(); + info->cost.code = code; + // smlPrintStatisticInfo(info); end: - taosThreadSpinDestroy(¶ms.lock); - tsem_destroy(¶ms.sem); - // ((STscObj *)taos)->schemalessType = 0; - pTscObj->schemalessType = 1; - uDebug("resultend:%s", request->msgBuf); + smlDestroyInfo(info); return (TAOS_RES *)request; } @@ -2656,59 +1525,26 @@ end: TAOS_RES *taos_schemaless_insert_ttl_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int32_t ttl, int64_t reqid) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); - if (!request) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } - - if (!lines) { - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&msg, "lines is null", NULL); - return (TAOS_RES *)request; - } - - return taos_schemaless_insert_inner(request, lines, NULL, NULL, numLines, protocol, precision, ttl); + return taos_schemaless_insert_inner(taos, lines, NULL, NULL, numLines, protocol, precision, ttl, reqid); } TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) { return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, TSDB_DEFAULT_TABLE_TTL, 0); } -TAOS_RES *taos_schemaless_insert_ttl(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int32_t ttl) { +TAOS_RES *taos_schemaless_insert_ttl(TAOS *taos, char *lines[], int numLines, int protocol, int precision, + int32_t ttl) { return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, ttl, 0); } -TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int64_t reqid) { - return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, TSDB_DEFAULT_TABLE_TTL, reqid); +TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision, + int64_t reqid) { + return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, TSDB_DEFAULT_TABLE_TTL, + reqid); } TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, - int precision, int32_t ttl, int64_t reqid) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); - if (!request) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } - - if (!lines || len <= 0) { - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&msg, "lines is null", NULL); - return (TAOS_RES *)request; - } - + int precision, int32_t ttl, int64_t reqid) { int numLines = 0; *totalRows = 0; char *tmp = lines; @@ -2721,16 +1557,21 @@ TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int tmp = lines + i + 1; } } - return taos_schemaless_insert_inner(request, NULL, lines, lines + len, numLines, protocol, precision, ttl); + return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, *totalRows, protocol, precision, ttl, reqid); } -TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int64_t reqid) { - return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, TSDB_DEFAULT_TABLE_TTL, reqid); +TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, + int precision, int64_t reqid) { + return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, + TSDB_DEFAULT_TABLE_TTL, reqid); } -TAOS_RES *taos_schemaless_insert_raw_ttl(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int32_t ttl) { +TAOS_RES *taos_schemaless_insert_raw_ttl(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, + int precision, int32_t ttl) { return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, ttl, 0); } -TAOS_RES *taos_schemaless_insert_raw(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision) { - return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, TSDB_DEFAULT_TABLE_TTL, 0); +TAOS_RES *taos_schemaless_insert_raw(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, + int precision) { + return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, + TSDB_DEFAULT_TABLE_TTL, 0); } diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c new file mode 100644 index 0000000000000000000000000000000000000000..e6b71c4aaa74eb8df1af8cfa56150b57fe4f9fe4 --- /dev/null +++ b/source/client/src/clientSmlJson.c @@ -0,0 +1,1220 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include "clientSml.h" + +#define OTD_JSON_SUB_FIELDS_NUM 2 + +#define JUMP_JSON_SPACE(start) \ +while(*(start)){\ + if(unlikely(*(start) > 32))\ + break;\ + else\ + (start)++;\ + } + +//SArray *smlJsonParseTags(char *start, char *end){ +// SArray *tags = taosArrayInit(4, sizeof(SSmlKv)); +// while(start < end){ +// SSmlKv kv = {0}; +// kv.type = TSDB_DATA_TYPE_NCHAR; +// bool isInQuote = false; +// while(start < end){ +// if(unlikely(!isInQuote && *start == '"')){ +// start++; +// kv.key = start; +// isInQuote = true; +// continue; +// } +// if(unlikely(isInQuote && *start == '"')){ +// kv.keyLen = start - kv.key; +// start++; +// break; +// } +// start++; +// } +// bool hasColon = false; +// while(start < end){ +// if(unlikely(!hasColon && *start == ':')){ +// start++; +// hasColon = true; +// continue; +// } +// if(unlikely(hasColon && kv.value == NULL && (*start > 32 && *start != '"'))){ +// kv.value = start; +// start++; +// continue; +// } +// +// if(unlikely(hasColon && kv.value != NULL && (*start == '"' || *start == ',' || *start == '}'))){ +// kv.length = start - kv.value; +// taosArrayPush(tags, &kv); +// start++; +// break; +// } +// start++; +// } +// } +// return tags; +//} + +//static int32_t smlParseTagsFromJSON(SSmlHandle *info, SSmlLineInfo *elements) { +// int32_t ret = TSDB_CODE_SUCCESS; +// +// if(is_same_child_table_telnet(elements, &info->preLine) == 0){ +// return TSDB_CODE_SUCCESS; +// } +// +// bool isSameMeasure = IS_SAME_SUPER_TABLE; +// +// int cnt = 0; +// SArray *preLineKV = info->preLineTagKV; +// bool isSuperKVInit = true; +// SArray *superKV = NULL; +// if(info->dataFormat){ +// if(unlikely(!isSameMeasure)){ +// SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); +// +// if(unlikely(sMeta == NULL)){ +// sMeta = smlBuildSTableMeta(info->dataFormat); +// STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); +// sMeta->tableMeta = pTableMeta; +// if(pTableMeta == NULL){ +// info->dataFormat = false; +// info->reRun = true; +// return TSDB_CODE_SUCCESS; +// } +// nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL); +// } +// info->currSTableMeta = sMeta->tableMeta; +// superKV = sMeta->tags; +// +// if(unlikely(taosArrayGetSize(superKV) == 0)){ +// isSuperKVInit = false; +// } +// taosArraySetSize(preLineKV, 0); +// } +// }else{ +// taosArraySetSize(preLineKV, 0); +// } +// +// SArray *tags = smlJsonParseTags(elements->tags, elements->tags + elements->tagsLen); +// int32_t tagNum = taosArrayGetSize(tags); +// if (tagNum == 0) { +// uError("SML:tag is empty:%s", elements->tags) +// taosArrayDestroy(tags); +// return TSDB_CODE_SML_INVALID_DATA; +// } +// for (int32_t i = 0; i < tagNum; ++i) { +// SSmlKv kv = *(SSmlKv*)taosArrayGet(tags, i); +// +// if(info->dataFormat){ +// if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ +// info->dataFormat = false; +// info->reRun = true; +// taosArrayDestroy(tags); +// return TSDB_CODE_SUCCESS; +// } +// +// if(isSameMeasure){ +// if(unlikely(cnt >= taosArrayGetSize(preLineKV))) { +// info->dataFormat = false; +// info->reRun = true; +// taosArrayDestroy(tags); +// return TSDB_CODE_SUCCESS; +// } +// SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); +// if(unlikely(kv.length > preKV->length)){ +// preKV->length = kv.length; +// SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); +// if(unlikely(NULL == tableMeta)){ +// uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); +// return TSDB_CODE_SML_INTERNAL_ERROR; +// } +// +// SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt); +// oldKV->length = kv.length; +// info->needModifySchema = true; +// } +// if(unlikely(!IS_SAME_KEY)){ +// info->dataFormat = false; +// info->reRun = true; +// taosArrayDestroy(tags); +// return TSDB_CODE_SUCCESS; +// } +// }else{ +// if(isSuperKVInit){ +// if(unlikely(cnt >= taosArrayGetSize(superKV))) { +// info->dataFormat = false; +// info->reRun = true; +// taosArrayDestroy(tags); +// return TSDB_CODE_SUCCESS; +// } +// SSmlKv *preKV = (SSmlKv *)taosArrayGet(superKV, cnt); +// if(unlikely(kv.length > preKV->length)) { +// preKV->length = kv.length; +// }else{ +// kv.length = preKV->length; +// } +// info->needModifySchema = true; +// +// if(unlikely(!IS_SAME_KEY)){ +// info->dataFormat = false; +// info->reRun = true; +// taosArrayDestroy(tags); +// return TSDB_CODE_SUCCESS; +// } +// }else{ +// taosArrayPush(superKV, &kv); +// } +// taosArrayPush(preLineKV, &kv); +// } +// }else{ +// taosArrayPush(preLineKV, &kv); +// } +// cnt++; +// } +// taosArrayDestroy(tags); +// +// SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); +// if (unlikely(tinfo == NULL)) { +// tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); +// if (unlikely(!tinfo)) { +// return TSDB_CODE_OUT_OF_MEMORY; +// } +// tinfo->tags = taosArrayDup(preLineKV, NULL); +// +// smlSetCTableName(tinfo); +// if (info->dataFormat) { +// info->currSTableMeta->uid = tinfo->uid; +// tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); +// if (tinfo->tableDataCtx == NULL) { +// smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); +// return TSDB_CODE_SML_INVALID_DATA; +// } +// } +// +// SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); +// *key = *elements; +// tinfo->key = key; +// nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet); +// } +// if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; +// +// return ret; +//} + +static char* smlJsonGetObj(char *payload){ + int leftBracketCnt = 0; + while(*payload) { + if (unlikely(*payload == '{')) { + leftBracketCnt++; + payload++; + continue; + } + if (unlikely(*payload == '}')) { + leftBracketCnt--; + payload++; + if (leftBracketCnt == 0) { + return payload; + } else if (leftBracketCnt < 0) { + return NULL; + } + continue; + } + payload++; + } + return NULL; +} + +int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset){ + int index = 0; + while(*(*start)){ + if((*start)[0] != '"'){ + (*start)++; + continue; + } + + if(unlikely(index >= OTD_JSON_FIELDS_NUM)) { + uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start) + break; + } + + char *sTmp = *start; + if((*start)[1] == 'm' && (*start)[2] == 'e' && (*start)[3] == 't' + && (*start)[4] == 'r' && (*start)[5] == 'i' && (*start)[6] == 'c' && (*start)[7] == '"'){ + + (*start) += 8; + bool isInQuote = false; + while(*(*start)){ + if(unlikely(!isInQuote && *(*start) == '"')){ + (*start)++; + offset[index++] = *start - sTmp; + element->measure = (*start); + isInQuote = true; + continue; + } + if(unlikely(isInQuote && *(*start) == '"')){ + element->measureLen = (*start) - element->measure; + break; + } + (*start)++; + } + }else if((*start)[1] == 't' && (*start)[2] == 'i' && (*start)[3] == 'm' + && (*start)[4] == 'e' && (*start)[5] == 's' && (*start)[6] == 't' + && (*start)[7] == 'a' && (*start)[8] == 'm' && (*start)[9] == 'p' && (*start)[10] == '"'){ + + (*start) += 11; + bool hasColon = false; + while(*(*start)){ + if(unlikely(!hasColon && *(*start) == ':')){ + (*start)++; + JUMP_JSON_SPACE((*start)) + offset[index++] = *start - sTmp; + element->timestamp = (*start); + hasColon = true; + continue; + } + if(unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32))){ + element->timestampLen = (*start) - element->timestamp; + break; + } + (*start)++; + } + }else if((*start)[1] == 'v' && (*start)[2] == 'a' && (*start)[3] == 'l' + && (*start)[4] == 'u' && (*start)[5] == 'e' && (*start)[6] == '"'){ + + (*start) += 7; + + bool hasColon = false; + while(*(*start)){ + if(unlikely(!hasColon && *(*start) == ':')){ + (*start)++; + JUMP_JSON_SPACE((*start)) + offset[index++] = *start - sTmp; + element->cols = (*start); + hasColon = true; + continue; + } + if(unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32))){ + element->colsLen = (*start) - element->cols; + break; + } + (*start)++; + } + }else if((*start)[1] == 't' && (*start)[2] == 'a' && (*start)[3] == 'g' + && (*start)[4] == 's' && (*start)[5] == '"'){ + (*start) += 6; + + while(*(*start)){ + if(unlikely(*(*start) == ':')){ + (*start)++; + JUMP_JSON_SPACE((*start)) + offset[index++] = *start - sTmp; + element->tags = (*start); + char* tmp = smlJsonGetObj((*start)); + if(tmp){ + element->tagsLen = tmp - (*start); + *start = tmp; + } + break; + } + (*start)++; + } + } + if(*(*start) == '\0'){ + break; + } + if(*(*start) == '}'){ + (*start)++; + break; + } + (*start)++; + } + + if(unlikely(index != OTD_JSON_FIELDS_NUM) || element->tags == NULL || element->cols == NULL || element->measure == NULL || element->timestamp == NULL) { + uError("elements != %d or element parse null", OTD_JSON_FIELDS_NUM) + return -1; + } + return 0; +} + +int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset){ + int index = 0; + while(*(*start)){ + if((*start)[0] != '"'){ + (*start)++; + continue; + } + + if(unlikely(index >= OTD_JSON_FIELDS_NUM)) { + uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start) + break; + } + + if((*start)[1] == 'm'){ + (*start) += offset[index++]; + element->measure = *start; + while(*(*start)){ + if(unlikely(*(*start) == '"')){ + element->measureLen = (*start) - element->measure; + break; + } + (*start)++; + } + }else if((*start)[1] == 't' && (*start)[2] == 'i'){ + (*start) += offset[index++]; + element->timestamp = *start; + while(*(*start)){ + if(unlikely(*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)){ + element->timestampLen = (*start) - element->timestamp; + break; + } + (*start)++; + } + }else if((*start)[1] == 'v'){ + (*start) += offset[index++]; + element->cols = *start; + while(*(*start)){ + if(unlikely( *(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)){ + element->colsLen = (*start) - element->cols; + break; + } + (*start)++; + } + }else if((*start)[1] == 't' && (*start)[2] == 'a'){ + (*start) += offset[index++]; + element->tags = (*start); + char* tmp = smlJsonGetObj((*start)); + if(tmp){ + element->tagsLen = tmp - (*start); + *start = tmp; + } + break; + } + if(*(*start) == '}'){ + (*start)++; + break; + } + (*start)++; + } + + if(unlikely(index != 0 && index != OTD_JSON_FIELDS_NUM)) { + uError("elements != %d", OTD_JSON_FIELDS_NUM) + return -1; + } + return 0; +} + +static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) { + elements->measureLen = strlen(metric->valuestring); + if (IS_INVALID_TABLE_LEN(elements->measureLen)) { + uError("OTD:0x%" PRIx64 " Metric lenght is 0 or large than 192", info->id); + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + + elements->measure = metric->valuestring; + return TSDB_CODE_SUCCESS; +} + +const char *jsonName[OTD_JSON_FIELDS_NUM] = {"metric", "timestamp", "value", "tags"}; +static int32_t smlGetJsonElements(cJSON *root, cJSON ***marks){ + for (int i = 0; i < OTD_JSON_FIELDS_NUM; ++i) { + cJSON *child = root->child; + while(child != NULL) + { + if(strcasecmp(child->string, jsonName[i]) == 0){ + *marks[i] = child; + break; + } + child = child->next; + } + if(*marks[i] == NULL){ + uError("smlGetJsonElements error, not find mark:%d:%s", i, jsonName[i]); + return -1; + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t smlConvertJSONBool(SSmlKv *pVal, char *typeStr, cJSON *value) { + if (strcasecmp(typeStr, "bool") != 0) { + uError("OTD:invalid type(%s) for JSON Bool", typeStr); + return TSDB_CODE_TSC_INVALID_JSON_TYPE; + } + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->i = value->valueint; + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { + // tinyint + if (strcasecmp(typeStr, "i8") == 0 || strcasecmp(typeStr, "tinyint") == 0) { + if (!IS_VALID_TINYINT(value->valuedouble)) { + uError("OTD:JSON value(%f) cannot fit in type(tinyint)", value->valuedouble); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_TINYINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->i = value->valuedouble; + return TSDB_CODE_SUCCESS; + } + // smallint + if (strcasecmp(typeStr, "i16") == 0 || strcasecmp(typeStr, "smallint") == 0) { + if (!IS_VALID_SMALLINT(value->valuedouble)) { + uError("OTD:JSON value(%f) cannot fit in type(smallint)", value->valuedouble); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_SMALLINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->i = value->valuedouble; + return TSDB_CODE_SUCCESS; + } + // int + if (strcasecmp(typeStr, "i32") == 0 || strcasecmp(typeStr, "int") == 0) { + if (!IS_VALID_INT(value->valuedouble)) { + uError("OTD:JSON value(%f) cannot fit in type(int)", value->valuedouble); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_INT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->i = value->valuedouble; + return TSDB_CODE_SUCCESS; + } + // bigint + if (strcasecmp(typeStr, "i64") == 0 || strcasecmp(typeStr, "bigint") == 0) { + pVal->type = TSDB_DATA_TYPE_BIGINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + if (value->valuedouble >= (double)INT64_MAX) { + pVal->i = INT64_MAX; + } else if (value->valuedouble <= (double)INT64_MIN) { + pVal->i = INT64_MIN; + } else { + pVal->i = value->valuedouble; + } + return TSDB_CODE_SUCCESS; + } + // float + if (strcasecmp(typeStr, "f32") == 0 || strcasecmp(typeStr, "float") == 0) { + if (!IS_VALID_FLOAT(value->valuedouble)) { + uError("OTD:JSON value(%f) cannot fit in type(float)", value->valuedouble); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_FLOAT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->f = value->valuedouble; + return TSDB_CODE_SUCCESS; + } + // double + if (strcasecmp(typeStr, "f64") == 0 || strcasecmp(typeStr, "double") == 0) { + pVal->type = TSDB_DATA_TYPE_DOUBLE; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->d = value->valuedouble; + return TSDB_CODE_SUCCESS; + } + + // if reach here means type is unsupported + uError("OTD:invalid type(%s) for JSON Number", typeStr); + return TSDB_CODE_TSC_INVALID_JSON_TYPE; +} + +static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) { + if (strcasecmp(typeStr, "binary") == 0) { + pVal->type = TSDB_DATA_TYPE_BINARY; + } else if (strcasecmp(typeStr, "nchar") == 0) { + pVal->type = TSDB_DATA_TYPE_NCHAR; + } else { + uError("OTD:invalid type(%s) for JSON String", typeStr); + return TSDB_CODE_TSC_INVALID_JSON_TYPE; + } + pVal->length = (int16_t)strlen(value->valuestring); + + if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + if (pVal->type == TSDB_DATA_TYPE_NCHAR && + pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + + pVal->value = value->valuestring; + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseValueFromJSONObj(cJSON *root, SSmlKv *kv) { + int32_t ret = TSDB_CODE_SUCCESS; + int32_t size = cJSON_GetArraySize(root); + + if (size != OTD_JSON_SUB_FIELDS_NUM) { + return TSDB_CODE_TSC_INVALID_JSON; + } + + cJSON *value = cJSON_GetObjectItem(root, "value"); + if (value == NULL) { + return TSDB_CODE_TSC_INVALID_JSON; + } + + cJSON *type = cJSON_GetObjectItem(root, "type"); + if (!cJSON_IsString(type)) { + return TSDB_CODE_TSC_INVALID_JSON; + } + + switch (value->type) { + case cJSON_True: + case cJSON_False: { + ret = smlConvertJSONBool(kv, type->valuestring, value); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + break; + } + case cJSON_Number: { + ret = smlConvertJSONNumber(kv, type->valuestring, value); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + break; + } + case cJSON_String: { + ret = smlConvertJSONString(kv, type->valuestring, value); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + break; + } + default: + return TSDB_CODE_TSC_INVALID_JSON_TYPE; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { + switch (root->type) { + case cJSON_True: + case cJSON_False: { + kv->type = TSDB_DATA_TYPE_BOOL; + kv->length = (int16_t)tDataTypes[kv->type].bytes; + kv->i = root->valueint; + break; + } + case cJSON_Number: { + kv->type = TSDB_DATA_TYPE_DOUBLE; + kv->length = (int16_t)tDataTypes[kv->type].bytes; + kv->d = root->valuedouble; + break; + } + case cJSON_String: { + /* set default JSON type to binary/nchar according to + * user configured parameter tsDefaultJSONStrType + */ + + char *tsDefaultJSONStrType = "nchar"; // todo + smlConvertJSONString(kv, tsDefaultJSONStrType, root); + break; + } + case cJSON_Object: { + int32_t ret = smlParseValueFromJSONObj(root, kv); + if (ret != TSDB_CODE_SUCCESS) { + uError("OTD:Failed to parse value from JSON Obj"); + return ret; + } + break; + } + default: + return TSDB_CODE_TSC_INVALID_JSON; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { + int32_t ret = TSDB_CODE_SUCCESS; + + bool isSameMeasure = IS_SAME_SUPER_TABLE; + + int cnt = 0; + SArray *preLineKV = info->preLineTagKV; + SArray *maxKVs = info->maxTagKVs; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if(info->dataFormat){ + if(unlikely(!isSameMeasure)){ + SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + + if(unlikely(sMeta == NULL)){ + STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); + if(pTableMeta == NULL){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + sMeta = smlBuildSTableMeta(info->dataFormat); + sMeta->tableMeta = pTableMeta; + nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL); + } + info->currSTableMeta = sMeta->tableMeta; + superKV = sMeta->tags; + + if(unlikely(taosArrayGetSize(superKV) == 0)){ + isSuperKVInit = false; + } + taosArraySetSize(maxKVs, 0); + } + }else{ + taosArraySetSize(maxKVs, 0); + } + taosArraySetSize(preLineKV, 0); + + int32_t tagNum = cJSON_GetArraySize(tags); + if(unlikely(tagNum == 0)){ + uError("SML:Tag should not be empty"); + return TSDB_CODE_TSC_INVALID_JSON; + } + for (int32_t i = 0; i < tagNum; ++i) { + cJSON *tag = cJSON_GetArrayItem(tags, i); + if (unlikely(tag == NULL)) { + return TSDB_CODE_TSC_INVALID_JSON; + } +// if(unlikely(tag == cMeasure)) continue; + size_t keyLen = strlen(tag->string); + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + uError("OTD:Tag key length is 0 or too large than 64"); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } + + // add kv to SSmlKv + SSmlKv kv ={.key = tag->string, .keyLen = keyLen}; + // value + ret = smlParseValueFromJSON(tag, &kv); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + return ret; + } + + if(info->dataFormat){ + if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(isSameMeasure){ + if(unlikely(cnt >= taosArrayGetSize(maxKVs))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(maxKVs, cnt); + if(unlikely(kv.length > maxKV->length)){ + maxKV->length = kv.length; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + if(unlikely(NULL == tableMeta)){ + uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); + return TSDB_CODE_SML_INTERNAL_ERROR; + } + + SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + if(isSuperKVInit){ + if(unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(superKV, cnt); + if(unlikely(kv.length > maxKV->length)) { + maxKV->length = kv.length; + }else{ + kv.length = maxKV->length; + } + info->needModifySchema = true; + + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + taosArrayPush(superKV, &kv); + } + taosArrayPush(maxKVs, &kv); + } + }else{ + taosArrayPush(maxKVs, &kv); + } + taosArrayPush(preLineKV, &kv); + cnt++; + } + + SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); + if (unlikely(tinfo == NULL)) { + tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); + if (unlikely(!tinfo)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tinfo->tags = taosArrayDup(preLineKV, NULL); + + smlSetCTableName(tinfo); + tinfo->uid = info->uid++; + if (info->dataFormat) { + info->currSTableMeta->uid = tinfo->uid; + tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); + if (tinfo->tableDataCtx == NULL) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); + smlDestroyTableInfo(info, tinfo); + return TSDB_CODE_SML_INVALID_DATA; + } + } + + SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); + *key = *elements; + if(info->parseJsonByLib){ + key->tags = taosMemoryMalloc(elements->tagsLen + 1); + memcpy(key->tags, elements->tags, elements->tagsLen); + key->tags[elements->tagsLen] = 0; + } + tinfo->key = key; + nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet); + } + if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; + + return ret; +} + +static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPrecision) { + int32_t size = cJSON_GetArraySize(root); + if (unlikely(size != OTD_JSON_SUB_FIELDS_NUM)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); + return -1; + } + + cJSON *value = cJSON_GetObjectItem(root, "value"); + if (unlikely(!cJSON_IsNumber(value))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); + return -1; + } + + cJSON *type = cJSON_GetObjectItem(root, "type"); + if (unlikely(!cJSON_IsString(type))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); + return -1; + } + + double timeDouble = value->valuedouble; + if (unlikely(smlDoubleToInt64OverFlow(timeDouble))) { + smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); + return -1; + } + + if (timeDouble == 0) { + return taosGetTimestampNs()/smlFactorNS[toPrecision]; + } + + if (timeDouble < 0) { + return timeDouble; + } + + int64_t tsInt64 = timeDouble; + size_t typeLen = strlen(type->valuestring); + if (typeLen == 1 && (type->valuestring[0] == 's' || type->valuestring[0] == 'S')) { + // seconds + int8_t fromPrecision = TSDB_TIME_PRECISION_SECONDS; + if(smlFactorS[toPrecision] < INT64_MAX / tsInt64){ + return tsInt64 * smlFactorS[toPrecision]; + } + return -1; + } else if (typeLen == 2 && (type->valuestring[1] == 's' || type->valuestring[1] == 'S')) { + switch (type->valuestring[0]) { + case 'm': + case 'M': + // milliseconds + return convertTimePrecision(tsInt64, TSDB_TIME_PRECISION_MILLI, toPrecision); + break; + case 'u': + case 'U': + // microseconds + return convertTimePrecision(tsInt64, TSDB_TIME_PRECISION_MICRO, toPrecision); + break; + case 'n': + case 'N': + return convertTimePrecision(tsInt64, TSDB_TIME_PRECISION_NANO, toPrecision); + break; + default: + return -1; + } + } else { + return -1; + } +} + +uint8_t smlGetTimestampLen(int64_t num) { + uint8_t len = 0; + while ((num /= 10) != 0) { + len++; + } + len++; + return len; +} + +static int64_t smlParseTSFromJSON(SSmlHandle *info, cJSON *timestamp) { + // Timestamp must be the first KV to parse + int32_t toPrecision = info->currSTableMeta ? info->currSTableMeta->tableInfo.precision : TSDB_TIME_PRECISION_NANO; + if (cJSON_IsNumber(timestamp)) { + // timestamp value 0 indicates current system time + double timeDouble = timestamp->valuedouble; + if (unlikely(smlDoubleToInt64OverFlow(timeDouble))) { + smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); + return -1; + } + + if (unlikely(timeDouble < 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, + "timestamp is negative", NULL); + return timeDouble; + }else if (unlikely(timeDouble == 0)) { + return taosGetTimestampNs()/smlFactorNS[toPrecision]; + } + + uint8_t tsLen = smlGetTimestampLen((int64_t)timeDouble); + + int8_t fromPrecision = smlGetTsTypeByLen(tsLen); + if (unlikely(fromPrecision == -1)) { + smlBuildInvalidDataMsg(&info->msgBuf, + "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", NULL); + return -1; + } + int64_t tsInt64 = timeDouble; + if(fromPrecision == TSDB_TIME_PRECISION_SECONDS){ + if(smlFactorS[toPrecision] < INT64_MAX / tsInt64){ + return tsInt64 * smlFactorS[toPrecision]; + } + return -1; + }else{ + return convertTimePrecision(timeDouble, fromPrecision, toPrecision); + } + } else if (cJSON_IsObject(timestamp)) { + return smlParseTSFromJSONObj(info, timestamp, toPrecision); + } else { + smlBuildInvalidDataMsg(&info->msgBuf, + "invalidate json", NULL); + return -1; + } +} + +static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo *elements) { + int32_t ret = TSDB_CODE_SUCCESS; + + cJSON *metricJson = NULL; + cJSON *tsJson = NULL; + cJSON *valueJson = NULL; + cJSON *tagsJson = NULL; + + int32_t size = cJSON_GetArraySize(root); + // outmost json fields has to be exactly 4 + if (size != OTD_JSON_FIELDS_NUM) { + uError("OTD:0x%" PRIx64 " Invalid number of JSON fields in data point %d", info->id, size); + return TSDB_CODE_TSC_INVALID_JSON; + } + + cJSON **marks[OTD_JSON_FIELDS_NUM] = {&metricJson, &tsJson, &valueJson, &tagsJson}; + ret = smlGetJsonElements(root, marks); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + return ret; + } + + // Parse metric + ret = smlParseMetricFromJSON(info, metricJson, elements); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uError("OTD:0x%" PRIx64 " Unable to parse metric from JSON payload", info->id); + return ret; + } + + // Parse metric value + SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN}; + ret = smlParseValueFromJSON(valueJson, &kv); + if (unlikely(ret)) { + uError("OTD:0x%" PRIx64 " Unable to parse metric value from JSON payload", info->id); + return ret; + } + + // Parse tags + bool needFree = info->dataFormat; + elements->tags = cJSON_PrintUnformatted(tagsJson); + elements->tagsLen = strlen(elements->tags); + if(is_same_child_table_telnet(elements, &info->preLine) != 0) { + ret = smlParseTagsFromJSON(info, tagsJson, elements); + if (unlikely(ret)) { + uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); + taosMemoryFree(elements->tags); + elements->tags = NULL; + return ret; + } + } + + if(needFree){ + taosMemoryFree(elements->tags); + elements->tags = NULL; + } + + if(unlikely(info->reRun)){ + return TSDB_CODE_SUCCESS; + } + + // Parse timestamp + // notice!!! put ts back to tag to ensure get meta->precision + int64_t ts = smlParseTSFromJSON(info, tsJson); + if (unlikely(ts < 0)) { + uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); + return TSDB_CODE_INVALID_TIMESTAMP; + } + SSmlKv kvTs = { .key = TS, .keyLen = TS_LEN, .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + + if(info->dataFormat){ + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); + } + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildRow(info->currTableDataCtx); + } + clearColValArray(info->currTableDataCtx->pValues); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + return ret; + } + }else{ + if(elements->colArray == NULL){ + elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + } + taosArrayPush(elements->colArray, &kvTs); + taosArrayPush(elements->colArray, &kv); + } + info->preLine = *elements; + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { + int32_t payloadNum = 0; + int32_t ret = TSDB_CODE_SUCCESS; + + if (unlikely(payload == NULL)) { + uError("SML:0x%" PRIx64 " empty JSON Payload", info->id); + return TSDB_CODE_TSC_INVALID_JSON; + } + + info->root = cJSON_Parse(payload); + if (unlikely(info->root == NULL)) { + uError("SML:0x%" PRIx64 " parse json failed:%s", info->id, payload); + return TSDB_CODE_TSC_INVALID_JSON; + } + + // multiple data points must be sent in JSON array + if (cJSON_IsArray(info->root)) { + payloadNum = cJSON_GetArraySize(info->root); + } else if (cJSON_IsObject(info->root)) { + payloadNum = 1; + } else { + uError("SML:0x%" PRIx64 " Invalid JSON Payload 3:%s", info->id, payload); + return TSDB_CODE_TSC_INVALID_JSON; + } + + info->lineNum = payloadNum; + info->dataFormat = true; + if (unlikely(info->lines != NULL)) { + taosMemoryFree(info->lines); + info->lines = NULL; + } + ret = smlClearForRerun(info); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } + + info->parseJsonByLib = true; + cJSON *head = (payloadNum == 1 && cJSON_IsObject(info->root)) ? info->root : info->root->child; + + int cnt = 0; + cJSON *dataPoint = head; + while (dataPoint) { + if(info->dataFormat) { + SSmlLineInfo element = {0}; + ret = smlParseJSONStringExt(info, dataPoint, &element); + }else{ + ret = smlParseJSONStringExt(info, dataPoint, info->lines + cnt); + } + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uError("SML:0x%" PRIx64 " Invalid JSON Payload 2:%s", info->id, payload); + return ret; + } + + if(unlikely(info->reRun)){ + cnt = 0; + dataPoint = head; + info->lineNum = payloadNum; + ret = smlClearForRerun(info); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } + continue; + } + cnt++; + dataPoint = dataPoint->next; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *elements) { + int32_t ret = TSDB_CODE_SUCCESS; + + if(info->offset[0] == 0){ + ret = smlJsonParseObjFirst(start, elements, info->offset); + }else{ + ret = smlJsonParseObj(start, elements, info->offset); + } + + if (ret != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if(unlikely(**start == '\0' && elements->measure == NULL)) return TSDB_CODE_SUCCESS; + + if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL); + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + + SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; + if (elements->colsLen == 0 || smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { + uError("SML:cols invalidate:%s", elements->cols); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + // Parse tags + if(is_same_child_table_telnet(elements, &info->preLine) != 0){ + char tmp = *(elements->tags + elements->tagsLen); + *(elements->tags + elements->tagsLen) = 0; + cJSON* tagsJson = cJSON_Parse(elements->tags); + *(elements->tags + elements->tagsLen) = tmp; + if (unlikely(tagsJson == NULL)) { + uError("SML:0x%" PRIx64 " parse json failed:%s", info->id, elements->tags); + return TSDB_CODE_TSC_INVALID_JSON; + } + + taosArrayPush(info->tagJsonArray, &tagsJson); + ret = smlParseTagsFromJSON(info, tagsJson, elements); + if (unlikely(ret)) { + uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); + return ret; + } + } + + if(unlikely(info->reRun)){ + return TSDB_CODE_SUCCESS; + } + + // Parse timestamp + // notice!!! put ts back to tag to ensure get meta->precision + int64_t ts = smlParseOpenTsdbTime(info, elements->timestamp, elements->timestampLen); + if (unlikely(ts < 0)) { + uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); + return TSDB_CODE_INVALID_TIMESTAMP; + } + SSmlKv kvTs = { .key = TS, .keyLen = TS_LEN, .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + + if(info->dataFormat){ + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); + } + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildRow(info->currTableDataCtx); + } + clearColValArray(info->currTableDataCtx->pValues); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + return ret; + } + }else{ + if(elements->colArray == NULL){ + elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + } + taosArrayPush(elements->colArray, &kvTs); + taosArrayPush(elements->colArray, &kv); + } + info->preLine = *elements; + + return TSDB_CODE_SUCCESS; +} + +int32_t smlParseJSON(SSmlHandle *info, char *payload) { + int32_t payloadNum = 1 << 15; + int32_t ret = TSDB_CODE_SUCCESS; + + uDebug("SML:0x%" PRIx64 "json:%s", info->id, payload); + int cnt = 0; + char *dataPointStart = payload; + while (1) { + if(info->dataFormat) { + SSmlLineInfo element = {0}; + ret = smlParseJSONString(info, &dataPointStart, &element); + }else{ + if(cnt >= payloadNum){ + payloadNum = payloadNum << 1; + void* tmp = taosMemoryRealloc(info->lines, payloadNum * sizeof(SSmlLineInfo)); + if(tmp != NULL){ + info->lines = (SSmlLineInfo*)tmp; + memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo)); + } + } + ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt); + } + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uError("SML:0x%" PRIx64 " Invalid JSON Payload 1:%s", info->id, payload); + return smlParseJSONExt(info, payload); + } + + if(unlikely(info->reRun)){ + cnt = 0; + dataPointStart = payload; + info->lineNum = payloadNum; + ret = smlClearForRerun(info); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } + continue; + } + + if(*dataPointStart == '\0') break; + cnt++; + } + info->lineNum = cnt; + + return TSDB_CODE_SUCCESS; +} diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c new file mode 100644 index 0000000000000000000000000000000000000000..890245f53d5f7cbf52613ca7804da3c43e9543f8 --- /dev/null +++ b/source/client/src/clientSmlLine.c @@ -0,0 +1,664 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include + +#include "clientSml.h" + +// comma , +//#define IS_SLASH_COMMA(sql) (*(sql) == COMMA && *((sql)-1) == SLASH) +#define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH) +// space +//#define IS_SLASH_SPACE(sql) (*(sql) == SPACE && *((sql)-1) == SLASH) +#define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH) +// equal = +//#define IS_SLASH_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) == SLASH) +#define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH) +// quote " +//#define IS_SLASH_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) == SLASH) +#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH) +// SLASH +//#define IS_SLASH_SLASH(sql) (*(sql) == SLASH && *((sql)-1) == SLASH) + +#define IS_SLASH_LETTER(sql) \ + (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL || *(sql) == QUOTE || *(sql) == SLASH)) \ +// (IS_SLASH_COMMA(sql) || IS_SLASH_SPACE(sql) || IS_SLASH_EQUAL(sql) || IS_SLASH_QUOTE(sql) || IS_SLASH_SLASH(sql)) + +#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) + +#define PROCESS_SLASH(key, keyLen) \ + for (int i = 1; i < keyLen; ++i) { \ + if (IS_SLASH_LETTER(key + i)) { \ + MOVE_FORWARD_ONE(key + i, keyLen - i); \ + i--; \ + keyLen--; \ + } \ + } + +#define BINARY_ADD_LEN 2 // "binary" 2 means " " +#define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" " + +uint8_t smlPrecisionConvert[7] = {TSDB_TIME_PRECISION_NANO, TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES, + TSDB_TIME_PRECISION_SECONDS, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO, + TSDB_TIME_PRECISION_NANO}; + +static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t len) { + uint8_t toPrecision = info->currSTableMeta ? info->currSTableMeta->tableInfo.precision : TSDB_TIME_PRECISION_NANO; + + if(unlikely(len == 0 || (len == 1 && data[0] == '0'))){ + return taosGetTimestampNs()/smlFactorNS[toPrecision]; + } + + uint8_t fromPrecision = smlPrecisionConvert[info->precision]; + + int64_t ts = smlGetTimeValue(data, len, fromPrecision, toPrecision); + if (unlikely(ts == -1)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); + return -1; + } + return ts; +} + +int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { + if (pVal->value[0] == '"'){ // binary + if (pVal->length >= 2 && pVal->value[pVal->length - 1] == '"') { + pVal->type = TSDB_DATA_TYPE_BINARY; + pVal->length -= BINARY_ADD_LEN; + if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + pVal->value += (BINARY_ADD_LEN - 1); + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if(pVal->value[0] == 'l' || pVal->value[0] == 'L'){ // nchar + if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= 3){ + pVal->type = TSDB_DATA_TYPE_NCHAR; + pVal->length -= NCHAR_ADD_LEN; + if (pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + pVal->value += (NCHAR_ADD_LEN - 1); + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pVal->value[0] == 't' || pVal->value[0] == 'T'){ + if(pVal->length == 1 || (pVal->length == 4 && (pVal->value[1] == 'r' || pVal->value[1] == 'R') + && (pVal->value[2] == 'u' || pVal->value[2] == 'U') + && (pVal->value[3] == 'e' || pVal->value[3] == 'E'))){ + pVal->i = TSDB_TRUE; + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pVal->value[0] == 'f' || pVal->value[0] == 'F'){ + if(pVal->length == 1 || (pVal->length == 5 && (pVal->value[1] == 'a' || pVal->value[1] == 'A') + && (pVal->value[2] == 'l' || pVal->value[2] == 'L') + && (pVal->value[3] == 's' || pVal->value[3] == 'S') + && (pVal->value[4] == 'e' || pVal->value[4] == 'E'))){ + pVal->i = TSDB_FALSE; + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + + // number + if (smlParseNumber(pVal, msg)) { + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return TSDB_CODE_SUCCESS; + } + + return TSDB_CODE_TSC_INVALID_VALUE; +} + +static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, + SSmlLineInfo* currElement, bool isSameMeasure, bool isSameCTable){ + if(isSameCTable){ + return TSDB_CODE_SUCCESS; + } + + int cnt = 0; + SArray *preLineKV = info->preLineTagKV; + SArray *maxKVs = info->maxTagKVs; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if(info->dataFormat){ + if(unlikely(!isSameMeasure)){ + SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); + + if(unlikely(sMeta == NULL)){ + STableMeta * pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); + if(pTableMeta == NULL){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + sMeta = smlBuildSTableMeta(info->dataFormat); + sMeta->tableMeta = pTableMeta; + nodeListSet(&info->superTables, currElement->measure, currElement->measureLen, sMeta, NULL); + } + info->currSTableMeta = sMeta->tableMeta; + superKV = sMeta->tags; + + if(unlikely(taosArrayGetSize(superKV) == 0)){ + isSuperKVInit = false; + } + taosArraySetSize(maxKVs, 0); + } + }else{ + taosArraySetSize(maxKVs, 0); + } + taosArraySetSize(preLineKV, 0); + + while (*sql < sqlEnd) { + if (unlikely(IS_SPACE(*sql))) { + break; + } + + bool hasSlash = false; + // parse key + const char *key = *sql; + size_t keyLen = 0; + while (*sql < sqlEnd) { + if (unlikely(IS_COMMA(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(IS_EQUAL(*sql))) { + keyLen = *sql - key; + (*sql)++; + break; + } + if(!hasSlash){ + hasSlash = (*(*sql) == SLASH); + } + (*sql)++; + } + if(unlikely(hasSlash)) { + PROCESS_SLASH(key, keyLen) + } + + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } + + // parse value + const char *value = *sql; + size_t valueLen = 0; + hasSlash = false; + while (*sql < sqlEnd) { + // parse value + if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { + break; + }else if (unlikely(IS_EQUAL(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + + if(!hasSlash){ + hasSlash = (*(*sql) == SLASH); + } + + (*sql)++; + } + valueLen = *sql - value; + + if (unlikely(valueLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); + return TSDB_CODE_SML_INVALID_DATA; + } + + if(unlikely(hasSlash)) { + PROCESS_SLASH(value, valueLen) + } + + if (unlikely(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + + SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen}; + if(info->dataFormat){ + if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(isSameMeasure){ + if(unlikely(cnt >= taosArrayGetSize(maxKVs))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(maxKVs, cnt); + if(unlikely(kv.length > maxKV->length)){ + maxKV->length = kv.length; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); + if(unlikely(NULL == tableMeta)){ + uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); + return TSDB_CODE_SML_INTERNAL_ERROR; + } + + SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + if(isSuperKVInit){ + if(unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(superKV, cnt); + if(unlikely(kv.length > maxKV->length)) { + maxKV->length = kv.length; + }else{ + kv.length = maxKV->length; + } + info->needModifySchema = true; + + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + taosArrayPush(superKV, &kv); + } + taosArrayPush(maxKVs, &kv); + } + }else{ + taosArrayPush(maxKVs, &kv); + } + taosArrayPush(preLineKV, &kv); + + cnt++; + if(IS_SPACE(*sql)){ + break; + } + (*sql)++; + } + + void* oneTable = nodeListGet(info->childTables, currElement->measure, currElement->measureTagsLen, NULL); + if ((oneTable != NULL)) { + return TSDB_CODE_SUCCESS; + } + + SSmlTableInfo *tinfo = smlBuildTableInfo(1, currElement->measure, currElement->measureLen); + if (unlikely(!tinfo)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tinfo->tags = taosArrayDup(preLineKV, NULL); + + smlSetCTableName(tinfo); + tinfo->uid = info->uid++; + if(info->dataFormat) { + info->currSTableMeta->uid = tinfo->uid; + tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); + if(tinfo->tableDataCtx == NULL){ + smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } + } + + nodeListSet(&info->childTables, currElement->measure, currElement->measureTagsLen, tinfo, NULL); + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, + SSmlLineInfo* currElement, bool isSameMeasure, bool isSameCTable){ + int cnt = 0; + SArray *preLineKV = info->preLineColKV; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if(info->dataFormat){ + if(unlikely(!isSameCTable)){ + SSmlTableInfo *oneTable = (SSmlTableInfo *)nodeListGet(info->childTables, currElement->measure, currElement->measureTagsLen, NULL); + if (unlikely(oneTable == NULL)) { + smlBuildInvalidDataMsg(&info->msgBuf, "child table should inside", currElement->measure); + return TSDB_CODE_SML_INVALID_DATA; + } + info->currTableDataCtx = oneTable->tableDataCtx; + } + + if(unlikely(!isSameMeasure)){ + SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); + + if(unlikely(sMeta == NULL)){ + STableMeta * pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); + if(pTableMeta == NULL){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + sMeta = smlBuildSTableMeta(info->dataFormat); + sMeta->tableMeta = pTableMeta; + nodeListSet(&info->superTables, currElement->measure, currElement->measureLen, sMeta, NULL); + } + info->currSTableMeta = sMeta->tableMeta; + superKV = sMeta->cols; + if(unlikely(taosArrayGetSize(superKV) == 0)){ + isSuperKVInit = false; + } + taosArraySetSize(preLineKV, 0); + } + } + + while (*sql < sqlEnd) { + if (unlikely(IS_SPACE(*sql))) { + break; + } + + bool hasSlash = false; + // parse key + const char *key = *sql; + size_t keyLen = 0; + while (*sql < sqlEnd) { + if (unlikely(IS_COMMA(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(IS_EQUAL(*sql))) { + keyLen = *sql - key; + (*sql)++; + break; + } + if(!hasSlash){ + hasSlash = (*(*sql) == SLASH); + } + (*sql)++; + } + if(unlikely(hasSlash)) { + PROCESS_SLASH(key, keyLen) + } + + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } + + // parse value + const char *value = *sql; + size_t valueLen = 0; + hasSlash = false; + bool isInQuote = false; + while (*sql < sqlEnd) { + // parse value + if (unlikely(IS_QUOTE(*sql))) { + isInQuote = !isInQuote; + (*sql)++; + continue; + } + if (!isInQuote){ + if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { + break; + } else if (unlikely(IS_EQUAL(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + } + if(!hasSlash){ + hasSlash = (*(*sql) == SLASH); + } + + (*sql)++; + } + valueLen = *sql - value; + + if (unlikely(isInQuote)) { + smlBuildInvalidDataMsg(&info->msgBuf, "only one quote", value); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(valueLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); + return TSDB_CODE_SML_INVALID_DATA; + } + if(unlikely(hasSlash)) { + PROCESS_SLASH(value, valueLen) + } + + SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen}; + int32_t ret = smlParseValue(&kv, &info->msgBuf); + if (ret != TSDB_CODE_SUCCESS) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlParseValue error", value); + return ret; + } + + if(info->dataFormat){ + //cnt begin 0, add ts so + 2 + if(unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + // bind data + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, cnt + 1); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uError("smlBuildCol error, retry"); + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(isSameMeasure){ + if(cnt >= taosArrayGetSize(preLineKV)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); + if(kv.type != maxKV->type){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > maxKV->length)){ + maxKV->length = kv.length; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); + if(unlikely(NULL == tableMeta)){ + uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); + return TSDB_CODE_SML_INTERNAL_ERROR; + } + + SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->cols, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + if(isSuperKVInit){ + if(unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(superKV, cnt); + if(unlikely(kv.type != maxKV->type)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(IS_VAR_DATA_TYPE(kv.type)){ + if(kv.length > maxKV->length) { + maxKV->length = kv.length; + }else{ + kv.length = maxKV->length; + } + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + taosArrayPush(superKV, &kv); + } + taosArrayPush(preLineKV, &kv); + } + }else{ + if(currElement->colArray == NULL){ + currElement->colArray = taosArrayInit(16, sizeof(SSmlKv)); + taosArraySetSize(currElement->colArray, 1); + } + taosArrayPush(currElement->colArray, &kv); //reserve for timestamp + } + + cnt++; + if(IS_SPACE(*sql)){ + break; + } + (*sql)++; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements) { + if (!sql) return TSDB_CODE_SML_INVALID_DATA; + JUMP_SPACE(sql, sqlEnd) + if (unlikely(*sql == COMMA)) return TSDB_CODE_SML_INVALID_DATA; + elements->measure = sql; + + // parse measure + while (sql < sqlEnd) { + if (unlikely((sql != elements->measure) && IS_SLASH_LETTER(sql))) { + MOVE_FORWARD_ONE(sql, sqlEnd - sql); + sqlEnd--; + continue; + } + if (unlikely(IS_COMMA(sql))) { + break; + } + + if (unlikely(IS_SPACE(sql))) { + break; + } + sql++; + } + elements->measureLen = sql - elements->measure; + if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL); + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + + // to get measureTagsLen before + const char* tmp = sql; + while (tmp < sqlEnd){ + if (unlikely(IS_SPACE(tmp))) { + break; + } + tmp++; + } + elements->measureTagsLen = tmp - elements->measure; + + bool isSameCTable = false; + bool isSameMeasure = false; + if(IS_SAME_CHILD_TABLE){ + isSameCTable = true; + isSameMeasure = true; + }else if(info->dataFormat) { + isSameMeasure = IS_SAME_SUPER_TABLE; + } + // parse tag + if (*sql == COMMA) sql++; + elements->tags = sql; + + int ret = smlParseTagKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable); + if(unlikely(ret != TSDB_CODE_SUCCESS)){ + return ret; + } + if(unlikely(info->reRun)){ + return TSDB_CODE_SUCCESS; + } + + sql = elements->measure + elements->measureTagsLen; + elements->tagsLen = sql - elements->tags; + + // parse cols + JUMP_SPACE(sql, sqlEnd) + elements->cols = sql; + + ret = smlParseColKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable); + if(unlikely(ret != TSDB_CODE_SUCCESS)){ + return ret; + } + + if(unlikely(info->reRun)){ + return TSDB_CODE_SUCCESS; + } + + elements->colsLen = sql - elements->cols; + if (unlikely(elements->colsLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "cols is empty", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } + + // parse timestamp + JUMP_SPACE(sql, sqlEnd) + elements->timestamp = sql; + while (sql < sqlEnd) { + if (unlikely(isspace(*sql))) { + break; + } + sql++; + } + elements->timestampLen = sql - elements->timestamp; + + int64_t ts = smlParseInfluxTime(info, elements->timestamp, elements->timestampLen); + if (unlikely(ts <= 0)) { + uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); + return TSDB_CODE_INVALID_TIMESTAMP; + } + // add ts to + SSmlKv kv = { .key = TS, .keyLen = TS_LEN, .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + if(info->dataFormat){ + smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0); + smlBuildRow(info->currTableDataCtx); + clearColValArray(info->currTableDataCtx->pValues); + }else{ + taosArraySet(elements->colArray, 0, &kv); + } + info->preLine = *elements; + + return ret; +} + diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c new file mode 100644 index 0000000000000000000000000000000000000000..d43ab6c9f97ca7d1c4523f03ff2bb9c73556fc4b --- /dev/null +++ b/source/client/src/clientSmlTelnet.c @@ -0,0 +1,347 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include + +#include "clientSml.h" + +int32_t is_same_child_table_telnet(const void *a, const void *b){ + SSmlLineInfo *t1 = (SSmlLineInfo *)a; + SSmlLineInfo *t2 = (SSmlLineInfo *)b; +// uError("is_same_child_table_telnet len:%d,%d %s,%s @@@ len:%d,%d %s,%s", t1->measureLen, t2->measureLen, +// t1->measure, t2->measure, t1->tagsLen, t2->tagsLen, t1->tags, t2->tags); + if(t1 == NULL || t2 == NULL || t1->measure == NULL || t2->measure == NULL + || t1->tags == NULL || t2->tags == NULL) + return 1; + return (((t1->measureLen == t2->measureLen) && memcmp(t1->measure, t2->measure, t1->measureLen) == 0) + && ((t1->tagsLen == t2->tagsLen) && memcmp(t1->tags, t2->tags, t1->tagsLen) == 0)) ? 0 : 1; +} + +int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len) { + uint8_t toPrecision = info->currSTableMeta ? info->currSTableMeta->tableInfo.precision : TSDB_TIME_PRECISION_NANO; + + if (unlikely(!data)) { + smlBuildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL); + return -1; + } + if (unlikely(len == 1 && data[0] == '0')) { + return taosGetTimestampNs()/smlFactorNS[toPrecision]; + } + int8_t fromPrecision = smlGetTsTypeByLen(len); + if (unlikely(fromPrecision == -1)) { + smlBuildInvalidDataMsg(&info->msgBuf, + "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data); + return -1; + } + int64_t ts = smlGetTimeValue(data, len, fromPrecision, toPrecision); + if (unlikely(ts == -1)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); + return -1; + } + return ts; +} + + +static void smlParseTelnetElement(char **sql, char *sqlEnd, char **data, int32_t *len) { + while (*sql < sqlEnd) { + if (unlikely((**sql != SPACE && !(*data)))) { + *data = *sql; + } else if (unlikely(**sql == SPACE && *data)) { + *len = *sql - *data; + break; + } + (*sql)++; + } +} + +static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) { + if(is_same_child_table_telnet(elements, &info->preLine) == 0){ + return TSDB_CODE_SUCCESS; + } + + bool isSameMeasure = IS_SAME_SUPER_TABLE; + + int cnt = 0; + SArray *preLineKV = info->preLineTagKV; + SArray *maxKVs = info->maxTagKVs; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if(info->dataFormat){ + if(!isSameMeasure){ + SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + + if(unlikely(sMeta == NULL)){ + STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); + if(pTableMeta == NULL){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + sMeta = smlBuildSTableMeta(info->dataFormat); + sMeta->tableMeta = pTableMeta; + nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL); + } + info->currSTableMeta = sMeta->tableMeta; + superKV = sMeta->tags; + + if(unlikely(taosArrayGetSize(superKV) == 0)){ + isSuperKVInit = false; + } + taosArraySetSize(maxKVs, 0); + } + }else{ + taosArraySetSize(maxKVs, 0); + } + + taosArraySetSize(preLineKV, 0); + const char *sql = data; + while (sql < sqlEnd) { + JUMP_SPACE(sql, sqlEnd) + if (unlikely(*sql == '\0')) break; + + const char *key = sql; + size_t keyLen = 0; + + // parse key + while (sql < sqlEnd) { + if (unlikely(*sql == SPACE)) { + smlBuildInvalidDataMsg(msg, "invalid data", sql); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(*sql == EQUAL)) { + keyLen = sql - key; + sql++; + break; + } + sql++; + } + + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } +// if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { +// smlBuildInvalidDataMsg(msg, "dumplicate key", key); +// return TSDB_CODE_TSC_DUP_NAMES; +// } + + // parse value + const char *value = sql; + size_t valueLen = 0; + while (sql < sqlEnd) { + // parse value + if (unlikely(*sql == SPACE)) { + break; + } + if (unlikely(*sql == EQUAL)) { + smlBuildInvalidDataMsg(msg, "invalid data", sql); + return TSDB_CODE_SML_INVALID_DATA; + } + sql++; + } + valueLen = sql - value; + + if (unlikely(valueLen == 0)) { + smlBuildInvalidDataMsg(msg, "invalid value", value); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (unlikely(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + + SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen}; + + if(info->dataFormat){ + if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(isSameMeasure){ + if(unlikely(cnt >= taosArrayGetSize(maxKVs))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(maxKVs, cnt); + if(unlikely(kv.length > maxKV->length)){ + maxKV->length = kv.length; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + if(unlikely(NULL == tableMeta)){ + uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); + return TSDB_CODE_SML_INTERNAL_ERROR; + } + + SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + if(isSuperKVInit){ + if(unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(superKV, cnt); + if(unlikely(kv.length > maxKV->length)) { + maxKV->length = kv.length; + }else{ + kv.length = maxKV->length; + } + info->needModifySchema = true; + + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + taosArrayPush(superKV, &kv); + } + taosArrayPush(maxKVs, &kv); + } + }else{ + taosArrayPush(maxKVs, &kv); + } + taosArrayPush(preLineKV, &kv); + cnt++; + } + SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); + if (unlikely(tinfo == NULL)) { + tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); + if (!tinfo) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tinfo->tags = taosArrayDup(preLineKV, NULL); + + smlSetCTableName(tinfo); + tinfo->uid = info->uid++; + if (info->dataFormat) { + info->currSTableMeta->uid = tinfo->uid; + tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); + if (tinfo->tableDataCtx == NULL) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); + smlDestroyTableInfo(info, tinfo); + return TSDB_CODE_SML_INVALID_DATA; + } + } + + SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); + *key = *elements; + tinfo->key = key; + nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet); + } + if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; + + return TSDB_CODE_SUCCESS; +} + +// format: =[ =] +int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements) { + if (!sql) return TSDB_CODE_SML_INVALID_DATA; + + // parse metric + smlParseTelnetElement(&sql, sqlEnd, &elements->measure, &elements->measureLen); + if (unlikely((!(elements->measure) || IS_INVALID_TABLE_LEN(elements->measureLen)))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + + // parse timestamp + smlParseTelnetElement(&sql, sqlEnd, &elements->timestamp, &elements->timestampLen); + if (unlikely(!elements->timestamp || elements->timestampLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); + return TSDB_CODE_SML_INVALID_DATA; + } + + bool needConverTime = false; // get TS before parse tag(get meta), so need conver time + if(info->dataFormat && info->currSTableMeta == NULL){ + needConverTime = true; + } + int64_t ts = smlParseOpenTsdbTime(info, elements->timestamp, elements->timestampLen); + if (unlikely(ts < 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); + return TSDB_CODE_INVALID_TIMESTAMP; + } + SSmlKv kvTs = { .key = TS, .keyLen = TS_LEN, .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + + // parse value + smlParseTelnetElement(&sql, sqlEnd, &elements->cols, &elements->colsLen); + if (unlikely(!elements->cols || elements->colsLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; + if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_VALUE; + } + + JUMP_SPACE(sql, sqlEnd) + + elements->tags = sql; + elements->tagsLen = sqlEnd - sql; + if (unlikely(!elements->tags || elements->tagsLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + int ret = smlParseTelnetTags(info, sql, sqlEnd, elements, &info->msgBuf); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + return ret; + } + + if(unlikely(info->reRun)){ + return TSDB_CODE_SUCCESS; + } + + if(info->dataFormat){ + if(needConverTime) { + kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision); + } + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); + } + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildRow(info->currTableDataCtx); + } + clearColValArray(info->currTableDataCtx->pValues); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + return ret; + } + }else{ + if(elements->colArray == NULL){ + elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + } + taosArrayPush(elements->colArray, &kvTs); + taosArrayPush(elements->colArray, &kv); + } + info->preLine = *elements; + + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 82ea9e0d8f8f6e70829567ce34e9b9d1ae444da4..f5b65371a7c90393b72ee059afb8060f1b373891 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -152,9 +152,10 @@ int32_t stmtRestoreQueryFields(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } -int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, SName* tbName, const char* sTableName, bool autoCreateTbl) { +int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, SName* tbName, const char* sTableName, + bool autoCreateTbl) { STscStmt* pStmt = (STscStmt*)stmt; - char tbFName[TSDB_TABLE_FNAME_LEN]; + char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(tbName, tbFName); memcpy(&pStmt->bInfo.sname, tbName, sizeof(*tbName)); @@ -171,12 +172,11 @@ int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, return TSDB_CODE_SUCCESS; } -int32_t stmtUpdateExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockHash, bool autoCreateTbl) { +int32_t stmtUpdateExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockHash) { STscStmt* pStmt = (STscStmt*)stmt; pStmt->sql.pVgHash = pVgHash; pStmt->exec.pBlockHash = pBlockHash; - pStmt->exec.autoCreateTbl = autoCreateTbl; return TSDB_CODE_SUCCESS; } @@ -186,7 +186,7 @@ int32_t stmtUpdateInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, SNam STscStmt* pStmt = (STscStmt*)stmt; STMT_ERR_RET(stmtUpdateBindInfo(stmt, pTableMeta, tags, tbName, sTableName, autoCreateTbl)); - STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash, autoCreateTbl)); + STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash)); pStmt->sql.autoCreateTbl = autoCreateTbl; @@ -214,16 +214,16 @@ int32_t stmtCacheBlock(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } - STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + STableDataCxt** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (!pSrc) { return TSDB_CODE_OUT_OF_MEMORY; } - STableDataBlocks* pDst = NULL; + STableDataCxt* pDst = NULL; - STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc)); + STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc, true)); SStmtTableCache cache = { - .pDataBlock = pDst, + .pDataCtx = pDst, .boundTags = pStmt->bInfo.boundTags, }; @@ -241,6 +241,8 @@ int32_t stmtCacheBlock(STscStmt* pStmt) { } int32_t stmtParseSql(STscStmt* pStmt) { + pStmt->exec.pCurrBlock = NULL; + SStmtCallback stmtCb = { .pStmt = pStmt, .getTbNameFn = stmtGetTbName, @@ -273,7 +275,7 @@ int32_t stmtCleanBindInfo(STscStmt* pStmt) { pStmt->bInfo.tbName[0] = 0; pStmt->bInfo.tbFName[0] = 0; if (!pStmt->bInfo.tagsCached) { - destroyBoundColumnInfo(pStmt->bInfo.boundTags); + qDestroyBoundColInfo(pStmt->bInfo.boundTags); taosMemoryFreeClear(pStmt->bInfo.boundTags); } memset(pStmt->bInfo.stbFName, 0, TSDB_TABLE_FNAME_LEN); @@ -289,29 +291,24 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) { size_t keyLen = 0; void* pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); while (pIter) { - STableDataBlocks* pBlocks = *(STableDataBlocks**)pIter; - char* key = taosHashGetKey(pIter, &keyLen); - STableMeta* pMeta = qGetTableMetaInDataBlock(pBlocks); + STableDataCxt* pBlocks = *(STableDataCxt**)pIter; + char* key = taosHashGetKey(pIter, &keyLen); + STableMeta* pMeta = qGetTableMetaInDataBlock(pBlocks); - if (keepTable && (strlen(pStmt->bInfo.tbFName) == keyLen) && strncmp(pStmt->bInfo.tbFName, key, keyLen) == 0) { - STMT_ERR_RET(qResetStmtDataBlock(pBlocks, true)); + if (keepTable && pBlocks == pStmt->exec.pCurrBlock) { + TSWAP(pBlocks->pData, pStmt->exec.pCurrTbData); + STMT_ERR_RET(qResetStmtDataBlock(pBlocks, false)); pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); continue; } - if (STMT_TYPE_MULTI_INSERT == pStmt->sql.type) { - qFreeStmtDataBlock(pBlocks); - } else { - qDestroyStmtDataBlock(pBlocks); - } + qDestroyStmtDataBlock(pBlocks); taosHashRemove(pStmt->exec.pBlockHash, key, keyLen); pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); } - pStmt->exec.autoCreateTbl = false; - if (keepTable) { return TSDB_CODE_SUCCESS; } @@ -319,6 +316,9 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) { taosHashCleanup(pStmt->exec.pBlockHash); pStmt->exec.pBlockHash = NULL; + tDestroySSubmitTbData(pStmt->exec.pCurrTbData, TSDB_MSG_FLG_ENCODE); + taosMemoryFreeClear(pStmt->exec.pCurrTbData); + STMT_ERR_RET(stmtCleanBindInfo(pStmt)); return TSDB_CODE_SUCCESS; @@ -337,8 +337,8 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { while (pIter) { SStmtTableCache* pCache = (SStmtTableCache*)pIter; - qDestroyStmtDataBlock(pCache->pDataBlock); - destroyBoundColumnInfo(pCache->boundTags); + qDestroyStmtDataBlock(pCache->pDataCtx); + qDestroyBoundColInfo(pCache->boundTags); taosMemoryFreeClear(pCache->boundTags); pIter = taosHashIterate(pStmt->sql.pTableCache, pIter); @@ -354,7 +354,8 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } -int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataBlocks* pDataBlock, STableDataBlocks** newBlock, uint64_t uid) { +int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataCxt* pDataBlock, STableDataCxt** newBlock, uint64_t uid, + uint64_t suid) { SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); SVgroupInfo vgInfo = {0}; SRequestConnInfo conn = {.pTrans = pStmt->taos->pAppInfo->pTransporter, @@ -366,7 +367,9 @@ int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataBlocks* pDataBlock, STab STMT_ERR_RET( taosHashPut(pStmt->sql.pVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo))); - STMT_ERR_RET(qRebuildStmtDataBlock(newBlock, pDataBlock, uid, vgInfo.vgId)); + STMT_ERR_RET(qRebuildStmtDataBlock(newBlock, pDataBlock, uid, suid, vgInfo.vgId, pStmt->sql.autoCreateTbl)); + + STMT_DLOG("tableDataCxt rebuilt, uid:%" PRId64 ", vgId:%d", uid, vgInfo.vgId); return TSDB_CODE_SUCCESS; } @@ -375,12 +378,13 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.needParse = true; pStmt->bInfo.inExecCache = false; - STableDataBlocks* pBlockInExec = - taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); - if (pBlockInExec) { + STableDataCxt** pCxtInExec = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + if (pCxtInExec) { pStmt->bInfo.needParse = false; pStmt->bInfo.inExecCache = true; + pStmt->exec.pCurrBlock = *pCxtInExec; + if (pStmt->sql.autoCreateTbl) { tscDebug("reuse stmt block for tb %s in execBlock", pStmt->bInfo.tbFName); return TSDB_CODE_SUCCESS; @@ -389,7 +393,10 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { if (NULL == pStmt->sql.pTableCache || taosHashGetSize(pStmt->sql.pTableCache) <= 0) { if (pStmt->bInfo.inExecCache) { - ASSERT(taosHashGetSize(pStmt->exec.pBlockHash) == 1); + if(ASSERT(taosHashGetSize(pStmt->exec.pBlockHash) == 1)){ + tscError("stmtGetFromCache error"); + return TSDB_CODE_TSC_STMT_CACHE_ERROR; + } pStmt->bInfo.needParse = false; tscDebug("reuse stmt block for tb %s in execBlock", pStmt->bInfo.tbFName); return TSDB_CODE_SUCCESS; @@ -407,18 +414,18 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pStmt->bInfo.tbSuid, sizeof(pStmt->bInfo.tbSuid)); if (pCache) { pStmt->bInfo.needParse = false; - pStmt->exec.autoCreateTbl = true; - pStmt->bInfo.tbUid = 0; - STableDataBlocks* pNewBlock = NULL; - STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataBlock, &pNewBlock, 0)); + STableDataCxt* pNewBlock = NULL; + STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataCtx, &pNewBlock, 0, pStmt->bInfo.tbSuid)); if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + pStmt->exec.pCurrBlock = pNewBlock; + tscDebug("reuse stmt block for tb %s in sqlBlock, suid:0x%" PRIx64, pStmt->bInfo.tbFName, pStmt->bInfo.tbSuid); return TSDB_CODE_SUCCESS; @@ -489,14 +496,16 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.boundTags = pCache->boundTags; pStmt->bInfo.tagsCached = true; - STableDataBlocks* pNewBlock = NULL; - STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataBlock, &pNewBlock, uid)); + STableDataCxt* pNewBlock = NULL; + STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataCtx, &pNewBlock, uid, suid)); if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + pStmt->exec.pCurrBlock = pNewBlock; + tscDebug("tb %s in sqlBlock list, set to current", pStmt->bInfo.tbFName); return TSDB_CODE_SUCCESS; @@ -614,8 +623,8 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { return TSDB_CODE_SUCCESS; } - STableDataBlocks** pDataBlock = - (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + STableDataCxt** pDataBlock = + (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_APP_ERROR); @@ -626,8 +635,6 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); - pStmt->exec.autoCreateTbl = true; - return TSDB_CODE_SUCCESS; } @@ -637,8 +644,8 @@ int stmtFetchTagFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); } - STableDataBlocks** pDataBlock = - (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + STableDataCxt** pDataBlock = + (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_APP_ERROR); @@ -655,8 +662,8 @@ int stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); } - STableDataBlocks** pDataBlock = - (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + STableDataCxt** pDataBlock = + (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_APP_ERROR); @@ -729,11 +736,18 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { return TSDB_CODE_SUCCESS; } - STableDataBlocks** pDataBlock = - (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); - if (NULL == pDataBlock) { - tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); - STMT_ERR_RET(TSDB_CODE_APP_ERROR); + STableDataCxt** pDataBlock = NULL; + + if (pStmt->exec.pCurrBlock) { + pDataBlock = &pStmt->exec.pCurrBlock; + } else { + pDataBlock = + (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + if (NULL == pDataBlock) { + tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); + STMT_ERR_RET(TSDB_CODE_TSC_STMT_CACHE_ERROR); + } + pStmt->exec.pCurrBlock = *pDataBlock; } if (colIdx < 0) { @@ -779,10 +793,10 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) { int32_t code = 0; int32_t finalCode = 0; size_t keyLen = 0; - STableDataBlocks** pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); + void* pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); while (pIter) { - STableDataBlocks* pBlock = *pIter; - char* key = taosHashGetKey(pIter, &keyLen); + STableDataCxt* pBlock = *(STableDataCxt**)pIter; + char* key = taosHashGetKey(pIter, &keyLen); STableMeta* pMeta = qGetTableMetaInDataBlock(pBlock); if (pMeta->uid) { @@ -848,7 +862,7 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) { pMeta->uid = pTableMeta->uid; pStmt->bInfo.tbUid = pTableMeta->uid; - taosMemoryFree(pTableMeta); + taosMemoryFree(pTableMeta); } pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); @@ -861,7 +875,6 @@ int stmtExec(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; int32_t code = 0; SSubmitRsp* pRsp = NULL; - bool autoCreateTbl = pStmt->exec.autoCreateTbl; STMT_DLOG_E("start to exec"); @@ -870,8 +883,13 @@ int stmtExec(TAOS_STMT* stmt) { if (STMT_TYPE_QUERY == pStmt->sql.type) { launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); } else { + tDestroySSubmitTbData(pStmt->exec.pCurrTbData, TSDB_MSG_FLG_ENCODE); + taosMemoryFreeClear(pStmt->exec.pCurrTbData); + + STMT_ERR_RET(qCloneCurrentTbData(pStmt->exec.pCurrBlock, &pStmt->exec.pCurrTbData)); + STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, pStmt->exec.pBlockHash)); - launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, (autoCreateTbl ? (void**)&pRsp : NULL)); + launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); } if (pStmt->exec.pRequest->code && NEED_CLIENT_HANDLE_ERROR(pStmt->exec.pRequest->code)) { @@ -894,15 +912,6 @@ _return: stmtCleanExecInfo(pStmt, (code ? false : true), false); - if (TSDB_CODE_SUCCESS == code && autoCreateTbl) { - if (NULL == pRsp) { - tscError("no submit resp got for auto create table"); - code = TSDB_CODE_APP_ERROR; - } else { - code = stmtUpdateTableUid(pStmt, pRsp); - } - } - tFreeSSubmitRsp(pRsp); ++pStmt->sql.runTimes; diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 90c405c2049d273b8705de2de11faf659fbd4e19..01c99c6e9ed16048a715839dc4ccb18ca04c3852 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -418,7 +418,10 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { static void tmqCommitRspCountDown(SMqCommitCbParamSet* pParamSet) { int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); - ASSERT(waitingRspNum >= 0); + if(ASSERT(waitingRspNum >= 0)){ + tscError("tmqCommitRspCountDown error:%d", waitingRspNum); + return; + } if (waitingRspNum == 0) { tmqCommitDone(pParamSet); } @@ -530,7 +533,6 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_commit_cb* userCb, void* userParam) { char* topic; int32_t vgId; - ASSERT(msg != NULL); if (TD_RES_TMQ(msg)) { SMqRspObj* pRspObj = (SMqRspObj*)msg; topic = pRspObj->topic; @@ -809,8 +811,6 @@ int32_t tmqHandleAllDelayedTask(tmq_t* tmq) { taosTmrReset(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, pRefId, tmqMgmt.timer, &tmq->commitTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__REPORT) { - } else { - ASSERT(0); } taosFreeQitem(pTaskType); } @@ -955,10 +955,6 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { const char* user = conf->user == NULL ? TSDB_DEFAULT_USER : conf->user; const char* pass = conf->pass == NULL ? TSDB_DEFAULT_PASS : conf->pass; - ASSERT(user); - ASSERT(pass); - ASSERT(conf->groupId[0]); - pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); pTmq->mqueue = taosOpenQueue(); pTmq->qall = taosAllocateQall(); @@ -1249,8 +1245,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp); tDecoderClear(&decoder); memcpy(&pRspWrapper->taosxRsp, pMsg->pData, sizeof(SMqRspHead)); - } else { - ASSERT(0); } taosMemoryFree(pMsg->pData); diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 54778e87a788190c55fafc0e4d0db732677d795d..76911e229a569403ccdcc36b4b90470d3f02618e 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -25,7 +25,7 @@ #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" -#include "../src/clientSml.c" +#include "../inc/clientSml.h" #include "taos.h" int main(int argc, char **argv) { @@ -40,11 +40,14 @@ TEST(testCase, smlParseInfluxString_Test) { msgBuf.len = 256; SSmlLineInfo elements = {0}; + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->protocol = TSDB_SML_LINE_PROTOCOL; + info->dataFormat = false; // case 1 char *tmp = "\\,st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; char *sql = (char *)taosMemoryCalloc(256, 1); memcpy(sql, tmp, strlen(tmp) + 1); - int ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measureLen, strlen(",st")); @@ -58,28 +61,23 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 2 false tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); - - // case 3 false - tmp = "st, t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); - ASSERT_EQ(ret, 0); - ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 1); - ASSERT_EQ(elements.colsLen, strlen("t1=3,t2=4,t3=t3")); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 4 tag is null tmp = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measureLen, strlen("st")); @@ -93,12 +91,14 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 5 tag is null tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql + 1); ASSERT_EQ(elements.measureLen, strlen("st")); @@ -111,91 +111,104 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + 1 + elements.measureTagsLen + 3 + elements.colsLen + 2); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 6 tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; + smlClearForRerun(info); // case 7 tmp = " st , "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); - ASSERT_EQ(ret, 0); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); + ASSERT_NE(ret, 0); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 8 false tmp = ", st , "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; + taosMemoryFree(sql); + smlDestroyInfo(info); + } TEST(testCase, smlParseCols_Error_Test) { - const char *data[] = {"c=\"89sd", // binary, nchar - "c=j\"89sd\"", - "c=\"89sd\"k", - "c=u", // bool - "c=truet", - "c=f64", // double - "c=8f64f", - "c=8ef64", - "c=f32", // float - "c=8f32f", - "c=8wef32", - "c=-3.402823466e+39f32", - "c=", // double - "c=8f", - "c=8we", - "c=i8", // tiny int - "c=-8i8f", - "c=8wei8", - "c=-999i8", - "c=u8", // u tiny int - "c=8fu8", - "c=8weu8", - "c=999u8", - "c=-8u8", - "c=i16", // small int - "c=8fi16u", - "c=8wei16", - "c=-67787i16", - "c=u16", // u small int - "c=8u16f", - "c=8weu16", - "c=-9u16", - "c=67787u16", - "c=i32", // int - "c=8i32f", - "c=8wei32", - "c=2147483649i32", - "c=u32", // u int - "c=8u32f", - "c=8weu32", - "c=-4u32", - "c=42949672958u32", - "c=i64", // big int - "c=8i64i", - "c=8wei64", - "c=-9223372036854775809i64", - "c=i", // big int - "c=8fi", - "c=8wei", - "c=9223372036854775808i", - "c=u64", // u big int - "c=8u64f", - "c=8weu64", - "c=-3.402823466e+39u64", - "c=-339u64", - "c=18446744073709551616u64", - "c=1,c=2", - "c=1=2"}; - - SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + const char *data[] = {"st,t=1 c=\"89sd 1626006833639000000", // binary, nchar + "st,t=1 c=j\"89sd\" 1626006833639000000", + "st,t=1 c=\"89sd\"k 1626006833639000000", + "st,t=1 c=u 1626006833639000000", // bool + "st,t=1 c=truet 1626006833639000000", + "st,t=1 c=f64 1626006833639000000", // double + "st,t=1 c=8f64f 1626006833639000000", + "st,t=1 c=8ef64 1626006833639000000", + "st,t=1 c=f32 1626006833639000000", // float + "st,t=1 c=8f32f 1626006833639000000", + "st,t=1 c=8wef32 1626006833639000000", + "st,t=1 c=-3.402823466e+39f32 1626006833639000000", + "st,t=1 c= 1626006833639000000", // double + "st,t=1 c=8f 1626006833639000000", + "st,t=1 c=8we 1626006833639000000", + "st,t=1 c=i8 1626006833639000000", // tiny int + "st,t=1 c=-8i8f 1626006833639000000", + "st,t=1 c=8wei8 1626006833639000000", + "st,t=1 c=-999i8 1626006833639000000", + "st,t=1 c=u8 1626006833639000000", // u tiny int + "st,t=1 c=8fu8 1626006833639000000", + "st,t=1 c=8weu8 1626006833639000000", + "st,t=1 c=999u8 1626006833639000000", + "st,t=1 c=-8u8 1626006833639000000", + "st,t=1 c=i16 1626006833639000000", // small int + "st,t=1 c=8fi16u 1626006833639000000", + "st,t=1 c=8wei16 1626006833639000000", + "st,t=1 c=-67787i16 1626006833639000000", + "st,t=1 c=u16 1626006833639000000", // u small int + "st,t=1 c=8u16f 1626006833639000000", + "st,t=1 c=8weu16 1626006833639000000", + "st,t=1 c=-9u16 1626006833639000000", + "st,t=1 c=67787u16 1626006833639000000", + "st,t=1 c=i32 1626006833639000000", // int + "st,t=1 c=8i32f 1626006833639000000", + "st,t=1 c=8wei32 1626006833639000000", + "st,t=1 c=2147483649i32 1626006833639000000", + "st,t=1 c=u32 1626006833639000000", // u int + "st,t=1 c=8u32f 1626006833639000000", + "st,t=1 c=8weu32 1626006833639000000", + "st,t=1 c=-4u32 1626006833639000000", + "st,t=1 c=42949672958u32 1626006833639000000", + "st,t=1 c=i64 1626006833639000000", // big int + "st,t=1 c=8i64i 1626006833639000000", + "st,t=1 c=8wei64 1626006833639000000", + "st,t=1 c=-9223372036854775809i64 1626006833639000000", + "st,t=1 c=i 1626006833639000000", // big int + "st,t=1 c=8fi 1626006833639000000", + "st,t=1 c=8wei 1626006833639000000", + "st,t=1 c=9223372036854775808i 1626006833639000000", + "st,t=1 c=u64 1626006833639000000", // u big int + "st,t=1 c=8u64f 1626006833639000000", + "st,t=1 c=8weu64 1626006833639000000", + "st,t=1 c=-3.402823466e+39u64 1626006833639000000", + "st,t=1 c=-339u64 1626006833639000000", + "st,t=1 c=18446744073709551616u64 1626006833639000000", + "st,t=1 c=1=2 1626006833639000000"}; + + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->protocol = TSDB_SML_LINE_PROTOCOL; + info->dataFormat = false; for (int i = 0; i < sizeof(data) / sizeof(data[0]); i++) { char msg[256] = {0}; SSmlMsgBuf msgBuf; @@ -204,76 +217,14 @@ TEST(testCase, smlParseCols_Error_Test) { int32_t len = strlen(data[i]); char *sql = (char *)taosMemoryCalloc(256, 1); memcpy(sql, data[i], len + 1); - SArray *cols = taosArrayInit(8, POINTER_BYTES); - int32_t ret = smlParseCols(sql, len, cols, NULL, false, dumplicateKey, &msgBuf); - printf("i:%d\n", i); + SSmlLineInfo elements = {0}; + int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); +// printf("i:%d\n", i); ASSERT_NE(ret, TSDB_CODE_SUCCESS); - taosHashClear(dumplicateKey); taosMemoryFree(sql); - for (int j = 0; j < taosArrayGetSize(cols); j++) { - void *kv = taosArrayGetP(cols, j); - taosMemoryFree(kv); - } - taosArrayDestroy(cols); - } - taosHashCleanup(dumplicateKey); -} - -TEST(testCase, smlParseCols_tag_Test) { - char msg[256] = {0}; - SSmlMsgBuf msgBuf; - msgBuf.buf = msg; - msgBuf.len = 256; - - SArray *cols = taosArrayInit(16, POINTER_BYTES); - ASSERT_NE(cols, nullptr); - SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - - const char *data = - "cbin=\"passit " - "helloc\",cnch=L\"iisdfsf\",cbool=false,cf64=4.31f64,cf64_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=" - "898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_" - "=l\"iuwq\""; - int32_t len = strlen(data); - int32_t ret = smlParseCols(data, len, cols, NULL, true, dumplicateKey, &msgBuf); - ASSERT_EQ(ret, TSDB_CODE_SUCCESS); - int32_t size = taosArrayGetSize(cols); - ASSERT_EQ(size, 19); - - // nchar - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, 0); - ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0); - ASSERT_EQ(kv->keyLen, 4); - ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); - ASSERT_EQ(kv->length, 15); - ASSERT_EQ(strncasecmp(kv->value, "\"passit", 7), 0); - - // nchar - kv = (SSmlKv *)taosArrayGetP(cols, 3); - ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); - ASSERT_EQ(kv->keyLen, 4); - ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); - ASSERT_EQ(kv->length, 7); - ASSERT_EQ(strncasecmp(kv->value, "4.31f64", 7), 0); - - for (int i = 0; i < size; i++) { - void *tmp = taosArrayGetP(cols, i); - taosMemoryFree(tmp); + taosArrayDestroy(elements.colArray); } - taosArrayClear(cols); - - // test tag is null - data = "t=3e"; - len = 0; - memset(msgBuf.buf, 0, msgBuf.len); - taosHashClear(dumplicateKey); - ret = smlParseCols(data, len, cols, NULL, true, dumplicateKey, &msgBuf); - ASSERT_EQ(ret, TSDB_CODE_SUCCESS); - size = taosArrayGetSize(cols); - ASSERT_EQ(size, 0); - - taosArrayDestroy(cols); - taosHashCleanup(dumplicateKey); + smlDestroyInfo(info); } TEST(testCase, smlParseCols_Test) { @@ -281,202 +232,183 @@ TEST(testCase, smlParseCols_Test) { SSmlMsgBuf msgBuf; msgBuf.buf = msg; msgBuf.len = 256; - - SArray *cols = taosArrayInit(16, POINTER_BYTES); - ASSERT_NE(cols, nullptr); - - SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->protocol = TSDB_SML_LINE_PROTOCOL; + info->dataFormat = false; + SSmlLineInfo elements = {0}; + info->msgBuf = msgBuf; const char *data = - "cb\\=in=\"pass\\,it " + "st,t=1 cb\\=in=\"pass\\,it " "hello,c=2\",cnch=L\"ii\\=sdfsf\",cbool=false,cf64=4.31f64,cf64_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=" "233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t," - "cboolf=f,cnch_=l\"iuwq\""; + "cboolf=f,cnch_=l\"iuwq\" 1626006833639000000"; int32_t len = strlen(data); char *sql = (char *)taosMemoryCalloc(1024, 1); memcpy(sql, data, len + 1); - int32_t ret = smlParseCols(sql, len, cols, NULL, false, dumplicateKey, &msgBuf); + int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); ASSERT_EQ(ret, TSDB_CODE_SUCCESS); - int32_t size = taosArrayGetSize(cols); - ASSERT_EQ(size, 19); + int32_t size = taosArrayGetSize(elements.colArray); + ASSERT_EQ(size, 20); // binary - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, 0); + SSmlKv *kv = (SSmlKv *)taosArrayGet(elements.colArray, 1); ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY); ASSERT_EQ(kv->length, 17); ASSERT_EQ(strncasecmp(kv->value, "pass,it ", 8), 0); - taosMemoryFree(kv); // nchar - kv = (SSmlKv *)taosArrayGetP(cols, 1); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 2); ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->length, 8); ASSERT_EQ(strncasecmp(kv->value, "ii=sd", 5), 0); - taosMemoryFree(kv); // bool - kv = (SSmlKv *)taosArrayGetP(cols, 2); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 3); ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, false); - taosMemoryFree(kv); // double - kv = (SSmlKv *)taosArrayGetP(cols, 3); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 4); ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); ASSERT_EQ(kv->length, 8); // ASSERT_EQ(kv->d, 4.31); printf("4.31 = kv->d:%f\n", kv->d); - taosMemoryFree(kv); // float - kv = (SSmlKv *)taosArrayGetP(cols, 4); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 5); ASSERT_EQ(strncasecmp(kv->key, "cf64_", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); ASSERT_EQ(kv->length, 8); // ASSERT_EQ(kv->f, 8.32); printf("8.32 = kv->d:%f\n", kv->d); - taosMemoryFree(kv); // float - kv = (SSmlKv *)taosArrayGetP(cols, 5); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 6); ASSERT_EQ(strncasecmp(kv->key, "cf32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); ASSERT_EQ(kv->length, 4); // ASSERT_EQ(kv->f, 8.23); printf("8.23 = kv->f:%f\n", kv->f); - taosMemoryFree(kv); // tiny int - kv = (SSmlKv *)taosArrayGetP(cols, 6); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 7); ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0); ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, -34); - taosMemoryFree(kv); // unsigned tiny int - kv = (SSmlKv *)taosArrayGetP(cols, 7); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 8); ASSERT_EQ(strncasecmp(kv->key, "cu8", 3), 0); ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UTINYINT); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->u, 89); - taosMemoryFree(kv); // small int - kv = (SSmlKv *)taosArrayGetP(cols, 8); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 9); ASSERT_EQ(strncasecmp(kv->key, "ci16", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_SMALLINT); ASSERT_EQ(kv->length, 2); ASSERT_EQ(kv->u, 233); - taosMemoryFree(kv); // unsigned smallint - kv = (SSmlKv *)taosArrayGetP(cols, 9); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 10); ASSERT_EQ(strncasecmp(kv->key, "cu16", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_USMALLINT); ASSERT_EQ(kv->length, 2); ASSERT_EQ(kv->u, 898); - taosMemoryFree(kv); // int - kv = (SSmlKv *)taosArrayGetP(cols, 10); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 11); ASSERT_EQ(strncasecmp(kv->key, "ci32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_INT); ASSERT_EQ(kv->length, 4); ASSERT_EQ(kv->u, 98289); - taosMemoryFree(kv); // unsigned int - kv = (SSmlKv *)taosArrayGetP(cols, 11); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 12); ASSERT_EQ(strncasecmp(kv->key, "cu32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UINT); ASSERT_EQ(kv->length, 4); ASSERT_EQ(kv->u, 12323); - taosMemoryFree(kv); // bigint - kv = (SSmlKv *)taosArrayGetP(cols, 12); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 13); ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(kv->length, 8); ASSERT_EQ(kv->i, -89238); - taosMemoryFree(kv); // bigint - kv = (SSmlKv *)taosArrayGetP(cols, 13); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 14); ASSERT_EQ(strncasecmp(kv->key, "ci", 2), 0); ASSERT_EQ(kv->keyLen, 2); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(kv->length, 8); ASSERT_EQ(kv->i, 989); - taosMemoryFree(kv); // unsigned bigint - kv = (SSmlKv *)taosArrayGetP(cols, 14); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 15); ASSERT_EQ(strncasecmp(kv->key, "cu64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UBIGINT); ASSERT_EQ(kv->length, 8); ASSERT_EQ(kv->u, 8989323); - taosMemoryFree(kv); // bool - kv = (SSmlKv *)taosArrayGetP(cols, 15); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 16); ASSERT_EQ(strncasecmp(kv->key, "cbooltrue", 9), 0); ASSERT_EQ(kv->keyLen, 9); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, true); - taosMemoryFree(kv); // bool - kv = (SSmlKv *)taosArrayGetP(cols, 16); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 17); ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, true); - taosMemoryFree(kv); // bool - kv = (SSmlKv *)taosArrayGetP(cols, 17); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 18); ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, false); - taosMemoryFree(kv); // nchar - kv = (SSmlKv *)taosArrayGetP(cols, 18); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 19); ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->length, 4); ASSERT_EQ(strncasecmp(kv->value, "iuwq", 4), 0); - taosMemoryFree(kv); - taosArrayDestroy(cols); - taosHashCleanup(dumplicateKey); + taosArrayDestroy(elements.colArray); taosMemoryFree(sql); + smlDestroyInfo(info); } TEST(testCase, smlGetTimestampLen_Test) { @@ -515,7 +447,9 @@ TEST(testCase, smlParseNumber_Test) { } TEST(testCase, smlParseTelnetLine_error_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->dataFormat = false; + info->protocol = TSDB_SML_TELNET_PROTOCOL; ASSERT_NE(info, nullptr); const char *sql[] = { @@ -532,479 +466,95 @@ TEST(testCase, smlParseTelnetLine_error_Test) { "sys.procs.running 1479496100 42 ", "sys.procs.running 1479496100 42 host= ", "sys.procs.running 1479496100 42or host=web01", - "sys.procs.running 1479496100 true host=web01", - "sys.procs.running 1479496100 \"binary\" host=web01", - "sys.procs.running 1479496100 L\"rfr\" host=web01", +// "sys.procs.running 1479496100 true host=web01", +// "sys.procs.running 1479496100 \"binary\" host=web01", +// "sys.procs.running 1479496100 L\"rfr\" host=web01", "sys.procs.running 1479496100 42 host=web01 cpu= ", - "sys.procs.running 1479496100 42 host=web01 host=w2", "sys.procs.running 1479496100 42 host=web01 host", "sys.procs.running 1479496100 42 host=web01=er", "sys.procs.running 1479496100 42 host= web01", }; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - int ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); + SSmlLineInfo elements = {0}; + int ret = smlParseTelnetString(info, (char*)sql[i], (char*)(sql[i] + strlen(sql[i])), &elements); +// printf("i:%d\n", i); ASSERT_NE(ret, 0); } smlDestroyInfo(info); } -TEST(testCase, smlParseTelnetLine_diff_type_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - - const char *sql[] = {"sys.procs.running 1479496104000 42 host=web01", - "sys.procs.running 1479496104000 42u8 host=web01", - "appywjnuct 1626006833641 True id=\"appywjnuct_40601_49808_1\" t0=t t1=127i8 " - "id=\"appywjnuct_40601_49808_2\" t2=32767i16 t3=2147483647i32 t4=9223372036854775807i64 " - "t5=11.12345f32 t6=22.123456789f64 t7=\"binaryTagValue\" t8=L\"ncharTagValue\""}; - - int ret = TSDB_CODE_SUCCESS; - for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); - if (ret != TSDB_CODE_SUCCESS) break; - } - ASSERT_NE(ret, 0); - smlDestroyInfo(info); -} - -TEST(testCase, smlParseTelnetLine_json_error_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); +TEST(testCase, smlParseTelnetLine_Test) { + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->dataFormat = false; + info->protocol = TSDB_SML_TELNET_PROTOCOL; ASSERT_NE(info, nullptr); const char *sql[] = { - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 13468464009999333322222223,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"web01\",\n" - " \"dc\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400i,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"web01\",\n" - " \"dc\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"groupid\": { \n" - " \"value\" : 2,\n" - " \"type\" : \"nchar\"\n" - " },\n" - " \"location\": { \n" - " \"value\" : \"北京\",\n" - " \"type\" : \"binary\"\n" - " },\n" - " \"id\": \"d1001\"\n" - " }\n" - " },\n" - "]", + "twudyr 1626006833641 \"abcd`~!@#$%^&*()_-{[}]|:;<.>?lfjal\" id=twudyr_17102_17825 t0=t t1=127i8 t2=32767i16 t3=2147483647i32 t4=9223372036854775807i64 t5=11.12345f32 t6=22.123456789f64 t7=\"abcd`~!@#$%^&*()_-{[}]|:;<.>?lfjal\" t8=L\"abcd`~!@#$%^&*()_-{[}]|:;<.>?lfjal\"", }; - - int ret = TSDB_CODE_SUCCESS; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); - ASSERT_NE(ret, 0); + SSmlLineInfo elements = {0}; + int ret = smlParseTelnetString(info, (char*)sql[i], (char*)(sql[i] + strlen(sql[i])), &elements); +// printf("i:%d\n", i); + ASSERT_EQ(ret, 0); } smlDestroyInfo(info); } -TEST(testCase, smlParseTelnetLine_diff_json_type1_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - - const char *sql[] = { - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": 8\n" - " }\n" - " },\n" - "]", - }; - - int ret = TSDB_CODE_SUCCESS; - for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); - if (ret != TSDB_CODE_SUCCESS) break; - } - ASSERT_NE(ret, 0); - smlDestroyInfo(info); -} - -TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - - const char *sql[] = { - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": \"18\",\n" - " \"tags\": {\n" - " \"host\": \"fff\"\n" - " }\n" - " },\n" - "]", - }; - int ret = TSDB_CODE_SUCCESS; - for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); - if (ret != TSDB_CODE_SUCCESS) break; - } - ASSERT_NE(ret, 0); - smlDestroyInfo(info); -} - -TEST(testCase, sml_col_4096_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - - const char *sql[] = { - "spgwgvldxv,id=spgwgvldxv_1,t0=f " - "c0=t,c1=t,c2=t,c3=t,c4=t,c5=t,c6=t,c7=t,c8=t,c9=t,c10=t,c11=t,c12=t,c13=t,c14=t,c15=t,c16=t,c17=t,c18=t,c19=t," - "c20=t,c21=t,c22=t,c23=t,c24=t,c25=t,c26=t,c27=t,c28=t,c29=t,c30=t,c31=t,c32=t,c33=t,c34=t,c35=t,c36=t,c37=t,c38=" - "t,c39=t,c40=t,c41=t,c42=t,c43=t,c44=t,c45=t,c46=t,c47=t,c48=t,c49=t,c50=t,c51=t,c52=t,c53=t,c54=t,c55=t,c56=t," - "c57=t,c58=t,c59=t,c60=t,c61=t,c62=t,c63=t,c64=t,c65=t,c66=t,c67=t,c68=t,c69=t,c70=t,c71=t,c72=t,c73=t,c74=t,c75=" - "t,c76=t,c77=t,c78=t,c79=t,c80=t,c81=t,c82=t,c83=t,c84=t,c85=t,c86=t,c87=t,c88=t,c89=t,c90=t,c91=t,c92=t,c93=t," - "c94=t,c95=t,c96=t,c97=t,c98=t,c99=t,c100=t," - "c101=t,c102=t,c103=t,c104=t,c105=t,c106=t,c107=t,c108=t,c109=t,c110=t,c111=t,c112=t,c113=t,c114=t,c115=t,c116=t," - "c117=t,c118=t,c119=t,c120=t,c121=t,c122=t,c123=t,c124=t,c125=t,c126=t,c127=t,c128=t,c129=t,c130=t,c131=t,c132=t," - "c133=t,c134=t,c135=t,c136=t,c137=t,c138=t,c139=t,c140=t,c141=t,c142=t,c143=t,c144=t,c145=t,c146=t,c147=t,c148=t," - "c149=t,c150=t,c151=t,c152=t,c153=t,c154=t,c155=t,c156=t,c157=t,c158=t,c159=t,c160=t,c161=t,c162=t,c163=t,c164=t," - "c165=t,c166=t,c167=t,c168=t,c169=t,c170=t,c171=t,c172=t,c173=t,c174=t,c175=t,c176=t,c177=t,c178=t,c179=t,c180=t," - "c181=t,c182=t,c183=t,c184=t,c185=t,c186=t,c187=t,c188=t,c189=t," - "c190=t,c191=t,c192=t,c193=t,c194=t,c195=t,c196=t,c197=t,c198=t,c199=t,c200=t,c201=t,c202=t,c203=t,c204=t,c205=t," - "c206=t,c207=t,c208=t,c209=t,c210=t,c211=t,c212=t,c213=t,c214=t,c215=t,c216=t,c217=t,c218=t,c219=t,c220=t,c221=t," - "c222=t,c223=t,c224=t,c225=t,c226=t,c227=t,c228=t,c229=t,c230=t,c231=t,c232=t,c233=t,c234=t,c235=t,c236=t,c237=t," - "c238=t,c239=t,c240=t,c241=t,c242=t,c243=t,c244=t,c245=t,c246=t,c247=t,c248=t,c249=t,c250=t,c251=t,c252=t,c253=t," - "c254=t,c255=t,c256=t,c257=t,c258=t,c259=t,c260=t,c261=t,c262=t,c263=t,c264=t,c265=t,c266=t,c267=t,c268=t,c269=t," - "c270=t,c271=t,c272=t,c273=t,c274=t,c275=t,c276=t,c277=t,c278=t," - "c279=t,c280=t,c281=t,c282=t,c283=t,c284=t,c285=t,c286=t,c287=t,c288=t,c289=t,c290=t,c291=t,c292=t,c293=t,c294=t," - "c295=t,c296=t,c297=t,c298=t,c299=t,c300=t,c301=t,c302=t,c303=t,c304=t,c305=t,c306=t,c307=t,c308=t,c309=t,c310=t," - "c311=t,c312=t,c313=t,c314=t,c315=t,c316=t,c317=t,c318=t,c319=t,c320=t,c321=t,c322=t,c323=t,c324=t,c325=t,c326=t," - "c327=t,c328=t,c329=t,c330=t,c331=t,c332=t,c333=t,c334=t,c335=t,c336=t,c337=t,c338=t,c339=t,c340=t,c341=t,c342=t," - "c343=t,c344=t,c345=t,c346=t,c347=t,c348=t,c349=t,c350=t,c351=t,c352=t,c353=t,c354=t,c355=t,c356=t,c357=t,c358=t," - "c359=t,c360=t,c361=t,c362=t,c363=t,c364=t,c365=t,c366=t,c367=t,c368=t,c369=t,c370=t,c371=t,c372=t,c373=t,c374=t," - "c375=t,c376=t,c377=t,c378=t,c379=t,c380=t,c381=t,c382=t,c383=t,c384=t,c385=t,c386=t,c387=t,c388=t,c389=t,c390=t," - "c391=t,c392=t,c393=t,c394=t,c395=t,c396=t,c397=t,c398=t,c399=t,c400=t,c401=t,c402=t,c403=t,c404=t,c405=t,c406=t," - "c407=t,c408=t,c409=t,c410=t,c411=t,c412=t,c413=t,c414=t,c415=t,c416=t,c417=t,c418=t,c419=t,c420=t,c421=t,c422=t," - "c423=t,c424=t,c425=t,c426=t,c427=t,c428=t,c429=t,c430=t,c431=t,c432=t,c433=t,c434=t,c435=t,c436=t,c437=t,c438=t," - "c439=t,c440=t,c441=t,c442=t,c443=t,c444=t,c445=t,c446=t," - "c447=t,c448=t,c449=t,c450=t,c451=t,c452=t,c453=t,c454=t,c455=t,c456=t,c457=t,c458=t,c459=t,c460=t,c461=t,c462=t," - "c463=t,c464=t,c465=t,c466=t,c467=t,c468=t,c469=t,c470=t,c471=t,c472=t,c473=t,c474=t,c475=t,c476=t,c477=t,c478=t," - "c479=t,c480=t,c481=t,c482=t,c483=t,c484=t,c485=t,c486=t,c487=t,c488=t,c489=t,c490=t,c491=t,c492=t,c493=t,c494=t," - "c495=t,c496=t,c497=t,c498=t,c499=t,c500=t,c501=t,c502=t,c503=t,c504=t,c505=t,c506=t,c507=t,c508=t,c509=t,c510=t," - "c511=t,c512=t,c513=t,c514=t,c515=t,c516=t,c517=t,c518=t,c519=t,c520=t,c521=t,c522=t,c523=t,c524=t,c525=t,c526=t," - "c527=t,c528=t,c529=t,c530=t,c531=t,c532=t,c533=t,c534=t,c535=t,c536=t,c537=t,c538=t,c539=t,c540=t,c541=t,c542=t," - "c543=t,c544=t,c545=t,c546=t,c547=t,c548=t,c549=t,c550=t,c551=t,c552=t,c553=t,c554=t,c555=t,c556=t,c557=t,c558=t," - "c559=t,c560=t,c561=t,c562=t,c563=t,c564=t,c565=t,c566=t,c567=t,c568=t,c569=t,c570=t,c571=t,c572=t,c573=t,c574=t," - "c575=t,c576=t,c577=t,c578=t,c579=t,c580=t,c581=t,c582=t,c583=t,c584=t,c585=t,c586=t,c587=t,c588=t,c589=t,c590=t," - "c591=t,c592=t,c593=t,c594=t,c595=t,c596=t,c597=t,c598=t,c599=t,c600=t,c601=t,c602=t,c603=t,c604=t,c605=t,c606=t," - "c607=t,c608=t,c609=t,c610=t,c611=t,c612=t,c613=t,c614=t," - "c615=t,c616=t,c617=t,c618=t,c619=t,c620=t,c621=t,c622=t,c623=t,c624=t,c625=t,c626=t,c627=t,c628=t,c629=t,c630=t," - "c631=t,c632=t,c633=t,c634=t,c635=t,c636=t,c637=t,c638=t,c639=t,c640=t,c641=t,c642=t,c643=t,c644=t,c645=t,c646=t," - "c647=t,c648=t,c649=t,c650=t,c651=t,c652=t,c653=t,c654=t,c655=t,c656=t,c657=t,c658=t,c659=t,c660=t,c661=t,c662=t," - "c663=t,c664=t,c665=t,c666=t,c667=t,c668=t,c669=t,c670=t,c671=t,c672=t,c673=t,c674=t,c675=t,c676=t,c677=t,c678=t," - "c679=t,c680=t,c681=t,c682=t,c683=t,c684=t,c685=t,c686=t,c687=t,c688=t,c689=t,c690=t,c691=t,c692=t,c693=t,c694=t," - "c695=t,c696=t,c697=t,c698=t,c699=t,c700=t,c701=t,c702=t,c703=t,c704=t,c705=t,c706=t,c707=t,c708=t,c709=t,c710=t," - "c711=t,c712=t,c713=t,c714=t,c715=t,c716=t,c717=t,c718=t,c719=t,c720=t,c721=t,c722=t,c723=t,c724=t,c725=t,c726=t," - "c727=t,c728=t,c729=t,c730=t,c731=t,c732=t,c733=t,c734=t,c735=t,c736=t,c737=t,c738=t,c739=t,c740=t,c741=t,c742=t," - "c743=t,c744=t,c745=t,c746=t,c747=t,c748=t,c749=t,c750=t,c751=t,c752=t,c753=t,c754=t,c755=t,c756=t,c757=t,c758=t," - "c759=t,c760=t,c761=t,c762=t,c763=t,c764=t,c765=t,c766=t,c767=t,c768=t,c769=t,c770=t,c771=t,c772=t,c773=t,c774=t," - "c775=t,c776=t,c777=t,c778=t,c779=t,c780=t,c781=t,c782=t," - "c783=t,c784=t,c785=t,c786=t,c787=t,c788=t,c789=t,c790=t,c791=t,c792=t,c793=t,c794=t,c795=t,c796=t,c797=t,c798=t," - "c799=t,c800=t,c801=t,c802=t,c803=t,c804=t,c805=t,c806=t,c807=t,c808=t,c809=t,c810=t,c811=t,c812=t,c813=t," - "c814=t,c815=t,c816=t,c817=t,c818=t,c819=t,c820=t,c821=t,c822=t,c823=t,c824=t,c825=t,c826=t,c827=t,c828=t,c829=t," - "c830=t,c831=t,c832=t,c833=t,c834=t,c835=t,c836=t,c837=t,c838=t,c839=t,c840=t,c841=t,c842=t,c843=t,c844=t,c845=t," - "c846=t,c847=t,c848=t,c849=t,c850=t,c851=t,c852=t,c853=t,c854=t,c855=t,c856=t,c857=t,c858=t,c859=t,c860=t,c861=t," - "c862=t," - "c863=t,c864=t,c865=t,c866=t,c867=t,c868=t,c869=t,c870=t,c871=t,c872=t,c873=t,c874=t,c875=t,c876=t,c877=t,c878=t," - "c879=t,c880=t,c881=t,c882=t,c883=t,c884=t,c885=t,c886=t,c887=t,c888=t,c889=t,c890=t,c891=t,c892=t,c893=t,c894=t," - "c895=t,c896=t,c897=t,c898=t,c899=t,c900=t,c901=t,c902=t,c903=t,c904=t,c905=t,c906=t,c907=t,c908=t,c909=t,c910=t," - "c911=t,c912=t,c913=t,c914=t,c915=t,c916=t,c917=t,c918=t,c919=t,c920=t,c921=t,c922=t,c923=t,c924=t,c925=t,c926=t," - "c927=t,c928=t,c929=t,c930=t,c931=t,c932=t,c933=t,c934=t,c935=t,c936=t,c937=t,c938=t,c939=t,c940=t,c941=t,c942=t," - "c943=t,c944=t,c945=t,c946=t,c947=t,c948=t,c949=t,c950=t,c951=t,c952=t,c953=t,c954=t,c955=t,c956=t,c957=t,c958=t," - "c959=t,c960=t,c961=t,c962=t,c963=t,c964=t,c965=t,c966=t,c967=t,c968=t,c969=t,c970=t,c971=t,c972=t,c973=t,c974=t," - "c975=t,c976=t,c977=t,c978=t,c979=t,c980=t,c981=t,c982=t,c983=t,c984=t,c985=t,c986=t,c987=t,c988=t,c989=t,c990=t," - "c991=t,c992=t,c993=t,c994=t,c995=t,c996=t,c997=t,c998=t,c999=t,c1000=t,c1001=t,c1002=t,c1003=t,c1004=t,c1005=t," - "c1006=t,c1007=t,c1008=t,c1009=t,c1010=t,c1011=t,c1012=t,c1013=t,c1014=t,c1015=t,c1016=t,c1017=t,c1018=t,c1019=t," - "c1020=t,c1021=t,c1022=t,c1023=t,c1024=t,c1025=t,c1026=t," - "c1027=t,c1028=t,c1029=t,c1030=t,c1031=t,c1032=t,c1033=t,c1034=t,c1035=t,c1036=t,c1037=t,c1038=t,c1039=t,c1040=t," - "c1041=t,c1042=t,c1043=t,c1044=t,c1045=t,c1046=t,c1047=t,c1048=t,c1049=t,c1050=t,c1051=t,c1052=t,c1053=t,c1054=t," - "c1055=t,c1056=t,c1057=t,c1058=t,c1059=t,c1060=t,c1061=t,c1062=t,c1063=t,c1064=t,c1065=t,c1066=t,c1067=t,c1068=t," - "c1069=t,c1070=t,c1071=t,c1072=t,c1073=t,c1074=t,c1075=t,c1076=t,c1077=t,c1078=t,c1079=t,c1080=t,c1081=t,c1082=t," - "c1083=t,c1084=t,c1085=t,c1086=t,c1087=t,c1088=t,c1089=t,c1090=t,c1091=t,c1092=t,c1093=t,c1094=t,c1095=t,c1096=t," - "c1097=t,c1098=t,c1099=t,c1100=t,c1101=t,c1102=t,c1103=t,c1104=t,c1105=t,c1106=t,c1107=t,c1108=t,c1109=t,c1110=t," - "c1111=t,c1112=t,c1113=t,c1114=t,c1115=t,c1116=t,c1117=t,c1118=t,c1119=t,c1120=t,c1121=t,c1122=t,c1123=t,c1124=t," - "c1125=t,c1126=t,c1127=t,c1128=t,c1129=t,c1130=t,c1131=t,c1132=t,c1133=t,c1134=t,c1135=t,c1136=t,c1137=t,c1138=t," - "c1139=t,c1140=t,c1141=t,c1142=t,c1143=t,c1144=t,c1145=t,c1146=t,c1147=t,c1148=t,c1149=t,c1150=t,c1151=t,c1152=t," - "c1153=t,c1154=t,c1155=t,c1156=t,c1157=t,c1158=t,c1159=t,c1160=t,c1161=t,c1162=t,c1163=t,c1164=t,c1165=t,c1166=t," - "c1167=t,c1168=t,c1169=t,c1170=t,c1171=t,c1172=t,c1173=t," - "c1174=t,c1175=t,c1176=t,c1177=t,c1178=t,c1179=t,c1180=t,c1181=t,c1182=t,c1183=t,c1184=t,c1185=t,c1186=t,c1187=t," - "c1188=t,c1189=t,c1190=t,c1191=t,c1192=t,c1193=t,c1194=t,c1195=t,c1196=t,c1197=t,c1198=t,c1199=t,c1200=t,c1201=t," - "c1202=t,c1203=t,c1204=t,c1205=t,c1206=t,c1207=t,c1208=t,c1209=t,c1210=t,c1211=t,c1212=t,c1213=t,c1214=t,c1215=t," - "c1216=t,c1217=t,c1218=t,c1219=t,c1220=t,c1221=t,c1222=t,c1223=t,c1224=t,c1225=t,c1226=t,c1227=t,c1228=t,c1229=t," - "c1230=t,c1231=t,c1232=t,c1233=t,c1234=t,c1235=t,c1236=t,c1237=t,c1238=t,c1239=t,c1240=t,c1241=t,c1242=t,c1243=t," - "c1244=t,c1245=t,c1246=t,c1247=t,c1248=t,c1249=t,c1250=t,c1251=t,c1252=t,c1253=t,c1254=t,c1255=t,c1256=t,c1257=t," - "c1258=t,c1259=t,c1260=t,c1261=t,c1262=t,c1263=t,c1264=t,c1265=t,c1266=t,c1267=t,c1268=t,c1269=t,c1270=t,c1271=t," - "c1272=t,c1273=t,c1274=t,c1275=t,c1276=t,c1277=t,c1278=t,c1279=t,c1280=t,c1281=t,c1282=t,c1283=t,c1284=t,c1285=t," - "c1286=t,c1287=t,c1288=t,c1289=t,c1290=t,c1291=t,c1292=t,c1293=t,c1294=t,c1295=t,c1296=t,c1297=t,c1298=t,c1299=t," - "c1300=t,c1301=t,c1302=t,c1303=t,c1304=t,c1305=t,c1306=t,c1307=t,c1308=t,c1309=t,c1310=t,c1311=t,c1312=t,c1313=t," - "c1314=t,c1315=t,c1316=t,c1317=t,c1318=t,c1319=t,c1320=t," - "c1321=t,c1322=t,c1323=t,c1324=t,c1325=t,c1326=t,c1327=t,c1328=t,c1329=t,c1330=t,c1331=t,c1332=t,c1333=t,c1334=t," - "c1335=t,c1336=t,c1337=t,c1338=t,c1339=t,c1340=t,c1341=t,c1342=t,c1343=t,c1344=t,c1345=t,c1346=t,c1347=t," - "c1348=t,c1349=t,c1350=t,c1351=t,c1352=t,c1353=t,c1354=t,c1355=t,c1356=t,c1357=t,c1358=t,c1359=t,c1360=t,c1361=t," - "c1362=t,c1363=t,c1364=t,c1365=t,c1366=t,c1367=t,c1368=t,c1369=t,c1370=t,c1371=t,c1372=t,c1373=t,c1374=t,c1375=t," - "c1376=t,c1377=t,c1378=t,c1379=t,c1380=t,c1381=t,c1382=t,c1383=t,c1384=t,c1385=t,c1386=t,c1387=t,c1388=t,c1389=t," - "c1390=t,c1391=t,c1392=t,c1393=t,c1394=t,c1395=t,c1396=t,c1397=t,c1398=t,c1399=t,c1400=t,c1401=t,c1402=t,c1403=t," - "c1404=t,c1405=t,c1406=t,c1407=t,c1408=t,c1409=t,c1410=t,c1411=t,c1412=t,c1413=t,c1414=t,c1415=t,c1416=t,c1417=t," - "c1418=t,c1419=t,c1420=t,c1421=t,c1422=t,c1423=t,c1424=t,c1425=t,c1426=t,c1427=t,c1428=t,c1429=t,c1430=t,c1431=t," - "c1432=t,c1433=t,c1434=t,c1435=t,c1436=t,c1437=t,c1438=t,c1439=t,c1440=t,c1441=t,c1442=t,c1443=t,c1444=t,c1445=t," - "c1446=t,c1447=t,c1448=t,c1449=t,c1450=t,c1451=t,c1452=t,c1453=t,c1454=t,c1455=t,c1456=t,c1457=t,c1458=t,c1459=t," - "c1460=t,c1461=t,c1462=t,c1463=t,c1464=t,c1465=t,c1466=t,c1467=t,c1468=t,c1469=t,c1470=t,c1471=t,c1472=t,c1473=t," - "c1474=t,c1475=t,c1476=t,c1477=t,c1478=t,c1479=t,c1480=t,c1481=t,c1482=t,c1483=t,c1484=t,c1485=t,c1486=t,c1487=t," - "c1488=t,c1489=t,c1490=t,c1491=t,c1492=t,c1493=t,c1494=t," - "c1495=t,c1496=t,c1497=t,c1498=t,c1499=t,c1500=t,c1501=t,c1502=t,c1503=t,c1504=t,c1505=t,c1506=t,c1507=t,c1508=t," - "c1509=t,c1510=t,c1511=t,c1512=t,c1513=t,c1514=t,c1515=t,c1516=t,c1517=t,c1518=t,c1519=t,c1520=t,c1521=t,c1522=t," - "c1523=t,c1524=t,c1525=t,c1526=t,c1527=t,c1528=t,c1529=t,c1530=t,c1531=t,c1532=t,c1533=t,c1534=t,c1535=t,c1536=t," - "c1537=t,c1538=t,c1539=t,c1540=t,c1541=t,c1542=t,c1543=t,c1544=t,c1545=t,c1546=t,c1547=t,c1548=t,c1549=t,c1550=t," - "c1551=t,c1552=t,c1553=t,c1554=t,c1555=t,c1556=t,c1557=t,c1558=t,c1559=t,c1560=t,c1561=t,c1562=t,c1563=t,c1564=t," - "c1565=t,c1566=t,c1567=t,c1568=t,c1569=t,c1570=t,c1571=t,c1572=t,c1573=t,c1574=t,c1575=t,c1576=t,c1577=t,c1578=t," - "c1579=t,c1580=t,c1581=t,c1582=t,c1583=t,c1584=t,c1585=t,c1586=t,c1587=t,c1588=t,c1589=t,c1590=t,c1591=t,c1592=t," - "c1593=t,c1594=t,c1595=t,c1596=t,c1597=t,c1598=t,c1599=t,c1600=t,c1601=t,c1602=t,c1603=t,c1604=t,c1605=t,c1606=t," - "c1607=t,c1608=t,c1609=t,c1610=t,c1611=t,c1612=t,c1613=t,c1614=t,c1615=t,c1616=t,c1617=t,c1618=t,c1619=t,c1620=t," - "c1621=t,c1622=t,c1623=t,c1624=t,c1625=t,c1626=t,c1627=t,c1628=t,c1629=t,c1630=t,c1631=t,c1632=t,c1633=t,c1634=t," - "c1635=t,c1636=t,c1637=t,c1638=t,c1639=t,c1640=t,c1641=t," - "c1642=t,c1643=t,c1644=t,c1645=t,c1646=t,c1647=t,c1648=t,c1649=t,c1650=t,c1651=t,c1652=t,c1653=t,c1654=t,c1655=t," - "c1656=t,c1657=t,c1658=t,c1659=t,c1660=t,c1661=t,c1662=t,c1663=t,c1664=t,c1665=t,c1666=t,c1667=t,c1668=t,c1669=t," - "c1670=t,c1671=t,c1672=t,c1673=t,c1674=t,c1675=t,c1676=t,c1677=t,c1678=t,c1679=t,c1680=t,c1681=t,c1682=t,c1683=t," - "c1684=t,c1685=t,c1686=t,c1687=t,c1688=t,c1689=t,c1690=t,c1691=t,c1692=t,c1693=t,c1694=t,c1695=t,c1696=t,c1697=t," - "c1698=t,c1699=t,c1700=t,c1701=t,c1702=t,c1703=t,c1704=t,c1705=t,c1706=t,c1707=t,c1708=t,c1709=t,c1710=t,c1711=t," - "c1712=t,c1713=t,c1714=t,c1715=t,c1716=t,c1717=t,c1718=t,c1719=t,c1720=t,c1721=t,c1722=t,c1723=t,c1724=t,c1725=t," - "c1726=t,c1727=t,c1728=t,c1729=t,c1730=t,c1731=t,c1732=t,c1733=t,c1734=t,c1735=t,c1736=t,c1737=t,c1738=t,c1739=t," - "c1740=t,c1741=t,c1742=t,c1743=t,c1744=t,c1745=t,c1746=t,c1747=t,c1748=t,c1749=t,c1750=t,c1751=t,c1752=t,c1753=t," - "c1754=t,c1755=t,c1756=t,c1757=t,c1758=t,c1759=t,c1760=t,c1761=t,c1762=t,c1763=t,c1764=t,c1765=t,c1766=t,c1767=t," - "c1768=t,c1769=t,c1770=t,c1771=t,c1772=t,c1773=t,c1774=t,c1775=t,c1776=t,c1777=t,c1778=t,c1779=t,c1780=t,c1781=t," - "c1782=t,c1783=t,c1784=t,c1785=t,c1786=t,c1787=t,c1788=t," - "c1789=t,c1790=t,c1791=t,c1792=t,c1793=t,c1794=t,c1795=t,c1796=t,c1797=t,c1798=t,c1799=t,c1800=t,c1801=t,c1802=t," - "c1803=t,c1804=t,c1805=t,c1806=t,c1807=t,c1808=t,c1809=t,c1810=t,c1811=t,c1812=t,c1813=t,c1814=t,c1815=t," - "c1816=t,c1817=t,c1818=t,c1819=t,c1820=t,c1821=t,c1822=t,c1823=t,c1824=t,c1825=t,c1826=t,c1827=t,c1828=t,c1829=t," - "c1830=t,c1831=t,c1832=t,c1833=t,c1834=t,c1835=t,c1836=t,c1837=t,c1838=t,c1839=t,c1840=t,c1841=t,c1842=t,c1843=t," - "c1844=t,c1845=t,c1846=t,c1847=t,c1848=t,c1849=t,c1850=t,c1851=t,c1852=t,c1853=t,c1854=t,c1855=t,c1856=t,c1857=t," - "c1858=t,c1859=t,c1860=t,c1861=t,c1862=t,c1863=t,c1864=t,c1865=t,c1866=t,c1867=t,c1868=t,c1869=t,c1870=t,c1871=t," - "c1872=t,c1873=t,c1874=t,c1875=t,c1876=t,c1877=t,c1878=t,c1879=t,c1880=t,c1881=t,c1882=t,c1883=t,c1884=t,c1885=t," - "c1886=t,c1887=t,c1888=t,c1889=t,c1890=t,c1891=t,c1892=t,c1893=t,c1894=t,c1895=t,c1896=t,c1897=t,c1898=t,c1899=t," - "c1900=t,c1901=t,c1902=t,c1903=t,c1904=t,c1905=t,c1906=t,c1907=t,c1908=t,c1909=t,c1910=t,c1911=t,c1912=t,c1913=t," - "c1914=t,c1915=t,c1916=t,c1917=t,c1918=t,c1919=t,c1920=t,c1921=t,c1922=t,c1923=t,c1924=t,c1925=t,c1926=t,c1927=t," - "c1928=t,c1929=t,c1930=t,c1931=t,c1932=t,c1933=t,c1934=t,c1935=t,c1936=t,c1937=t,c1938=t,c1939=t,c1940=t,c1941=t," - "c1942=t,c1943=t,c1944=t,c1945=t,c1946=t,c1947=t,c1948=t,c1949=t,c1950=t,c1951=t,c1952=t,c1953=t,c1954=t,c1955=t," - "c1956=t,c1957=t,c1958=t,c1959=t,c1960=t,c1961=t,c1962=t," - "c1963=t,c1964=t,c1965=t,c1966=t,c1967=t,c1968=t,c1969=t,c1970=t,c1971=t,c1972=t,c1973=t,c1974=t,c1975=t,c1976=t," - "c1977=t,c1978=t,c1979=t,c1980=t,c1981=t,c1982=t,c1983=t,c1984=t,c1985=t,c1986=t,c1987=t,c1988=t,c1989=t,c1990=t," - "c1991=t,c1992=t,c1993=t,c1994=t,c1995=t,c1996=t,c1997=t,c1998=t,c1999=t,c2000=t,c2001=t,c2002=t,c2003=t,c2004=t," - "c2005=t,c2006=t,c2007=t,c2008=t,c2009=t,c2010=t,c2011=t,c2012=t,c2013=t,c2014=t,c2015=t,c2016=t,c2017=t,c2018=t," - "c2019=t,c2020=t,c2021=t,c2022=t,c2023=t,c2024=t,c2025=t,c2026=t,c2027=t,c2028=t,c2029=t,c2030=t,c2031=t,c2032=t," - "c2033=t,c2034=t,c2035=t,c2036=t,c2037=t,c2038=t,c2039=t,c2040=t,c2041=t,c2042=t,c2043=t,c2044=t,c2045=t,c2046=t," - "c2047=t,c2048=t,c2049=t,c2050=t,c2051=t,c2052=t,c2053=t,c2054=t,c2055=t,c2056=t,c2057=t,c2058=t,c2059=t,c2060=t," - "c2061=t,c2062=t,c2063=t,c2064=t,c2065=t,c2066=t,c2067=t,c2068=t,c2069=t,c2070=t,c2071=t,c2072=t,c2073=t,c2074=t," - "c2075=t,c2076=t,c2077=t,c2078=t,c2079=t,c2080=t,c2081=t,c2082=t,c2083=t,c2084=t,c2085=t,c2086=t,c2087=t,c2088=t," - "c2089=t,c2090=t,c2091=t,c2092=t,c2093=t,c2094=t,c2095=t,c2096=t,c2097=t,c2098=t,c2099=t,c2100=t,c2101=t,c2102=t," - "c2103=t,c2104=t,c2105=t,c2106=t,c2107=t,c2108=t,c2109=t," - "c2110=t,c2111=t,c2112=t,c2113=t,c2114=t,c2115=t,c2116=t,c2117=t,c2118=t,c2119=t,c2120=t,c2121=t,c2122=t,c2123=t," - "c2124=t,c2125=t,c2126=t,c2127=t,c2128=t,c2129=t,c2130=t,c2131=t,c2132=t,c2133=t,c2134=t,c2135=t,c2136=t,c2137=t," - "c2138=t,c2139=t,c2140=t,c2141=t,c2142=t,c2143=t,c2144=t,c2145=t,c2146=t,c2147=t,c2148=t,c2149=t,c2150=t,c2151=t," - "c2152=t,c2153=t,c2154=t,c2155=t,c2156=t,c2157=t,c2158=t,c2159=t,c2160=t,c2161=t,c2162=t,c2163=t,c2164=t,c2165=t," - "c2166=t,c2167=t,c2168=t,c2169=t,c2170=t,c2171=t,c2172=t,c2173=t,c2174=t,c2175=t,c2176=t,c2177=t,c2178=t,c2179=t," - "c2180=t,c2181=t,c2182=t,c2183=t,c2184=t,c2185=t,c2186=t,c2187=t,c2188=t,c2189=t,c2190=t,c2191=t,c2192=t,c2193=t," - "c2194=t,c2195=t,c2196=t,c2197=t,c2198=t,c2199=t,c2200=t,c2201=t,c2202=t,c2203=t,c2204=t,c2205=t,c2206=t,c2207=t," - "c2208=t,c2209=t,c2210=t,c2211=t,c2212=t,c2213=t,c2214=t,c2215=t,c2216=t,c2217=t,c2218=t,c2219=t,c2220=t,c2221=t," - "c2222=t,c2223=t,c2224=t,c2225=t,c2226=t,c2227=t,c2228=t,c2229=t,c2230=t,c2231=t,c2232=t,c2233=t,c2234=t,c2235=t," - "c2236=t,c2237=t,c2238=t,c2239=t,c2240=t,c2241=t,c2242=t,c2243=t,c2244=t,c2245=t,c2246=t,c2247=t,c2248=t,c2249=t," - "c2250=t,c2251=t,c2252=t,c2253=t,c2254=t,c2255=t,c2256=t," - "c2257=t,c2258=t,c2259=t,c2260=t,c2261=t,c2262=t,c2263=t,c2264=t,c2265=t,c2266=t,c2267=t,c2268=t,c2269=t,c2270=t," - "c2271=t,c2272=t,c2273=t,c2274=t,c2275=t,c2276=t,c2277=t,c2278=t,c2279=t,c2280=t,c2281=t,c2282=t,c2283=t," - "c2284=t,c2285=t,c2286=t,c2287=t,c2288=t,c2289=t,c2290=t,c2291=t,c2292=t,c2293=t,c2294=t,c2295=t,c2296=t,c2297=t," - "c2298=t,c2299=t,c2300=t,c2301=t,c2302=t,c2303=t,c2304=t,c2305=t,c2306=t,c2307=t,c2308=t,c2309=t,c2310=t,c2311=t," - "c2312=t,c2313=t,c2314=t,c2315=t,c2316=t,c2317=t,c2318=t,c2319=t,c2320=t,c2321=t,c2322=t,c2323=t,c2324=t,c2325=t," - "c2326=t,c2327=t,c2328=t,c2329=t,c2330=t,c2331=t,c2332=t,c2333=t,c2334=t,c2335=t,c2336=t,c2337=t,c2338=t,c2339=t," - "c2340=t,c2341=t,c2342=t,c2343=t,c2344=t,c2345=t,c2346=t,c2347=t,c2348=t,c2349=t,c2350=t,c2351=t,c2352=t,c2353=t," - "c2354=t,c2355=t,c2356=t,c2357=t,c2358=t,c2359=t,c2360=t,c2361=t,c2362=t,c2363=t,c2364=t,c2365=t,c2366=t,c2367=t," - "c2368=t,c2369=t,c2370=t,c2371=t,c2372=t,c2373=t,c2374=t,c2375=t,c2376=t,c2377=t,c2378=t,c2379=t,c2380=t,c2381=t," - "c2382=t,c2383=t,c2384=t,c2385=t,c2386=t,c2387=t,c2388=t,c2389=t,c2390=t,c2391=t,c2392=t,c2393=t,c2394=t,c2395=t," - "c2396=t,c2397=t,c2398=t,c2399=t,c2400=t,c2401=t,c2402=t,c2403=t,c2404=t,c2405=t,c2406=t,c2407=t,c2408=t,c2409=t," - "c2410=t,c2411=t,c2412=t,c2413=t,c2414=t,c2415=t,c2416=t,c2417=t,c2418=t,c2419=t,c2420=t,c2421=t,c2422=t,c2423=t," - "c2424=t,c2425=t,c2426=t,c2427=t,c2428=t,c2429=t,c2430=t," - "c2431=t,c2432=t,c2433=t,c2434=t,c2435=t,c2436=t,c2437=t,c2438=t,c2439=t,c2440=t,c2441=t,c2442=t,c2443=t,c2444=t," - "c2445=t,c2446=t,c2447=t,c2448=t,c2449=t,c2450=t,c2451=t,c2452=t,c2453=t,c2454=t,c2455=t,c2456=t,c2457=t,c2458=t," - "c2459=t,c2460=t,c2461=t,c2462=t,c2463=t,c2464=t,c2465=t,c2466=t,c2467=t,c2468=t,c2469=t,c2470=t,c2471=t,c2472=t," - "c2473=t,c2474=t,c2475=t,c2476=t,c2477=t,c2478=t,c2479=t,c2480=t,c2481=t,c2482=t,c2483=t,c2484=t,c2485=t,c2486=t," - "c2487=t,c2488=t,c2489=t,c2490=t,c2491=t,c2492=t,c2493=t,c2494=t,c2495=t,c2496=t,c2497=t,c2498=t,c2499=t,c2500=t," - "c2501=t,c2502=t,c2503=t,c2504=t,c2505=t,c2506=t,c2507=t,c2508=t,c2509=t,c2510=t,c2511=t,c2512=t,c2513=t,c2514=t," - "c2515=t,c2516=t,c2517=t,c2518=t,c2519=t,c2520=t,c2521=t,c2522=t,c2523=t,c2524=t,c2525=t,c2526=t,c2527=t,c2528=t," - "c2529=t,c2530=t,c2531=t,c2532=t,c2533=t,c2534=t,c2535=t,c2536=t,c2537=t,c2538=t,c2539=t,c2540=t,c2541=t,c2542=t," - "c2543=t,c2544=t,c2545=t,c2546=t,c2547=t,c2548=t,c2549=t,c2550=t,c2551=t,c2552=t,c2553=t,c2554=t,c2555=t,c2556=t," - "c2557=t,c2558=t,c2559=t,c2560=t,c2561=t,c2562=t,c2563=t,c2564=t,c2565=t,c2566=t,c2567=t,c2568=t,c2569=t,c2570=t," - "c2571=t,c2572=t,c2573=t,c2574=t,c2575=t,c2576=t,c2577=t," - "c2578=t,c2579=t,c2580=t,c2581=t,c2582=t,c2583=t,c2584=t,c2585=t,c2586=t,c2587=t,c2588=t,c2589=t,c2590=t,c2591=t," - "c2592=t,c2593=t,c2594=t,c2595=t,c2596=t,c2597=t,c2598=t,c2599=t,c2600=t,c2601=t,c2602=t,c2603=t,c2604=t,c2605=t," - "c2606=t,c2607=t,c2608=t,c2609=t,c2610=t,c2611=t,c2612=t,c2613=t,c2614=t,c2615=t,c2616=t,c2617=t,c2618=t,c2619=t," - "c2620=t,c2621=t,c2622=t,c2623=t,c2624=t,c2625=t,c2626=t,c2627=t,c2628=t,c2629=t,c2630=t,c2631=t,c2632=t,c2633=t," - "c2634=t,c2635=t,c2636=t,c2637=t,c2638=t,c2639=t,c2640=t,c2641=t,c2642=t,c2643=t,c2644=t,c2645=t,c2646=t,c2647=t," - "c2648=t,c2649=t,c2650=t,c2651=t,c2652=t,c2653=t,c2654=t,c2655=t,c2656=t,c2657=t,c2658=t,c2659=t,c2660=t,c2661=t," - "c2662=t,c2663=t,c2664=t,c2665=t,c2666=t,c2667=t,c2668=t,c2669=t,c2670=t,c2671=t,c2672=t,c2673=t,c2674=t,c2675=t," - "c2676=t,c2677=t,c2678=t,c2679=t,c2680=t,c2681=t,c2682=t,c2683=t,c2684=t,c2685=t,c2686=t,c2687=t,c2688=t,c2689=t," - "c2690=t,c2691=t,c2692=t,c2693=t,c2694=t,c2695=t,c2696=t,c2697=t,c2698=t,c2699=t,c2700=t,c2701=t,c2702=t,c2703=t," - "c2704=t,c2705=t,c2706=t,c2707=t,c2708=t,c2709=t,c2710=t,c2711=t,c2712=t,c2713=t,c2714=t,c2715=t,c2716=t,c2717=t," - "c2718=t,c2719=t,c2720=t,c2721=t,c2722=t,c2723=t,c2724=t," - "c2725=t,c2726=t,c2727=t,c2728=t,c2729=t,c2730=t,c2731=t,c2732=t,c2733=t,c2734=t,c2735=t,c2736=t,c2737=t,c2738=t," - "c2739=t,c2740=t,c2741=t,c2742=t,c2743=t,c2744=t,c2745=t,c2746=t,c2747=t,c2748=t,c2749=t,c2750=t,c2751=t,c2752=t," - "c2753=t,c2754=t,c2755=t,c2756=t,c2757=t,c2758=t,c2759=t,c2760=t,c2761=t,c2762=t,c2763=t,c2764=t,c2765=t,c2766=t," - "c2767=t,c2768=t,c2769=t,c2770=t,c2771=t,c2772=t,c2773=t,c2774=t,c2775=t,c2776=t,c2777=t,c2778=t,c2779=t,c2780=t," - "c2781=t,c2782=t,c2783=t,c2784=t,c2785=t,c2786=t,c2787=t,c2788=t,c2789=t,c2790=t,c2791=t,c2792=t,c2793=t,c2794=t," - "c2795=t,c2796=t,c2797=t,c2798=t,c2799=t,c2800=t,c2801=t,c2802=t,c2803=t,c2804=t,c2805=t,c2806=t,c2807=t,c2808=t," - "c2809=t,c2810=t,c2811=t,c2812=t,c2813=t,c2814=t,c2815=t,c2816=t,c2817=t,c2818=t,c2819=t,c2820=t,c2821=t,c2822=t," - "c2823=t,c2824=t,c2825=t,c2826=t,c2827=t,c2828=t,c2829=t,c2830=t,c2831=t,c2832=t,c2833=t,c2834=t,c2835=t,c2836=t," - "c2837=t,c2838=t,c2839=t,c2840=t,c2841=t,c2842=t,c2843=t,c2844=t,c2845=t,c2846=t,c2847=t,c2848=t,c2849=t,c2850=t," - "c2851=t,c2852=t,c2853=t,c2854=t,c2855=t,c2856=t,c2857=t,c2858=t,c2859=t,c2860=t,c2861=t,c2862=t,c2863=t,c2864=t," - "c2865=t,c2866=t,c2867=t,c2868=t,c2869=t,c2870=t,c2871=t," - "c2872=t,c2873=t,c2874=t,c2875=t,c2876=t,c2877=t,c2878=t,c2879=t,c2880=t,c2881=t,c2882=t,c2883=t,c2884=t,c2885=t," - "c2886=t,c2887=t,c2888=t,c2889=t,c2890=t,c2891=t,c2892=t,c2893=t,c2894=t,c2895=t,c2896=t,c2897=t,c2898=t,c2899=t," - "c2900=t,c2901=t,c2902=t,c2903=t,c2904=t,c2905=t,c2906=t,c2907=t,c2908=t,c2909=t,c2910=t,c2911=t,c2912=t,c2913=t," - "c2914=t,c2915=t,c2916=t,c2917=t,c2918=t,c2919=t,c2920=t,c2921=t,c2922=t,c2923=t,c2924=t,c2925=t,c2926=t,c2927=t," - "c2928=t,c2929=t,c2930=t,c2931=t,c2932=t,c2933=t,c2934=t,c2935=t,c2936=t,c2937=t,c2938=t,c2939=t,c2940=t,c2941=t," - "c2942=t,c2943=t,c2944=t,c2945=t,c2946=t,c2947=t,c2948=t,c2949=t,c2950=t,c2951=t,c2952=t,c2953=t,c2954=t,c2955=t," - "c2956=t,c2957=t,c2958=t,c2959=t,c2960=t,c2961=t,c2962=t,c2963=t,c2964=t,c2965=t,c2966=t,c2967=t,c2968=t,c2969=t," - "c2970=t,c2971=t,c2972=t,c2973=t,c2974=t,c2975=t,c2976=t,c2977=t,c2978=t,c2979=t,c2980=t,c2981=t,c2982=t,c2983=t," - "c2984=t,c2985=t,c2986=t,c2987=t,c2988=t,c2989=t,c2990=t,c2991=t,c2992=t,c2993=t,c2994=t,c2995=t,c2996=t,c2997=t," - "c2998=t,c2999=t,c3000=t,c3001=t,c3002=t,c3003=t,c3004=t,c3005=t,c3006=t,c3007=t,c3008=t,c3009=t,c3010=t,c3011=t," - "c3012=t,c3013=t,c3014=t,c3015=t,c3016=t,c3017=t,c3018=t," - "c3019=t,c3020=t,c3021=t,c3022=t,c3023=t,c3024=t,c3025=t,c3026=t,c3027=t,c3028=t,c3029=t,c3030=t,c3031=t,c3032=t," - "c3033=t,c3034=t,c3035=t,c3036=t,c3037=t,c3038=t,c3039=t,c3040=t,c3041=t,c3042=t,c3043=t,c3044=t,c3045=t,c3046=t," - "c3047=t,c3048=t,c3049=t,c3050=t,c3051=t,c3052=t,c3053=t,c3054=t,c3055=t,c3056=t,c3057=t,c3058=t,c3059=t,c3060=t," - "c3061=t,c3062=t,c3063=t,c3064=t,c3065=t,c3066=t,c3067=t,c3068=t,c3069=t,c3070=t,c3071=t,c3072=t,c3073=t,c3074=t," - "c3075=t,c3076=t,c3077=t,c3078=t,c3079=t,c3080=t,c3081=t,c3082=t,c3083=t,c3084=t,c3085=t,c3086=t,c3087=t,c3088=t," - "c3089=t,c3090=t,c3091=t,c3092=t,c3093=t,c3094=t,c3095=t,c3096=t,c3097=t,c3098=t,c3099=t,c3100=t,c3101=t,c3102=t," - "c3103=t,c3104=t,c3105=t,c3106=t,c3107=t,c3108=t,c3109=t,c3110=t,c3111=t,c3112=t,c3113=t,c3114=t,c3115=t,c3116=t," - "c3117=t,c3118=t,c3119=t,c3120=t,c3121=t,c3122=t,c3123=t,c3124=t,c3125=t,c3126=t,c3127=t,c3128=t,c3129=t,c3130=t," - "c3131=t,c3132=t,c3133=t,c3134=t,c3135=t,c3136=t,c3137=t,c3138=t,c3139=t,c3140=t,c3141=t,c3142=t,c3143=t,c3144=t," - "c3145=t,c3146=t,c3147=t,c3148=t,c3149=t,c3150=t,c3151=t,c3152=t,c3153=t,c3154=t,c3155=t,c3156=t,c3157=t,c3158=t," - "c3159=t,c3160=t,c3161=t,c3162=t,c3163=t,c3164=t,c3165=t," - "c3166=t,c3167=t,c3168=t,c3169=t,c3170=t,c3171=t,c3172=t,c3173=t,c3174=t,c3175=t,c3176=t,c3177=t,c3178=t,c3179=t," - "c3180=t,c3181=t,c3182=t,c3183=t,c3184=t,c3185=t,c3186=t,c3187=t,c3188=t,c3189=t,c3190=t,c3191=t,c3192=t,c3193=t," - "c3194=t,c3195=t,c3196=t,c3197=t,c3198=t,c3199=t,c3200=t,c3201=t,c3202=t,c3203=t,c3204=t,c3205=t,c3206=t,c3207=t," - "c3208=t,c3209=t,c3210=t,c3211=t,c3212=t,c3213=t,c3214=t,c3215=t,c3216=t,c3217=t,c3218=t,c3219=t,c3220=t,c3221=t," - "c3222=t,c3223=t,c3224=t,c3225=t,c3226=t,c3227=t,c3228=t,c3229=t,c3230=t,c3231=t,c3232=t,c3233=t,c3234=t,c3235=t," - "c3236=t,c3237=t,c3238=t,c3239=t,c3240=t,c3241=t,c3242=t,c3243=t,c3244=t,c3245=t,c3246=t,c3247=t,c3248=t,c3249=t," - "c3250=t,c3251=t,c3252=t,c3253=t,c3254=t,c3255=t,c3256=t,c3257=t,c3258=t,c3259=t,c3260=t,c3261=t,c3262=t,c3263=t," - "c3264=t,c3265=t,c3266=t,c3267=t,c3268=t,c3269=t,c3270=t,c3271=t,c3272=t,c3273=t,c3274=t,c3275=t,c3276=t,c3277=t," - "c3278=t,c3279=t,c3280=t,c3281=t,c3282=t,c3283=t,c3284=t,c3285=t,c3286=t,c3287=t,c3288=t,c3289=t,c3290=t,c3291=t," - "c3292=t,c3293=t,c3294=t,c3295=t,c3296=t,c3297=t,c3298=t,c3299=t,c3300=t,c3301=t,c3302=t,c3303=t,c3304=t,c3305=t," - "c3306=t,c3307=t,c3308=t,c3309=t,c3310=t,c3311=t,c3312=t," - "c3313=t,c3314=t,c3315=t,c3316=t,c3317=t,c3318=t,c3319=t,c3320=t,c3321=t,c3322=t,c3323=t,c3324=t,c3325=t,c3326=t," - "c3327=t,c3328=t,c3329=t,c3330=t,c3331=t,c3332=t,c3333=t,c3334=t,c3335=t,c3336=t,c3337=t,c3338=t,c3339=t,c3340=t," - "c3341=t,c3342=t,c3343=t,c3344=t,c3345=t,c3346=t,c3347=t,c3348=t,c3349=t,c3350=t,c3351=t,c3352=t,c3353=t,c3354=t," - "c3355=t,c3356=t,c3357=t,c3358=t,c3359=t,c3360=t,c3361=t,c3362=t,c3363=t,c3364=t,c3365=t,c3366=t,c3367=t,c3368=t," - "c3369=t,c3370=t,c3371=t,c3372=t,c3373=t,c3374=t,c3375=t,c3376=t,c3377=t,c3378=t,c3379=t,c3380=t,c3381=t,c3382=t," - "c3383=t,c3384=t,c3385=t,c3386=t,c3387=t,c3388=t,c3389=t,c3390=t,c3391=t,c3392=t,c3393=t,c3394=t,c3395=t,c3396=t," - "c3397=t,c3398=t,c3399=t,c3400=t,c3401=t,c3402=t,c3403=t,c3404=t,c3405=t,c3406=t,c3407=t,c3408=t,c3409=t,c3410=t," - "c3411=t,c3412=t,c3413=t,c3414=t,c3415=t,c3416=t,c3417=t,c3418=t,c3419=t,c3420=t,c3421=t,c3422=t,c3423=t,c3424=t," - "c3425=t,c3426=t,c3427=t,c3428=t,c3429=t,c3430=t,c3431=t,c3432=t,c3433=t,c3434=t,c3435=t,c3436=t,c3437=t,c3438=t," - "c3439=t,c3440=t,c3441=t,c3442=t,c3443=t,c3444=t,c3445=t,c3446=t,c3447=t,c3448=t,c3449=t,c3450=t,c3451=t,c3452=t," - "c3453=t,c3454=t,c3455=t,c3456=t,c3457=t,c3458=t,c3459=t," - "c3460=t,c3461=t,c3462=t,c3463=t,c3464=t,c3465=t,c3466=t,c3467=t,c3468=t,c3469=t,c3470=t,c3471=t,c3472=t,c3473=t," - "c3474=t,c3475=t,c3476=t,c3477=t,c3478=t,c3479=t,c3480=t,c3481=t,c3482=t,c3483=t,c3484=t,c3485=t,c3486=t,c3487=t," - "c3488=t,c3489=t,c3490=t,c3491=t,c3492=t,c3493=t,c3494=t,c3495=t,c3496=t,c3497=t,c3498=t,c3499=t,c3500=t,c3501=t," - "c3502=t,c3503=t,c3504=t,c3505=t,c3506=t,c3507=t,c3508=t,c3509=t,c3510=t,c3511=t,c3512=t,c3513=t," - "c3514=t,c3515=t,c3516=t,c3517=t,c3518=t,c3519=t,c3520=t,c3521=t,c3522=t,c3523=t,c3524=t,c3525=t,c3526=t,c3527=t," - "c3528=t,c3529=t,c3530=t,c3531=t,c3532=t,c3533=t,c3534=t,c3535=t,c3536=t,c3537=t,c3538=t,c3539=t,c3540=t,c3541=t," - "c3542=t,c3543=t,c3544=t,c3545=t,c3546=t,c3547=t,c3548=t,c3549=t,c3550=t,c3551=t,c3552=t,c3553=t,c3554=t,c3555=t," - "c3556=t,c3557=t,c3558=t,c3559=t,c3560=t,c3561=t,c3562=t,c3563=t,c3564=t,c3565=t,c3566=t,c3567=t,c3568=t,c3569=t," - "c3570=t,c3571=t,c3572=t,c3573=t,c3574=t,c3575=t,c3576=t,c3577=t,c3578=t,c3579=t,c3580=t,c3581=t,c3582=t,c3583=t," - "c3584=t,c3585=t,c3586=t,c3587=t,c3588=t,c3589=t,c3590=t,c3591=t,c3592=t,c3593=t,c3594=t,c3595=t,c3596=t,c3597=t," - "c3598=t,c3599=t,c3600=t,c3601=t,c3602=t,c3603=t,c3604=t,c3605=t,c3606=t,c3607=t,c3608=t,c3609=t,c3610=t,c3611=t," - "c3612=t,c3613=t,c3614=t,c3615=t,c3616=t,c3617=t,c3618=t,c3619=t,c3620=t,c3621=t,c3622=t,c3623=t,c3624=t,c3625=t," - "c3626=t,c3627=t,c3628=t,c3629=t,c3630=t,c3631=t,c3632=t,c3633=t,c3634=t,c3635=t,c3636=t,c3637=t,c3638=t,c3639=t," - "c3640=t,c3641=t,c3642=t,c3643=t,c3644=t,c3645=t,c3646=t,c3647=t,c3648=t,c3649=t,c3650=t,c3651=t,c3652=t,c3653=t," - "c3654=t,c3655=t,c3656=t,c3657=t,c3658=t,c3659=t,c3660=t," - "c3661=t,c3662=t,c3663=t,c3664=t,c3665=t,c3666=t,c3667=t,c3668=t,c3669=t,c3670=t,c3671=t,c3672=t,c3673=t,c3674=t," - "c3675=t,c3676=t,c3677=t,c3678=t,c3679=t,c3680=t,c3681=t,c3682=t,c3683=t,c3684=t,c3685=t,c3686=t,c3687=t,c3688=t," - "c3689=t,c3690=t,c3691=t,c3692=t,c3693=t,c3694=t,c3695=t,c3696=t,c3697=t,c3698=t,c3699=t,c3700=t,c3701=t,c3702=t," - "c3703=t,c3704=t,c3705=t,c3706=t,c3707=t,c3708=t,c3709=t,c3710=t,c3711=t,c3712=t,c3713=t,c3714=t,c3715=t,c3716=t," - "c3717=t,c3718=t,c3719=t,c3720=t,c3721=t,c3722=t,c3723=t,c3724=t,c3725=t,c3726=t,c3727=t,c3728=t,c3729=t,c3730=t," - "c3731=t,c3732=t,c3733=t,c3734=t,c3735=t,c3736=t,c3737=t,c3738=t,c3739=t,c3740=t,c3741=t,c3742=t,c3743=t,c3744=t," - "c3745=t,c3746=t,c3747=t,c3748=t,c3749=t,c3750=t,c3751=t,c3752=t,c3753=t,c3754=t,c3755=t,c3756=t,c3757=t,c3758=t," - "c3759=t,c3760=t,c3761=t,c3762=t,c3763=t,c3764=t,c3765=t,c3766=t,c3767=t,c3768=t,c3769=t,c3770=t,c3771=t,c3772=t," - "c3773=t,c3774=t,c3775=t,c3776=t,c3777=t,c3778=t,c3779=t,c3780=t,c3781=t,c3782=t,c3783=t,c3784=t,c3785=t,c3786=t," - "c3787=t,c3788=t,c3789=t,c3790=t,c3791=t,c3792=t,c3793=t,c3794=t,c3795=t,c3796=t,c3797=t,c3798=t,c3799=t,c3800=t," - "c3801=t,c3802=t,c3803=t,c3804=t,c3805=t,c3806=t,c3807=t," - "c3808=t,c3809=t,c3810=t,c3811=t,c3812=t,c3813=t,c3814=t,c3815=t,c3816=t,c3817=t,c3818=t,c3819=t,c3820=t,c3821=t," - "c3822=t,c3823=t,c3824=t,c3825=t,c3826=t,c3827=t,c3828=t,c3829=t,c3830=t,c3831=t,c3832=t,c3833=t,c3834=t,c3835=t," - "c3836=t,c3837=t,c3838=t,c3839=t,c3840=t,c3841=t,c3842=t,c3843=t,c3844=t,c3845=t,c3846=t,c3847=t,c3848=t,c3849=t," - "c3850=t,c3851=t,c3852=t,c3853=t,c3854=t,c3855=t,c3856=t,c3857=t,c3858=t,c3859=t,c3860=t,c3861=t,c3862=t,c3863=t," - "c3864=t,c3865=t,c3866=t,c3867=t,c3868=t,c3869=t,c3870=t,c3871=t,c3872=t,c3873=t,c3874=t,c3875=t,c3876=t,c3877=t," - "c3878=t,c3879=t,c3880=t,c3881=t,c3882=t,c3883=t,c3884=t,c3885=t,c3886=t,c3887=t,c3888=t,c3889=t,c3890=t,c3891=t," - "c3892=t,c3893=t,c3894=t,c3895=t,c3896=t,c3897=t,c3898=t,c3899=t,c3900=t,c3901=t,c3902=t,c3903=t,c3904=t,c3905=t," - "c3906=t,c3907=t,c3908=t,c3909=t,c3910=t,c3911=t,c3912=t,c3913=t,c3914=t,c3915=t,c3916=t,c3917=t,c3918=t,c3919=t," - "c3920=t,c3921=t,c3922=t,c3923=t,c3924=t,c3925=t,c3926=t,c3927=t,c3928=t,c3929=t,c3930=t,c3931=t,c3932=t,c3933=t," - "c3934=t,c3935=t,c3936=t,c3937=t,c3938=t,c3939=t,c3940=t,c3941=t,c3942=t,c3943=t,c3944=t,c3945=t,c3946=t,c3947=t," - "c3948=t,c3949=t,c3950=t,c3951=t,c3952=t,c3953=t,c3954=t," - "c3955=t,c3956=t,c3957=t,c3958=t,c3959=t,c3960=t,c3961=t,c3962=t,c3963=t,c3964=t,c3965=t,c3966=t,c3967=t,c3968=t," - "c3969=t,c3970=t,c3971=t,c3972=t,c3973=t,c3974=t,c3975=t,c3976=t,c3977=t,c3978=t,c3979=t,c3980=t,c3981=t,c3982=t," - "c3983=t,c3984=t,c3985=t,c3986=t,c3987=t,c3988=t,c3989=t,c3990=t,c3991=t,c3992=t,c3993=t,c3994=t,c3995=t,c3996=t," - "c3997=t,c3998=t,c3999=t,c4000=t,c4001=t,c4002=t,c4003=t,c4004=t,c4005=t,c4006=t,c4007=t,c4008=t,c4009=t,c4010=t," - "c4011=t,c4012=t,c4013=t,c4014=t,c4015=t,c4016=t,c4017=t,c4018=t,c4019=t,c4020=t,c4021=t,c4022=t,c4023=t,c4024=t," - "c4025=t,c4026=t,c4027=t,c4028=t,c4029=t,c4030=t,c4031=t,c4032=t,c4033=t,c4034=t,c4035=t,c4036=t,c4037=t,c4038=t," - "c4039=t,c4040=t,c4041=t,c4042=t,c4043=t,c4044=t,c4045=t,c4046=t,c4047=t,c4048=t,c4049=t,c4050=t,c4051=t,c4052=t," - "c4053=t,c4054=t,c4055=t,c4056=t,c4057=t,c4058=t,c4059=t,c4060=t,c4061=t,c4062=t,c4063=t,c4064=t,c4065=t,c4066=t," - "c4067=t,c4068=t,c4069=t,c4070=t,c4071=t,c4072=t,c4073=t,c4074=t,c4075=t,c4076=t,c4077=t,c4078=t,c4079=t,c4080=t," - "c4081=t,c4082=t,c4083=t,c4084=t,c4085=t,c4086=t,c4087=t,c4088=t,c4089=t,c4090=t,c4091=t,c4092=t,c4093=t " - "1626006833640000000"}; - - int ret = TSDB_CODE_SUCCESS; - for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseInfluxLine(info, sql[i], strlen(sql[i])); - if (ret != TSDB_CODE_SUCCESS) break; +//TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { +// SSmlHandle *info = smlBuildSmlInfo(NULL); +// info->protocol = TSDB_SML_JSON_PROTOCOL; +// ASSERT_NE(info, nullptr); +// +// const char *sql[] = { +// "[{\"metric\":\"sys.cpu.nice\",\"timestamp\": 1346846400,\"value\": 18,\"tags\": {\"host\": \"lga\"}},{\"metric\": \"sys.sdfa\",\"timestamp\": 1346846400,\"value\": \"18\",\"tags\": {\"host\": 8932}},]", +// }; +// for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { +// char *dataPointStart = (char *)sql[i]; +// int8_t offset[4] = {0}; +// while (1) { +// SSmlLineInfo elements = {0}; +// if(offset[0] == 0){ +// smlJsonParseObjFirst(&dataPointStart, &elements, offset); +// }else{ +// smlJsonParseObj(&dataPointStart, &elements, offset); +// } +// if(*dataPointStart == '\0') break; +// +// SArray *tags = smlJsonParseTags(elements.tags, elements.tags + elements.tagsLen); +// size_t num = taosArrayGetSize(tags); +// ASSERT_EQ(num, 1); +// +// taosArrayDestroy(tags); +// } +// } +// smlDestroyInfo(info); +//} + +TEST(testCase, smlParseNumber_performance_Test) { + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + SSmlKv kv; + + char* str[3] = {"2893f64", "2323u32", "93u8"}; + for (int i = 0; i < 3; ++i) { + int64_t t1 = taosGetTimestampUs(); + for (int j = 0; j < 10000000; ++j) { + kv.value = str[i]; + kv.length = strlen(str[i]); + smlParseNumber(&kv, &msgBuf); + } + printf("smlParseNumber:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t1); + printf("\n"); + int64_t t2 = taosGetTimestampUs(); + for (int j = 0; j < 10000000; ++j) { + kv.value = str[i]; + kv.length = strlen(str[i]); + smlParseNumberOld(&kv, &msgBuf); + } + printf("smlParseNumberOld:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t2); + printf("\n\n"); } - ASSERT_NE(ret, 0); - smlDestroyInfo(info); -} +} \ No newline at end of file diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 43f272d599cdfdd65b6bbdd59f26523f811244bc..9b5b32cf695ff6d38ff09fe9eb778dbb2679a43c 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -19,7 +19,7 @@ #include "tlog.h" #include "tname.h" -#define MALLOC_ALIGN_BYTES 256 +#define MALLOC_ALIGN_BYTES 32 int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { @@ -37,7 +37,8 @@ int32_t colDataGetFullLength(const SColumnInfoData* pColumnInfoData, int32_t num if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { return pColumnInfoData->varmeta.length + sizeof(int32_t) * numOfRows; } else { - return ((pColumnInfoData->info.type == TSDB_DATA_TYPE_NULL) ? 0 : pColumnInfoData->info.bytes * numOfRows) + BitmapLen(numOfRows); + return ((pColumnInfoData->info.type == TSDB_DATA_TYPE_NULL) ? 0 : pColumnInfoData->info.bytes * numOfRows) + + BitmapLen(numOfRows); } } @@ -277,7 +278,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int pColumnInfoData->varmeta.allocLen = len + oldLen; } - if (pColumnInfoData->pData && pSource->pData) { // TD-20382 + if (pColumnInfoData->pData && pSource->pData) { // TD-20382 memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len); } pColumnInfoData->varmeta.length = len + oldLen; @@ -314,8 +315,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, const SDataBlockInfo* pBlockInfo) { - if (pColumnInfoData->info.type != pSource->info.type || - (pBlockInfo != NULL && pBlockInfo->capacity < numOfRows)) { + if (pColumnInfoData->info.type != pSource->info.type || (pBlockInfo != NULL && pBlockInfo->capacity < numOfRows)) { return TSDB_CODE_FAILED; } @@ -356,14 +356,14 @@ size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { return taosArrayGetSiz size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; } int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex) { - if (pDataBlock->info.rows > 0) { -// ASSERT(pDataBlock->info.dataLoad == 1); - } - if (pDataBlock == NULL || pDataBlock->info.rows <= 0 || pDataBlock->info.dataLoad == 0) { return 0; } + if (pDataBlock->info.rows > 0) { + // ASSERT(pDataBlock->info.dataLoad == 1); + } + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); if (numOfCols <= 0) { return -1; @@ -1166,7 +1166,8 @@ void blockDataEmpty(SSDataBlock* pDataBlock) { * the all NULL value in this column. It is an internal representation of all NULL value column, and no visible to * any users. The length of TSDB_DATA_TYPE_NULL is 0, and it is an special case. */ -static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows, bool clearPayload) { +static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows, + bool clearPayload) { if (numOfRows <= 0 || numOfRows <= pBlockInfo->capacity) { return TSDB_CODE_SUCCESS; } @@ -1225,7 +1226,7 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* return TSDB_CODE_SUCCESS; } -void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) { +void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) { pColumn->hasNull = false; if (IS_VAR_DATA_TYPE(pColumn->info.type)) { @@ -1274,7 +1275,6 @@ void blockDataFreeRes(SSDataBlock* pBlock) { taosArrayDestroy(pBlock->pDataBlock); pBlock->pDataBlock = NULL; taosMemoryFreeClear(pBlock->pBlockAgg); - taosMemoryFree(pBlock->info.pTag); memset(&pBlock->info, 0, sizeof(SDataBlockInfo)); } @@ -1850,7 +1850,7 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) { char pBuf[128] = {0}; int32_t sz = taosArrayGetSize(dataBlocks); for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGetP(dataBlocks, i); + SSDataBlock* pDataBlock = taosArrayGet(dataBlocks, i); size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); int32_t rows = pDataBlock->info.rows; @@ -1862,21 +1862,37 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) { for (int32_t k = 0; k < numOfCols; k++) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + if (k == 0) { + printf("cols:%d |", (int32_t)numOfCols); + } if (colDataIsNull(pColInfoData, rows, j, NULL)) { printf(" %15s |", "NULL"); continue; } + switch (pColInfoData->info.type) { case TSDB_DATA_TYPE_TIMESTAMP: formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); printf(" %25s |", pBuf); break; case TSDB_DATA_TYPE_BOOL: - printf(" %15d |", *(int32_t*)var); + printf(" %15" PRIi8 " |", *(int8_t*)var); + break; + case TSDB_DATA_TYPE_TINYINT: + printf(" %15" PRIi8 " |", *(int8_t*)var); + break; + case TSDB_DATA_TYPE_SMALLINT: + printf(" %15" PRIi16 " |", *(int16_t*)var); break; case TSDB_DATA_TYPE_INT: printf(" %15d |", *(int32_t*)var); break; + case TSDB_DATA_TYPE_UTINYINT: + printf(" %15" PRIu8 " |", *(uint8_t*)var); + break; + case TSDB_DATA_TYPE_USMALLINT: + printf(" %15" PRIu16 " |", *(uint16_t*)var); + break; case TSDB_DATA_TYPE_UINT: printf(" %15u |", *(uint32_t*)var); break; @@ -1926,9 +1942,10 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) int32_t len = 0; len += snprintf(dumpBuf + len, size - len, "===stream===%s|block type %d|child id %d|group id:%" PRIu64 "|uid:%" PRId64 - "|rows:%d|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "\n", + "|rows:%d|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "|tbl:%s\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.id.groupId, - pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version, pDataBlock->info.calWin.skey, pDataBlock->info.calWin.ekey); + pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version, + pDataBlock->info.calWin.skey, pDataBlock->info.calWin.ekey, pDataBlock->info.parTbName); if (len >= size - 1) return dumpBuf; for (int32_t j = 0; j < rows; j++) { @@ -2030,6 +2047,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) * @param suid * */ +#if 0 int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataBlock, STSchema* pTSchema, int32_t vgId, tb_uid_t suid) { int32_t bufSize = sizeof(SSubmitReq); @@ -2193,35 +2211,189 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB return TSDB_CODE_SUCCESS; } +#endif + +int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDataBlock, const STSchema* pTSchema, + int64_t uid, int32_t vgId, tb_uid_t suid) { + SSubmitReq2* pReq = *ppReq; + SArray* pVals = NULL; + int32_t numOfBlks = 0; + int32_t sz = 1; + + terrno = TSDB_CODE_SUCCESS; + + if (NULL == pReq) { + if (!(pReq = taosMemoryMalloc(sizeof(SSubmitReq2)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { + goto _end; + } + } + + for (int32_t i = 0; i < sz; ++i) { + int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); + int32_t rows = pDataBlock->info.rows; + + if (colNum <= 1) { // invalid if only with TS col + continue; + } + + // the rsma result should has the same column number with schema. + ASSERT(colNum == pTSchema->numOfCols); + + SSubmitTbData tbData = {0}; + + if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow*)))) { + goto _end; + } + tbData.suid = suid; + tbData.uid = uid; + tbData.sver = pTSchema->version; + + if (!pVals && !(pVals = taosArrayInit(colNum, sizeof(SColVal)))) { + taosArrayDestroy(tbData.aRowP); + goto _end; + } + + for (int32_t j = 0; j < rows; ++j) { // iterate by row + + taosArrayClear(pVals); + + bool isStartKey = false; + int32_t offset = 0; + for (int32_t k = 0; k < colNum; ++k) { // iterate by column + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + const STColumn* pCol = &pTSchema->columns[k]; + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + ASSERT(pColInfoData->info.type == pCol->type); + if (!isStartKey) { + isStartKey = true; + ASSERT(PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId); + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, (SValue){.val = *(TSKEY*)var}); + taosArrayPush(pVals, &cv); + } else if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, (SValue){.val = *(int64_t*)var}); + taosArrayPush(pVals, &cv); + } + break; + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY + ASSERT(pColInfoData->info.type == pCol->type); + if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + void* data = colDataGetVarData(pColInfoData, j); + SValue sv = (SValue){.nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } + break; + } + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_MEDIUMBLOB: + uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); + ASSERT(0); + break; + default: + if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { + if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); // should use pCol->type + taosArrayPush(pVals, &cv); + } else { + SValue sv; + if (pCol->type == pColInfoData->info.type) { + memcpy(&sv.val, var, tDataTypes[pCol->type].bytes); + } else { + /** + * 1. sum/avg would convert to int64_t/uint64_t/double during aggregation + * 2. below conversion may lead to overflow or loss, the app should select the right data type. + */ + char tv[8] = {0}; + if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) { + float v = 0; + GET_TYPED_DATA(v, float, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) { + double v = 0; + GET_TYPED_DATA(v, double, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) { + int64_t v = 0; + GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else { + uint64_t v = 0; + GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } + memcpy(&sv.val, tv, tDataTypes[pCol->type].bytes); + } + SColVal cv = COL_VAL_VALUE(pCol->colId, pColInfoData->info.type, sv); + taosArrayPush(pVals, &cv); + } + } else { + uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); + ASSERT(0); + } + break; + } + } + SRow* pRow = NULL; + if ((terrno = tRowBuild(pVals, pTSchema, &pRow)) < 0) { + tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + goto _end; + } + ASSERT(pRow); + taosArrayPush(tbData.aRowP, &pRow); + } + + taosArrayPush(pReq->aSubmitTbData, &tbData); + } +_end: + taosArrayDestroy(pVals); + if (terrno != 0) { + *ppReq = NULL; + if (pReq) tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + return TSDB_CODE_FAILED; + } + *ppReq = pReq; + return TSDB_CODE_SUCCESS; +} char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { if (stbFullName[0] == 0) { return NULL; } - SArray* tags = taosArrayInit(0, sizeof(void*)); + SArray* tags = taosArrayInit(0, sizeof(SSmlKv)); if (tags == NULL) { return NULL; } - SSmlKv* pTag = taosMemoryCalloc(1, sizeof(SSmlKv)); - if (pTag == NULL) { - taosArrayDestroy(tags); - return NULL; - } - void* cname = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1); if (cname == NULL) { taosArrayDestroy(tags); - taosMemoryFree(pTag); return NULL; } - pTag->key = "group_id"; - pTag->keyLen = strlen(pTag->key); - pTag->type = TSDB_DATA_TYPE_UBIGINT; - pTag->u = groupId; - pTag->length = sizeof(uint64_t); + SSmlKv pTag = {.key = "group_id", + .keyLen = sizeof("group_id") - 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .u = groupId, + .length = sizeof(uint64_t)}; taosArrayPush(tags, &pTag); RandTableName rname = { @@ -2233,7 +2405,6 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { buildChildTableName(&rname); - taosMemoryFree(pTag); taosArrayDestroy(tags); if ((rname.ctbShortName && rname.ctbShortName[0]) == 0) { diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index fc49c033794c946a9881b6ae1efeabcb101e99a7..5e001a96872306d9369d388f8823a2baecdbf04f 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -20,6 +20,8 @@ #include "tdatablock.h" #include "tlog.h" +static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData); + // SBuffer ================================ void tBufferDestroy(SBuffer *pBuffer) { tFree(pBuffer->pBuf); @@ -61,9 +63,9 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); #define KV_FLG_MID ((uint8_t)0x20) #define KV_FLG_BIG ((uint8_t)0x30) -#define ROW_BIT_NONE ((uint8_t)0x0) -#define ROW_BIT_NULL ((uint8_t)0x1) -#define ROW_BIT_VALUE ((uint8_t)0x2) +#define BIT_FLG_NONE ((uint8_t)0x0) +#define BIT_FLG_NULL ((uint8_t)0x1) +#define BIT_FLG_VALUE ((uint8_t)0x2) #pragma pack(push, 1) typedef struct { @@ -95,28 +97,29 @@ typedef struct { } \ } while (0) -int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { +int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) { int32_t code = 0; - ASSERT(taosArrayGetSize(aColVal) > 0); + ASSERT(TARRAY_SIZE(aColVal) > 0); ASSERT(((SColVal *)aColVal->pData)[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); ASSERT(((SColVal *)aColVal->pData)[0].type == TSDB_DATA_TYPE_TIMESTAMP); // scan --------------- - uint8_t flag = 0; - int32_t iColVal = 1; - const int32_t nColVal = taosArrayGetSize(aColVal); - SColVal *pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - int32_t iTColumn = 1; - STColumn *pTColumn = pTSchema->columns + iTColumn; - int32_t ntp = 0; - int32_t nkv = 0; - int32_t maxIdx = 0; - int32_t nIdx = 0; + SRow *pRow = NULL; + SColVal *colVals = (SColVal *)TARRAY_DATA(aColVal); + uint8_t flag = 0; + int32_t iColVal = 1; + const int32_t nColVal = TARRAY_SIZE(aColVal); + SColVal *pColVal = (iColVal < nColVal) ? &colVals[iColVal] : NULL; + int32_t iTColumn = 1; + const STColumn *pTColumn = pTSchema->columns + iTColumn; + int32_t ntp = 0; + int32_t nkv = 0; + int32_t maxIdx = 0; + int32_t nIdx = 0; while (pTColumn) { if (pColVal) { if (pColVal->cid == pTColumn->colId) { - ntp += TYPE_BYTES[pTColumn->type]; if (COL_VAL_IS_VALUE(pColVal)) { // VALUE flag |= HAS_VALUE; maxIdx = nkv; @@ -139,17 +142,15 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } else if (pColVal->cid > pTColumn->colId) { // NONE flag |= HAS_NONE; - ntp += TYPE_BYTES[pTColumn->type]; pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } } else { // NONE flag |= HAS_NONE; - ntp += TYPE_BYTES[pTColumn->type]; pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } } @@ -161,17 +162,17 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { ntp = sizeof(SRow); break; case HAS_VALUE: - ntp = sizeof(SRow) + ntp; + ntp = sizeof(SRow) + pTSchema->flen + ntp; break; case (HAS_NULL | HAS_NONE): ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1); break; case (HAS_VALUE | HAS_NONE): case (HAS_VALUE | HAS_NULL): - ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1) + ntp; + ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntp; break; case (HAS_VALUE | HAS_NULL | HAS_NONE): - ntp = sizeof(SRow) + BIT2_SIZE(pTSchema->numOfCols - 1) + ntp; + ntp = sizeof(SRow) + BIT2_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntp; break; default: ASSERT(0); @@ -196,12 +197,14 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } // alloc -------------- - SRow *pRow = NULL; - code = tBufferReserve(pBuffer, nRow, (void **)&pRow); - if (code) return code; + pRow = taosMemoryMalloc(nRow); + if (NULL == pRow) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } // build -------------- - pColVal = (SColVal *)taosArrayGet(aColVal, 0); + pColVal = &colVals[0]; pRow->flag = flag; pRow->rsv = 0; @@ -214,10 +217,10 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } iColVal = 1; - pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (iColVal < nColVal) ? &colVals[iColVal] : NULL; iTColumn = 1; pTColumn = pTSchema->columns + iTColumn; - if (flag & 0xf0) { // KV + if (flag >> 4) { // KV SKVIdx *pIdx = (SKVIdx *)pRow->data; int32_t iIdx = 0; int32_t nv = 0; @@ -266,11 +269,11 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } else if (pColVal->cid > pTColumn->colId) { // NONE pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } } else { // NONE pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; @@ -306,12 +309,20 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { break; } + if (pb) { + if (flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) { + memset(pb, 0, BIT2_SIZE(pTSchema->numOfCols - 1)); + } else { + memset(pb, 0, BIT1_SIZE(pTSchema->numOfCols - 1)); + } + } + // build impl while (pTColumn) { if (pColVal) { if (pColVal->cid == pTColumn->colId) { if (COL_VAL_IS_VALUE(pColVal)) { // VALUE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_VALUE); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_VALUE); if (IS_VAR_DATA_TYPE(pTColumn->type)) { *(int32_t *)(pf + pTColumn->offset) = nv; @@ -324,24 +335,24 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { memcpy(pf + pTColumn->offset, &pColVal->value.val, TYPE_BYTES[pTColumn->type]); } } else if (COL_VAL_IS_NONE(pColVal)) { // NONE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NONE); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); } else { // NULL - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NULL); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NULL); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); } pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } else if (pColVal->cid > pTColumn->colId) { // NONE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NONE); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } } else { // NONE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NONE); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } @@ -349,6 +360,12 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } _exit: + if (code) { + *ppRow = NULL; + tRowDestroy(pRow); + } else { + *ppRow = pRow; + } return code; } @@ -376,7 +393,7 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { return; } - if (pRow->flag & 0xf0) { // KV Row + if (pRow->flag >> 4) { // KV Row SKVIdx *pIdx = (SKVIdx *)pRow->data; uint8_t *pv = NULL; if (pRow->flag & KV_FLG_LIT) { @@ -432,64 +449,190 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); } else { // Tuple Row - uint8_t *pf = NULL; - uint8_t *pv = NULL; - uint8_t bv = ROW_BIT_VALUE; + if (pRow->flag == HAS_VALUE) { + pColVal->cid = pTColumn->colId; + pColVal->type = pTColumn->type; + pColVal->flag = CV_FLAG_VALUE; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + uint8_t *pData = pRow->data + pTSchema->flen + *(int32_t *)(pRow->data + pTColumn->offset); + pData += tGetU32v(pData, &pColVal->value.nData); + if (pColVal->value.nData) { + pColVal->value.pData = pData; + } else { + pColVal->value.pData = NULL; + } + } else { + memcpy(&pColVal->value.val, pRow->data + pTColumn->offset, TYPE_BYTES[pTColumn->type]); + } + } else { + uint8_t *pf; + uint8_t *pv; + uint8_t bv = BIT_FLG_VALUE; - switch (pRow->flag) { - case HAS_VALUE: - pf = pRow->data; - pv = pf + pTSchema->flen; - break; - case (HAS_NULL | HAS_NONE): - bv = GET_BIT1(pRow->data, iCol - 1); - break; - case (HAS_VALUE | HAS_NONE): - bv = GET_BIT1(pRow->data, iCol - 1); - if (bv) bv++; - pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; - break; - case (HAS_VALUE | HAS_NULL): - bv = GET_BIT1(pRow->data, iCol - 1); - bv++; - pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; - break; - case (HAS_VALUE | HAS_NULL | HAS_NONE): - bv = GET_BIT2(pRow->data, iCol - 1); - pf = pRow->data + BIT2_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; - break; - default: - break; - } + switch (pRow->flag) { + case (HAS_NULL | HAS_NONE): + bv = GET_BIT1(pRow->data, iCol - 1); + break; + case (HAS_VALUE | HAS_NONE): + bv = GET_BIT1(pRow->data, iCol - 1); + if (bv) bv++; + pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL): + bv = GET_BIT1(pRow->data, iCol - 1); + bv++; + pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + bv = GET_BIT2(pRow->data, iCol - 1); + pf = pRow->data + BIT2_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + default: + ASSERT(0); + break; + } - if (bv == ROW_BIT_NONE) { - *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); - return; - } else if (bv == ROW_BIT_NULL) { - *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); - return; - } + if (bv == BIT_FLG_NONE) { + *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + return; + } else if (bv == BIT_FLG_NULL) { + *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); + return; + } - pColVal->cid = pTColumn->colId; - pColVal->type = pTColumn->type; - pColVal->flag = CV_FLAG_VALUE; - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset); - pData += tGetU32v(pData, &pColVal->value.nData); - if (pColVal->value.nData) { - pColVal->value.pData = pData; + pColVal->cid = pTColumn->colId; + pColVal->type = pTColumn->type; + pColVal->flag = CV_FLAG_VALUE; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset); + pData += tGetU32v(pData, &pColVal->value.nData); + if (pColVal->value.nData) { + pColVal->value.pData = pData; + } else { + pColVal->value.pData = NULL; + } } else { - pColVal->value.pData = NULL; + memcpy(&pColVal->value.val, pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]); } - } else { - memcpy(&pColVal->value.val, pv + pTColumn->offset, pTColumn->bytes); } } } +void tRowDestroy(SRow *pRow) { + if (pRow) taosMemoryFree(pRow); +} + +static int32_t tRowPCmprFn(const void *p1, const void *p2) { + if ((*(SRow **)p1)->ts < (*(SRow **)p2)->ts) { + return -1; + } else if ((*(SRow **)p1)->ts > (*(SRow **)p2)->ts) { + return 1; + } + + return 0; +} +static void tRowPDestroy(SRow **ppRow) { tRowDestroy(*ppRow); } +static int32_t tRowMergeImpl(SArray *aRowP, STSchema *pTSchema, int32_t iStart, int32_t iEnd, int8_t flag) { + int32_t code = 0; + + int32_t nRow = iEnd - iStart; + SRowIter **aIter = NULL; + SArray *aColVal = NULL; + SRow *pRow = NULL; + + aIter = taosMemoryCalloc(nRow, sizeof(SRowIter *)); + if (aIter == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t i = 0; i < nRow; i++) { + SRow *pRowT = taosArrayGetP(aRowP, iStart + i); + + code = tRowIterOpen(pRowT, pTSchema, &aIter[i]); + if (code) goto _exit; + } + + // merge + aColVal = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)); + if (aColVal == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t iCol = 0; iCol < pTSchema->numOfCols; iCol++) { + SColVal *pColVal = NULL; + for (int32_t iRow = 0; iRow < nRow; iRow++) { + SColVal *pColValT = tRowIterNext(aIter[iRow]); + + // todo: take strategy according to the flag + if (COL_VAL_IS_VALUE(pColValT)) { + pColVal = pColValT; + } else if (COL_VAL_IS_NULL(pColValT)) { + if (pColVal == NULL) { + pColVal = pColValT; + } + } + } + + if (pColVal) taosArrayPush(aColVal, pColVal); + } + + // build + code = tRowBuild(aColVal, pTSchema, &pRow); + if (code) goto _exit; + + taosArrayRemoveBatch(aRowP, iStart, nRow, (FDelete)tRowPDestroy); + taosArrayInsert(aRowP, iStart, &pRow); + +_exit: + if (aIter) { + for (int32_t i = 0; i < nRow; i++) { + tRowIterClose(&aIter[i]); + } + taosMemoryFree(aIter); + } + if (aColVal) taosArrayDestroy(aColVal); + if (code) tRowDestroy(pRow); + return code; +} + +void tRowSort(SArray *aRowP) { + if (TARRAY_SIZE(aRowP) <= 1) return; + taosArraySort(aRowP, tRowPCmprFn); +} + +int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag) { + int32_t code = 0; + + int32_t iStart = 0; + while (iStart < aRowP->size) { + SRow *pRow = (SRow *)taosArrayGetP(aRowP, iStart); + + int32_t iEnd = iStart + 1; + while (iEnd < aRowP->size) { + SRow *pRowT = (SRow *)taosArrayGetP(aRowP, iEnd); + + if (pRow->ts != pRowT->ts) break; + + iEnd++; + } + + if (iEnd - iStart > 1) { + code = tRowMergeImpl(aRowP, pTSchema, iStart, iEnd, flag); + if (code) return code; + } + + // the array is also changing, so the iStart just ++ instead of iEnd + iStart++; + } + + return code; +} + // SRowIter ======================================== struct SRowIter { SRow *pRow; @@ -505,10 +648,9 @@ struct SRowIter { uint8_t *pb; uint8_t *pf; }; - uint8_t *pv; }; - - SColVal cv; + uint8_t *pv; + SColVal cv; }; int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { @@ -528,15 +670,15 @@ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { if (pRow->flag == HAS_NONE || pRow->flag == HAS_NULL) goto _exit; - if (pRow->flag & 0xf0) { + if (pRow->flag >> 4) { pIter->iCol = 0; pIter->pIdx = (SKVIdx *)pRow->data; if (pRow->flag & KV_FLG_LIT) { pIter->pv = pIter->pIdx->idx + pIter->pIdx->nCol; } else if (pRow->flag & KV_FLG_MID) { - pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 1); + pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 1); // * sizeof(uint16_t) } else { - pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 2); + pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 2); // * sizeof(uint32_t) } } else { switch (pRow->flag) { @@ -608,7 +750,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { goto _exit; } - if (pIter->pRow->flag & 0xf0) { // KV + if (pIter->pRow->flag >> 4) { // KV if (pIter->iCol < pIter->pIdx->nCol) { uint8_t *pData; @@ -656,7 +798,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { goto _exit; } } else { // Tuple - uint8_t bv = ROW_BIT_VALUE; + uint8_t bv = BIT_FLG_VALUE; if (pIter->pb) { switch (pIter->pRow->flag) { case (HAS_NULL | HAS_NONE): @@ -677,10 +819,10 @@ SColVal *tRowIterNext(SRowIter *pIter) { break; } - if (bv == ROW_BIT_NONE) { + if (bv == BIT_FLG_NONE) { pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); goto _exit; - } else if (bv == ROW_BIT_NULL) { + } else if (bv == BIT_FLG_NULL) { pIter->cv = COL_VAL_NULL(pTColumn->colId, pTColumn->type); goto _exit; } @@ -698,7 +840,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { pIter->cv.value.pData = NULL; } } else { - memcpy(&pIter->cv.value.val, pIter->pv + pTColumn->offset, pTColumn->bytes); + memcpy(&pIter->cv.value.val, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]); } goto _exit; } @@ -708,7 +850,266 @@ _exit: return &pIter->cv; } -// STSchema ======================================== +static int32_t tRowAppendNoneToColData(SColData *aColData, int32_t nColData) { + int32_t code = 0; + + for (int32_t iColData = 0; iColData < nColData; iColData++) { + SColData *pColData = &aColData[iColData]; + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + } + +_exit: + return code; +} +static int32_t tRowAppendNullToColData(SColData *aColData, int32_t nColData, STSchema *pSchema) { + int32_t code = 0; + + int32_t iColData = 0; + SColData *pColData = &aColData[iColData]; + int32_t iTColumn = 1; + STColumn *pTColumn = &pSchema->columns[iTColumn]; + + while (pColData) { + if (pTColumn) { + if (pTColumn->colId == pColData->cid) { // NULL + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + pTColumn = (++iTColumn < pSchema->numOfCols) ? &pSchema->columns[iTColumn] : NULL; + } else if (pTColumn->colId > pColData->cid) { // NONE + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } else { + pTColumn = (++iTColumn < pSchema->numOfCols) ? &pSchema->columns[iTColumn] : NULL; + } + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } + } + +_exit: + return code; +} +static int32_t tRowAppendTupleToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData) { + int32_t code = 0; + + int32_t iColData = 0; + SColData *pColData = &aColData[iColData]; + int32_t iTColumn = 1; + STColumn *pTColumn = &pTSchema->columns[iTColumn]; + + uint8_t *pb = NULL, *pf = NULL, *pv = NULL; + + switch (pRow->flag) { + case HAS_VALUE: + pf = pRow->data; + pv = pf + pTSchema->flen; + break; + case (HAS_NULL | HAS_NONE): + pb = pRow->data; + break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + pb = pRow->data; + pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + pb = pRow->data; + pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + default: + ASSERT(0); + break; + } + + while (pColData) { + if (pTColumn) { + if (pTColumn->colId == pColData->cid) { + ASSERT(pTColumn->type == pColData->type); + if (pb) { + uint8_t bv; + switch (pRow->flag) { + case (HAS_NULL | HAS_NONE): + bv = GET_BIT1(pb, iTColumn - 1); + break; + case (HAS_VALUE | HAS_NONE): + bv = GET_BIT1(pb, iTColumn - 1); + if (bv) bv++; + break; + case (HAS_VALUE | HAS_NULL): + bv = GET_BIT1(pb, iTColumn - 1) + 1; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + bv = GET_BIT2(pb, iTColumn - 1); + break; + default: + ASSERT(0); + break; + } + + if (bv == BIT_FLG_NONE) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + goto _continue; + } else if (bv == BIT_FLG_NULL) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + goto _continue; + } + } + + if (IS_VAR_DATA_TYPE(pColData->type)) { + uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset); + uint32_t nData; + pData += tGetU32v(pData, &nData); + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pf + pTColumn->offset, + TYPE_BYTES[pColData->type]); + if (code) goto _exit; + } + + _continue: + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + } else if (pTColumn->colId > pColData->cid) { // NONE + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } else { + pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + } + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } + } + +_exit: + return code; +} +static int32_t tRowAppendKVToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData) { + int32_t code = 0; + + SKVIdx *pKVIdx = (SKVIdx *)pRow->data; + uint8_t *pv = NULL; + int32_t iColData = 0; + SColData *pColData = &aColData[iColData]; + int32_t iTColumn = 1; + STColumn *pTColumn = &pTSchema->columns[iTColumn]; + int32_t iCol = 0; + + if (pRow->flag & KV_FLG_LIT) { + pv = pKVIdx->idx + pKVIdx->nCol; + } else if (pRow->flag & KV_FLG_MID) { + pv = pKVIdx->idx + (pKVIdx->nCol << 1); + } else if (pRow->flag & KV_FLG_BIG) { + pv = pKVIdx->idx + (pKVIdx->nCol << 2); + } else { + ASSERT(0); + } + + while (pColData) { + if (pTColumn) { + if (pTColumn->colId == pColData->cid) { + while (iCol < pKVIdx->nCol) { + uint8_t *pData; + if (pRow->flag & KV_FLG_LIT) { + pData = pv + ((uint8_t *)pKVIdx->idx)[iCol]; + } else if (pRow->flag & KV_FLG_MID) { + pData = pv + ((uint16_t *)pKVIdx->idx)[iCol]; + } else if (pRow->flag & KV_FLG_BIG) { + pData = pv + ((uint32_t *)pKVIdx->idx)[iCol]; + } else { + ASSERT(0); + } + + int16_t cid; + pData += tGetI16v(pData, &cid); + + if (TABS(cid) == pTColumn->colId) { + if (cid < 0) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } else { + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + uint32_t nData; + pData += tGetU32v(pData, &nData); + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, 0); + if (code) goto _exit; + } + } + iCol++; + goto _continue; + } else if (TABS(cid) > pTColumn->colId) { // NONE + break; + } else { + iCol++; + } + } + + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + + _continue: + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + } else if (pTColumn->colId > pColData->cid) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } else { + pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + } + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } + } + +_exit: + return code; +} +int32_t tRowAppendToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData) { + ASSERT(pRow->sver == pTSchema->version); + ASSERT(nColData > 0); + + int32_t code = 0; + + if (pRow->flag == HAS_NONE) { + code = tRowAppendNoneToColData(aColData, nColData); + goto _exit; + } else if (pRow->flag == HAS_NULL) { + code = tRowAppendNullToColData(aColData, nColData, pTSchema); + goto _exit; + } + + if (pRow->flag >> 4) { // KV row + code = tRowAppendKVToColData(pRow, pTSchema, aColData, nColData); + if (code) goto _exit; + } else { + code = tRowAppendTupleToColData(pRow, pTSchema, aColData, nColData); + if (code) goto _exit; + } + +_exit: + return code; +} // STag ======================================== static int tTagValCmprFn(const void *p1, const void *p2) { @@ -1086,120 +1487,47 @@ void tTagSetCid(const STag *pTag, int16_t iTag, int16_t cid) { tPutI16v(p + offset, cid); } -#if 1 // =================================================================================================================== -int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) { - if (pBuilder == NULL) return -1; +// STSchema ======================================== +STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version) { + STSchema *pTSchema = taosMemoryCalloc(1, sizeof(STSchema) + sizeof(STColumn) * numOfCols); + if (pTSchema == NULL) return NULL; - pBuilder->tCols = 256; - pBuilder->columns = (STColumn *)taosMemoryMalloc(sizeof(STColumn) * pBuilder->tCols); - if (pBuilder->columns == NULL) return -1; + pTSchema->numOfCols = numOfCols; + pTSchema->version = version; - tdResetTSchemaBuilder(pBuilder, version); - return 0; -} + // timestamp column + ASSERT(aSchema[0].type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(aSchema[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID); + pTSchema->columns[0].colId = aSchema[0].colId; + pTSchema->columns[0].type = aSchema[0].type; + pTSchema->columns[0].flags = aSchema[0].flags; + pTSchema->columns[0].bytes = TYPE_BYTES[aSchema[0].type]; + pTSchema->columns[0].offset = -1; -void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder) { - if (pBuilder) { - taosMemoryFreeClear(pBuilder->columns); - } -} + // other columns + for (int32_t iCol = 1; iCol < numOfCols; iCol++) { + SSchema *pSchema = &aSchema[iCol]; + STColumn *pTColumn = &pTSchema->columns[iCol]; -void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) { - pBuilder->nCols = 0; - pBuilder->tlen = 0; - pBuilder->flen = 0; - pBuilder->version = version; -} + pTColumn->colId = pSchema->colId; + pTColumn->type = pSchema->type; + pTColumn->flags = pSchema->flags; + pTColumn->offset = pTSchema->flen; -int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes) { - if (!isValidDataType(type)) return -1; + if (IS_VAR_DATA_TYPE(pSchema->type)) { + pTColumn->bytes = pSchema->bytes; + pTSchema->tlen += (TYPE_BYTES[pSchema->type] + pSchema->bytes); // todo: remove + } else { + pTColumn->bytes = TYPE_BYTES[pSchema->type]; + pTSchema->tlen += TYPE_BYTES[pSchema->type]; // todo: remove + } - if (pBuilder->nCols >= pBuilder->tCols) { - pBuilder->tCols *= 2; - STColumn *columns = (STColumn *)taosMemoryRealloc(pBuilder->columns, sizeof(STColumn) * pBuilder->tCols); - if (columns == NULL) return -1; - pBuilder->columns = columns; + pTSchema->flen += TYPE_BYTES[pTColumn->type]; } - STColumn *pCol = &(pBuilder->columns[pBuilder->nCols]); - pCol->type = type; - pCol->colId = colId; - pCol->flags = flags; - if (pBuilder->nCols == 0) { - pCol->offset = -1; - } else { - pCol->offset = pBuilder->flen; - pBuilder->flen += TYPE_BYTES[type]; - } - - if (IS_VAR_DATA_TYPE(type)) { - pCol->bytes = bytes; - pBuilder->tlen += (TYPE_BYTES[type] + bytes); - } else { - pCol->bytes = TYPE_BYTES[type]; - pBuilder->tlen += TYPE_BYTES[type]; - } - - pBuilder->nCols++; - - ASSERT(pCol->offset < pBuilder->flen); - - return 0; -} - -STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { - if (pBuilder->nCols <= 0) return NULL; - - int tlen = sizeof(STSchema) + sizeof(STColumn) * pBuilder->nCols; - - STSchema *pSchema = (STSchema *)taosMemoryMalloc(tlen); - if (pSchema == NULL) return NULL; - - pSchema->version = pBuilder->version; - pSchema->numOfCols = pBuilder->nCols; - pSchema->tlen = pBuilder->tlen; - pSchema->flen = pBuilder->flen; - -#ifdef TD_SUPPORT_BITMAP - pSchema->tlen += (int)TD_BITMAP_BYTES(pSchema->numOfCols); -#endif - - memcpy(&pSchema->columns[0], pBuilder->columns, sizeof(STColumn) * pBuilder->nCols); - - return pSchema; -} - -#endif - -STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version) { - STSchema *pTSchema = taosMemoryCalloc(1, sizeof(STSchema) + sizeof(STColumn) * numOfCols); - if (pTSchema == NULL) return NULL; - - pTSchema->numOfCols = numOfCols; - pTSchema->version = version; - - // timestamp column - ASSERT(aSchema[0].type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(aSchema[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID); - pTSchema->columns[0].colId = aSchema[0].colId; - pTSchema->columns[0].type = aSchema[0].type; - pTSchema->columns[0].flags = aSchema[0].flags; - pTSchema->columns[0].bytes = aSchema[0].bytes; - pTSchema->columns[0].offset = -1; - - // other columns - for (int32_t iCol = 1; iCol < numOfCols; iCol++) { - SSchema *pSchema = &aSchema[iCol]; - STColumn *pTColumn = &pTSchema->columns[iCol]; - - pTColumn->colId = pSchema->colId; - pTColumn->type = pSchema->type; - pTColumn->flags = pSchema->flags; - pTColumn->bytes = pSchema->bytes; - pTColumn->offset = pTSchema->flen; - - pTSchema->flen += TYPE_BYTES[pTColumn->type]; - } +#if 1 // todo : remove this + pTSchema->tlen += (int32_t)TD_BITMAP_BYTES(numOfCols); +#endif return pTSchema; } @@ -1213,7 +1541,7 @@ void tColDataDestroy(void *ph) { SColData *pColData = (SColData *)ph; tFree(pColData->pBitMap); - tFree((uint8_t *)pColData->aOffset); + tFree(pColData->aOffset); tFree(pColData->pData); } @@ -1230,25 +1558,37 @@ void tColDataClear(SColData *pColData) { pColData->nData = 0; } -static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVal) { +void tColDataDeepClear(SColData *pColData) { + pColData->pBitMap = NULL; + pColData->aOffset = NULL; + pColData->pData = NULL; + + tColDataClear(pColData); +} + +static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; if (IS_VAR_DATA_TYPE(pColData->type)) { - code = tRealloc((uint8_t **)(&pColData->aOffset), sizeof(int32_t) * (pColData->nVal + 1)); + code = tRealloc((uint8_t **)(&pColData->aOffset), (pColData->nVal + 1) << 2); if (code) goto _exit; pColData->aOffset[pColData->nVal] = pColData->nData; - if (pColVal->value.nData) { - code = tRealloc(&pColData->pData, pColData->nData + pColVal->value.nData); + if (nData) { + code = tRealloc(&pColData->pData, pColData->nData + nData); if (code) goto _exit; - memcpy(pColData->pData + pColData->nData, pColVal->value.pData, pColVal->value.nData); - pColData->nData += pColVal->value.nData; + memcpy(pColData->pData + pColData->nData, pData, nData); + pColData->nData += nData; } } else { ASSERT(pColData->nData == tDataTypes[pColData->type].bytes * pColData->nVal); code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes); if (code) goto _exit; - memcpy(pColData->pData + pColData->nData, &pColVal->value.val, tDataTypes[pColData->type].bytes); + if (pData) { + memcpy(pColData->pData + pColData->nData, pData, TYPE_BYTES[pColData->type]); + } else { + memset(pColData->pData + pColData->nData, 0, TYPE_BYTES[pColData->type]); + } pColData->nData += tDataTypes[pColData->type].bytes; } pColData->nVal++; @@ -1256,21 +1596,21 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVa _exit: return code; } -static FORCE_INLINE int32_t tColDataAppendValue00(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue00(SColData *pColData, uint8_t *pData, uint32_t nData) { pColData->flag = HAS_VALUE; - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue01(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue01(SColData *pColData, uint8_t *pData, uint32_t nData) { pColData->flag = HAS_NONE; pColData->nVal++; return 0; } -static FORCE_INLINE int32_t tColDataAppendValue02(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue02(SColData *pColData, uint8_t *pData, uint32_t nData) { pColData->flag = HAS_NULL; pColData->nVal++; return 0; } -static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; int32_t nBit = BIT1_SIZE(pColData->nVal + 1); @@ -1278,7 +1618,7 @@ static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 0, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); pColData->flag |= HAS_VALUE; @@ -1296,13 +1636,13 @@ static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *p } } - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue11(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue11(SColData *pColData, uint8_t *pData, uint32_t nData) { pColData->nVal++; return 0; } -static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; int32_t nBit = BIT1_SIZE(pColData->nVal + 1); @@ -1310,14 +1650,14 @@ static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 0, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); pColData->flag |= HAS_NULL; pColData->nVal++; return code; } -static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; int32_t nBit = BIT1_SIZE(pColData->nVal + 1); @@ -1325,7 +1665,7 @@ static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 0, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); pColData->flag |= HAS_VALUE; @@ -1343,9 +1683,9 @@ static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, SColVal *p } } - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; int32_t nBit = BIT1_SIZE(pColData->nVal + 1); @@ -1353,18 +1693,18 @@ static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 255, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); pColData->flag |= HAS_NONE; pColData->nVal++; return code; } -static FORCE_INLINE int32_t tColDataAppendValue22(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue22(SColData *pColData, uint8_t *pData, uint32_t nData) { pColData->nVal++; return 0; } -static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; pColData->flag |= HAS_VALUE; @@ -1374,9 +1714,9 @@ static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, SColVal *p if (code) return code; for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal)); + SET_BIT2_EX(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal)); } - SET_BIT2(pBitMap, pColData->nVal, 2); + SET_BIT2_EX(pBitMap, pColData->nVal, 2); tFree(pColData->pBitMap); pColData->pBitMap = pBitMap; @@ -1395,32 +1735,32 @@ static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, SColVal *p } } - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue31(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue31(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); pColData->nVal++; return code; } -static FORCE_INLINE int32_t tColDataAppendValue32(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue32(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); pColData->nVal++; return code; } #define tColDataAppendValue40 tColDataPutValue -static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; pColData->flag |= HAS_NONE; @@ -1430,11 +1770,11 @@ static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 255, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; pColData->flag |= HAS_NULL; @@ -1444,31 +1784,31 @@ static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 255, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; pColData->flag |= HAS_NULL; @@ -1478,25 +1818,25 @@ static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, SColVal *p if (code) return code; for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0); + SET_BIT2_EX(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0); } - SET_BIT2(pBitMap, pColData->nVal, 1); + SET_BIT2_EX(pBitMap, pColData->nVal, 1); tFree(pColData->pBitMap); pColData->pBitMap = pBitMap; - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; pColData->flag |= HAS_NONE; @@ -1506,52 +1846,52 @@ static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, SColVal *p if (code) return code; for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1); + SET_BIT2_EX(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1); } - SET_BIT2(pBitMap, pColData->nVal, 0); + SET_BIT2_EX(pBitMap, pColData->nVal, 0); tFree(pColData->pBitMap); pColData->pBitMap = pBitMap; - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT2(pColData->pBitMap, pColData->nVal, 2); + SET_BIT2_EX(pColData->pBitMap, pColData->nVal, 2); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT2(pColData->pBitMap, pColData->nVal, 0); + SET_BIT2_EX(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT2(pColData->pBitMap, pColData->nVal, 1); + SET_BIT2_EX(pColData->pBitMap, pColData->nVal, 1); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, SColVal *pColVal) = { +static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData) = { {tColDataAppendValue00, tColDataAppendValue01, tColDataAppendValue02}, // 0 {tColDataAppendValue10, tColDataAppendValue11, tColDataAppendValue12}, // HAS_NONE {tColDataAppendValue20, tColDataAppendValue21, tColDataAppendValue22}, // HAS_NULL @@ -1563,7 +1903,9 @@ static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, SColVal *pCo }; int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) { ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type); - return tColDataAppendValueImpl[pColData->flag][pColVal->flag](pColData, pColVal); + return tColDataAppendValueImpl[pColData->flag][pColVal->flag]( + pColData, IS_VAR_DATA_TYPE(pColData->type) ? pColVal->value.pData : (uint8_t *)&pColVal->value.val, + pColVal->value.nData); } static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NONE @@ -1572,7 +1914,8 @@ static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SCo static FORCE_INLINE void tColDataGetValue2(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NULL *pColVal = COL_VAL_NULL(pColData->cid, pColData->type); } -static FORCE_INLINE void tColDataGetValue3(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NULL|HAS_NONE +static FORCE_INLINE void tColDataGetValue3(SColData *pColData, int32_t iVal, + SColVal *pColVal) { // HAS_NULL|HAS_NONE switch (GET_BIT1(pColData->pBitMap, iVal)) { case 0: *pColVal = COL_VAL_NONE(pColData->cid, pColData->type); @@ -1687,46 +2030,776 @@ uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal) { return v; } -int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { +int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMalloc, void *arg) { int32_t code = 0; - int32_t size; - - ASSERT(pColDataSrc->nVal > 0); - ASSERT(pColDataDest->cid == pColDataSrc->cid); - ASSERT(pColDataDest->type == pColDataSrc->type); - pColDataDest->smaOn = pColDataSrc->smaOn; - pColDataDest->nVal = pColDataSrc->nVal; - pColDataDest->flag = pColDataSrc->flag; + *pColData = *pColDataFrom; // bitmap - if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) { - size = BIT2_SIZE(pColDataSrc->nVal); - code = tRealloc(&pColDataDest->pBitMap, size); - if (code) goto _exit; - memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size); + switch (pColData->flag) { + case (HAS_NULL | HAS_NONE): + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + pColData->pBitMap = xMalloc(arg, BIT1_SIZE(pColData->nVal)); + if (pColData->pBitMap == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT1_SIZE(pColData->nVal)); + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + pColData->pBitMap = xMalloc(arg, BIT2_SIZE(pColData->nVal)); + if (pColData->pBitMap == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT2_SIZE(pColData->nVal)); + break; + default: + pColData->pBitMap = NULL; + break; } // offset - if (IS_VAR_DATA_TYPE(pColDataDest->type)) { - size = sizeof(int32_t) * pColDataSrc->nVal; + if (IS_VAR_DATA_TYPE(pColData->type) && (pColData->flag & HAS_VALUE)) { + pColData->aOffset = xMalloc(arg, pColData->nVal << 2); + if (pColData->aOffset == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(pColData->aOffset, pColDataFrom->aOffset, pColData->nVal << 2); + } else { + pColData->aOffset = NULL; + } - code = tRealloc((uint8_t **)&pColDataDest->aOffset, size); - if (code) goto _exit; + // value + if (pColData->nData) { + pColData->pData = xMalloc(arg, pColData->nData); + if (pColData->pData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } - memcpy(pColDataDest->aOffset, pColDataSrc->aOffset, size); + memcpy(pColData->pData, pColDataFrom->pData, pColData->nData); + } else { + pColData->pData = NULL; } - // value - pColDataDest->nData = pColDataSrc->nData; - code = tRealloc(&pColDataDest->pData, pColDataSrc->nData); - if (code) goto _exit; - memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData); +_exit: + return code; +} + +int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t bytes, int32_t nRows, char *lengthOrbitmap, + char *data) { + int32_t code = 0; + + if (IS_VAR_DATA_TYPE(type)) { // var-length data type + for (int32_t i = 0; i < nRows; ++i) { + int32_t offset = *((int32_t *)lengthOrbitmap + i); + if (offset == -1) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, (uint8_t *)varDataVal(data + offset), + varDataLen(data + offset)); + } + } + } else { // fixed-length data type + bool allValue = true; + bool allNull = true; + for (int32_t i = 0; i < nRows; ++i) { + if (!colDataIsNull_f(lengthOrbitmap, i)) { + allNull = false; + } else { + allValue = false; + } + } + + if (allValue) { + // optimize (todo) + for (int32_t i = 0; i < nRows; ++i) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, (uint8_t *)data + bytes * i, bytes); + } + } else if (allNull) { + // optimize (todo) + for (int32_t i = 0; i < nRows; ++i) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } + } else { + for (int32_t i = 0; i < nRows; ++i) { + if (colDataIsNull_f(lengthOrbitmap, i)) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, (uint8_t *)data + bytes * i, bytes); + } + } + } + } _exit: return code; } +int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) { + int32_t code = 0; + + ASSERT(pColData->type == pBind->buffer_type); + + if (IS_VAR_DATA_TYPE(pBind->buffer_type)) { // var-length data type + for (int32_t i = 0; i < pBind->num; ++i) { + if (pBind->is_null && pBind->is_null[i]) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE]( + pColData, (uint8_t *)pBind->buffer + pBind->buffer_length * i, pBind->length[i]); + } + } + } else { // fixed-length data type + bool allValue; + bool allNull; + if (pBind->is_null) { + bool same = (memcmp(pBind->is_null, pBind->is_null + 1, pBind->num - 1) == 0); + allNull = (same && pBind->is_null[0] != 0); + allValue = (same && pBind->is_null[0] == 0); + } else { + allNull = false; + allValue = true; + } + + if (allValue) { + // optimize (todo) + for (int32_t i = 0; i < pBind->num; ++i) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE]( + pColData, (uint8_t *)pBind->buffer + TYPE_BYTES[pColData->type] * i, pBind->buffer_length); + } + } else if (allNull) { + // optimize (todo) + for (int32_t i = 0; i < pBind->num; ++i) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } + } else { + for (int32_t i = 0; i < pBind->num; ++i) { + if (pBind->is_null[i]) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE]( + pColData, (uint8_t *)pBind->buffer + TYPE_BYTES[pColData->type] * i, pBind->buffer_length); + } + } + } + } + +_exit: + return code; +} + +static int32_t tColDataSwapValue(SColData *pColData, int32_t i, int32_t j) { + int32_t code = 0; + + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nData1 = pColData->aOffset[i + 1] - pColData->aOffset[i]; + int32_t nData2 = (j < pColData->nVal - 1) ? pColData->aOffset[j + 1] - pColData->aOffset[j] + : pColData->nData - pColData->aOffset[j]; + uint8_t *pData = taosMemoryMalloc(TMAX(nData1, nData2)); + if (pData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + if (nData1 > nData2) { + memcpy(pData, pColData->pData + pColData->aOffset[i], nData1); + memcpy(pColData->pData + pColData->aOffset[i], pColData->pData + pColData->aOffset[j], nData2); + // memmove(pColData->pData + pColData->aOffset[i] + nData2, pColData->pData + pColData->aOffset[i] + nData1, + // pColData->aOffset[j] - pColData->aOffset[i + 1]); + memmove(pColData->pData + pColData->aOffset[i] + nData2, pColData->pData + pColData->aOffset[i + 1], + pColData->aOffset[j] - pColData->aOffset[i + 1]); + memcpy(pColData->pData + pColData->aOffset[j] + nData2 - nData1, pData, nData1); + } else { + memcpy(pData, pColData->pData + pColData->aOffset[j], nData2); + memcpy(pColData->pData + pColData->aOffset[j] + nData2 - nData1, pColData->pData + pColData->aOffset[i], nData1); + // memmove(pColData->pData + pColData->aOffset[j] + nData2 - nData1, pColData->pData + pColData->aOffset[i] + + // nData1, + // pColData->aOffset[j] - pColData->aOffset[i + 1]); + memmove(pColData->pData + pColData->aOffset[i] + nData2, pColData->pData + pColData->aOffset[i + 1], + pColData->aOffset[j] - pColData->aOffset[i + 1]); + memcpy(pColData->pData + pColData->aOffset[i], pData, nData2); + } + for (int32_t k = i + 1; k <= j; ++k) { + pColData->aOffset[k] = pColData->aOffset[k] + nData2 - nData1; + } + + taosMemoryFree(pData); + } else { + uint64_t val; + memcpy(&val, &pColData->pData[TYPE_BYTES[pColData->type] * i], TYPE_BYTES[pColData->type]); + memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * i], &pColData->pData[TYPE_BYTES[pColData->type] * j], + TYPE_BYTES[pColData->type]); + memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * j], &val, TYPE_BYTES[pColData->type]); + } + +_exit: + return code; +} + +static void tColDataSwap(SColData *pColData, int32_t i, int32_t j) { + ASSERT(i < j); + ASSERT(j < pColData->nVal); + + switch (pColData->flag) { + case HAS_NONE: + case HAS_NULL: + break; + case (HAS_NULL | HAS_NONE): { + uint8_t bv = GET_BIT1(pColData->pBitMap, i); + SET_BIT1(pColData->pBitMap, i, GET_BIT1(pColData->pBitMap, j)); + SET_BIT1(pColData->pBitMap, j, bv); + } break; + case HAS_VALUE: { + tColDataSwapValue(pColData, i, j); + } break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): { + uint8_t bv = GET_BIT1(pColData->pBitMap, i); + SET_BIT1(pColData->pBitMap, i, GET_BIT1(pColData->pBitMap, j)); + SET_BIT1(pColData->pBitMap, j, bv); + tColDataSwapValue(pColData, i, j); + } break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): { + uint8_t bv = GET_BIT2(pColData->pBitMap, i); + SET_BIT2(pColData->pBitMap, i, GET_BIT2(pColData->pBitMap, j)); + SET_BIT2(pColData->pBitMap, j, bv); + tColDataSwapValue(pColData, i, j); + } break; + default: + ASSERT(0); + break; + } +} + +static int32_t tColDataCopyRowCell(SColData *pFromColData, int32_t iFromRow, SColData *pToColData, int32_t iToRow) { + int32_t code = TSDB_CODE_SUCCESS; + + if (IS_VAR_DATA_TYPE(pToColData->type)) { + int32_t nData = (iFromRow < pFromColData->nVal - 1) + ? pFromColData->aOffset[iFromRow + 1] - pFromColData->aOffset[iFromRow] + : pFromColData->nData - pFromColData->aOffset[iFromRow]; + if (iToRow == 0) { + pToColData->aOffset[iToRow] = 0; + } + + if (iToRow < pToColData->nVal - 1) { + pToColData->aOffset[iToRow + 1] = pToColData->aOffset[iToRow] + nData; + } + + memcpy(pToColData->pData + pToColData->aOffset[iToRow], pFromColData->pData + pFromColData->aOffset[iFromRow], + nData); + } else { + memcpy(&pToColData->pData[TYPE_BYTES[pToColData->type] * iToRow], + &pFromColData->pData[TYPE_BYTES[pToColData->type] * iFromRow], TYPE_BYTES[pToColData->type]); + } + return code; +} + +static int32_t tColDataCopyRowSingleCol(SColData *pFromColData, int32_t iFromRow, SColData *pToColData, + int32_t iToRow) { + int32_t code = TSDB_CODE_SUCCESS; + + switch (pFromColData->flag) { + case HAS_NONE: + case HAS_NULL: + break; + case (HAS_NULL | HAS_NONE): { + SET_BIT1(pToColData->pBitMap, iToRow, GET_BIT1(pFromColData->pBitMap, iFromRow)); + } break; + case HAS_VALUE: { + tColDataCopyRowCell(pFromColData, iFromRow, pToColData, iToRow); + } break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): { + SET_BIT1(pToColData->pBitMap, iToRow, GET_BIT1(pFromColData->pBitMap, iFromRow)); + tColDataCopyRowCell(pFromColData, iFromRow, pToColData, iToRow); + } break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): { + SET_BIT2(pToColData->pBitMap, iToRow, GET_BIT2(pFromColData->pBitMap, iFromRow)); + tColDataCopyRowCell(pFromColData, iFromRow, pToColData, iToRow); + } break; + default: + return -1; + } + + return code; +} + +static int32_t tColDataCopyRow(SColData *aFromColData, int32_t iFromRow, SColData *aToColData, int32_t iToRow, + int32_t nColData) { + int32_t code = TSDB_CODE_SUCCESS; + + for (int32_t i = 0; i < nColData; i++) { + code = tColDataCopyRowSingleCol(&aFromColData[i], iFromRow, &aToColData[i], iToRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + return code; +} + +static int32_t tColDataCopyRowAppend(SColData *aFromColData, int32_t iFromRow, SColData *aToColData, int32_t nColData) { + int32_t code = TSDB_CODE_SUCCESS; + + for (int32_t i = 0; i < nColData; i++) { + SColVal cv = {0}; + tColDataGetValue(&aFromColData[i], iFromRow, &cv); + code = tColDataAppendValue(&aToColData[i], &cv); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + return code; +} + +static int32_t tColDataMergeSortMerge(SColData *aColData, int32_t start, int32_t mid, int32_t end, int32_t nColData) { + SColData *aDstColData = NULL; + TSKEY *aKey = (TSKEY *)aColData[0].pData; + + int32_t i = start, j = mid + 1, k = 0; + + if (end > start) { + aDstColData = taosMemoryCalloc(1, sizeof(SColData) * nColData); + for (int c = 0; c < nColData; ++c) { + tColDataInit(&aDstColData[c], aColData[c].cid, aColData[c].type, aColData[c].smaOn); + } + if (aDstColData == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + /* + for (int32_t i = 0; i < nColData; i++) { + tColDataCopy(&aColData[i], &aDstColData[i], tColDataDefaultMalloc, NULL); + } + */ + } + + while (i <= mid && j <= end) { + if (aKey[i] <= aKey[j]) { + // tColDataCopyRow(aColData, i++, aDstColData, k++); + tColDataCopyRowAppend(aColData, i++, aDstColData, nColData); + } else { + // tColDataCopyRow(aColData, j++, aDstColData, k++); + tColDataCopyRowAppend(aColData, j++, aDstColData, nColData); + } + } + + while (i <= mid) { + // tColDataCopyRow(aColData, i++, aDstColData, k++); + tColDataCopyRowAppend(aColData, i++, aDstColData, nColData); + } + + while (j <= end) { + // tColDataCopyRow(aColData, j++, aDstColData, k++); + tColDataCopyRowAppend(aColData, j++, aDstColData, nColData); + } + + for (i = start, k = 0; i <= end; ++i, ++k) { + tColDataCopyRow(aDstColData, k, aColData, i, nColData); + } + + if (aDstColData) { + for (int32_t i = 0; i < nColData; i++) { + tColDataDestroy(&aDstColData[i]); + } + taosMemoryFree(aDstColData); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t tColDataMergeSort(SColData *aColData, int32_t start, int32_t end, int32_t nColData) { + int32_t ret = TSDB_CODE_SUCCESS; + int32_t mid; + + if (start >= end) { + return TSDB_CODE_SUCCESS; + } + + mid = (start + end) / 2; + + ret = tColDataMergeSort(aColData, start, mid, nColData); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + ret = tColDataMergeSort(aColData, mid + 1, end, nColData); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + return tColDataMergeSortMerge(aColData, start, mid, end, nColData); +} + +static int32_t tColDataSort(SColData *aColData, int32_t nColData) { + int32_t nVal = aColData[0].nVal; + + if (nVal < 2) return TSDB_CODE_SUCCESS; + + return tColDataMergeSort(aColData, 0, nVal - 1, nColData); +} +static void tColDataMergeImpl(SColData *pColData, int32_t iStart, int32_t iEnd /* not included */) { + switch (pColData->flag) { + case HAS_NONE: + case HAS_NULL: { + pColData->nVal -= (iEnd - iStart - 1); + } break; + case (HAS_NULL | HAS_NONE): { + if (GET_BIT1(pColData->pBitMap, iStart) == 0) { + for (int32_t i = iStart + 1; i < iEnd; ++i) { + if (GET_BIT1(pColData->pBitMap, i) == 1) { + SET_BIT1(pColData->pBitMap, iStart, 1); + break; + } + } + } + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + SET_BIT1(pColData->pBitMap, j, GET_BIT1(pColData->pBitMap, i)); + } + + pColData->nVal -= (iEnd - iStart - 1); + + uint8_t flag = 0; + for (int32_t i = 0; i < pColData->nVal; ++i) { + uint8_t bv = GET_BIT1(pColData->pBitMap, i); + if (bv == BIT_FLG_NONE) { + flag |= HAS_NONE; + } else if (bv == BIT_FLG_NULL) { + flag |= HAS_NULL; + } else { + ASSERT(0); + } + + if (flag == pColData->flag) break; + } + pColData->flag = flag; + } break; + case HAS_VALUE: { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nDiff = pColData->aOffset[iEnd - 1] - pColData->aOffset[iStart]; + + memmove(pColData->pData + pColData->aOffset[iStart], pColData->pData + pColData->aOffset[iEnd - 1], + pColData->nData - pColData->aOffset[iEnd - 1]); + pColData->nData -= nDiff; + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + pColData->aOffset[j] = pColData->aOffset[i] - nDiff; + } + } else { + memmove(pColData->pData + TYPE_BYTES[pColData->type] * iStart, + pColData->pData + TYPE_BYTES[pColData->type] * (iEnd - 1), + TYPE_BYTES[pColData->type] * (pColData->nVal - iEnd + 1)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + pColData->nVal -= (iEnd - iStart - 1); + } break; + case (HAS_VALUE | HAS_NONE): { + uint8_t bv; + int32_t iv; + for (int32_t i = iEnd - 1; i >= iStart; --i) { + bv = GET_BIT1(pColData->pBitMap, i); + if (bv) { + iv = i; + break; + } + } + + if (bv) { // has a value + if (IS_VAR_DATA_TYPE(pColData->type)) { + if (iv != iStart) { + memmove(&pColData->pData[pColData->aOffset[iStart]], &pColData->pData[pColData->aOffset[iv]], + iv < (pColData->nVal - 1) ? pColData->aOffset[iv + 1] - pColData->aOffset[iv] + : pColData->nData - pColData->aOffset[iv]); + } + // TODO + ASSERT(0); + } else { + if (iv != iStart) { + memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * iStart], + &pColData->pData[TYPE_BYTES[pColData->type] * iv], TYPE_BYTES[pColData->type]); + } + memmove(&pColData->pData[TYPE_BYTES[pColData->type] * (iStart + 1)], + &pColData->pData[TYPE_BYTES[pColData->type] * iEnd], + TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + SET_BIT1(pColData->pBitMap, iStart, 1); + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + SET_BIT1(pColData->pBitMap, j, GET_BIT1(pColData->pBitMap, i)); + } + + uint8_t flag = HAS_VALUE; + for (int32_t i = 0; i < pColData->nVal - (iEnd - iStart - 1); ++i) { + if (GET_BIT1(pColData->pBitMap, i) == 0) { + flag |= HAS_NONE; + } + + if (flag == pColData->flag) break; + } + pColData->flag = flag; + } else { // all NONE + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nDiff = pColData->aOffset[iEnd - 1] - pColData->aOffset[iStart]; + + memmove(&pColData->pData[pColData->aOffset[iStart]], &pColData->pData[pColData->aOffset[iEnd - 1]], + pColData->nData - pColData->aOffset[iEnd - 1]); + pColData->nData -= nDiff; + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + pColData->aOffset[j] = pColData->aOffset[i] - nDiff; + } + } else { + memmove(pColData->pData + TYPE_BYTES[pColData->type] * (iStart + 1), + pColData->pData + TYPE_BYTES[pColData->type] * iEnd, + TYPE_BYTES[pColData->type] * (pColData->nVal - iEnd + 1)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + SET_BIT1(pColData->pBitMap, j, GET_BIT1(pColData->pBitMap, i)); + } + } + pColData->nVal -= (iEnd - iStart - 1); + } break; + case (HAS_VALUE | HAS_NULL): { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nDiff = pColData->aOffset[iEnd - 1] - pColData->aOffset[iStart]; + + memmove(pColData->pData + pColData->aOffset[iStart], pColData->pData + pColData->aOffset[iEnd - 1], + pColData->nData - pColData->aOffset[iEnd - 1]); + pColData->nData -= nDiff; + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + pColData->aOffset[j] = pColData->aOffset[i] - nDiff; + } + } else { + memmove(pColData->pData + TYPE_BYTES[pColData->type] * iStart, + pColData->pData + TYPE_BYTES[pColData->type] * (iEnd - 1), + TYPE_BYTES[pColData->type] * (pColData->nVal - iEnd + 1)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + for (int32_t i = iEnd - 1, j = iStart; i < pColData->nVal; ++i, ++j) { + SET_BIT1(pColData->pBitMap, j, GET_BIT1(pColData->pBitMap, i)); + } + + pColData->nVal -= (iEnd - iStart - 1); + + uint8_t flag = 0; + for (int32_t i = 0; i < pColData->nVal; ++i) { + if (GET_BIT1(pColData->pBitMap, i)) { + flag |= HAS_VALUE; + } else { + flag |= HAS_NULL; + } + + if (flag == pColData->flag) break; + } + pColData->flag = flag; + } break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): { + uint8_t bv; + int32_t iv; + for (int32_t i = iEnd - 1; i >= iStart; --i) { + bv = GET_BIT2(pColData->pBitMap, i); + if (bv) { + iv = i; + break; + } + } + + if (bv) { + // TODO + ASSERT(0); + } else { // ALL NONE + if (IS_VAR_DATA_TYPE(pColData->type)) { + // TODO + ASSERT(0); + } else { + memmove(pColData->pData + TYPE_BYTES[pColData->type] * (iStart + 1), + pColData->pData + TYPE_BYTES[pColData->type] * iEnd, + TYPE_BYTES[pColData->type] * (pColData->nVal - iEnd)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + SET_BIT2(pColData->pBitMap, j, GET_BIT2(pColData->pBitMap, i)); + } + } + pColData->nVal -= (iEnd - iStart - 1); + } break; + default: + ASSERT(0); + break; + } +} +static void tColDataMerge(SColData *aColData, int32_t nColData) { + int32_t iStart = 0; + for (;;) { + if (iStart >= aColData[0].nVal - 1) break; + + int32_t iEnd = iStart + 1; + while (iEnd < aColData[0].nVal) { + if (((TSKEY *)aColData[0].pData)[iEnd] != ((TSKEY *)aColData[0].pData)[iStart]) break; + + iEnd++; + } + + if (iEnd - iStart > 1) { + for (int32_t i = 0; i < nColData; i++) { + tColDataMergeImpl(&aColData[i], iStart, iEnd); + } + } + + iStart++; + } +} +void tColDataSortMerge(SArray *colDataArr) { + int32_t nColData = TARRAY_SIZE(colDataArr); + SColData *aColData = (SColData *)TARRAY_DATA(colDataArr); + + if (aColData[0].nVal <= 1) goto _exit; + + ASSERT(aColData[0].type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(aColData[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); + ASSERT(aColData[0].flag == HAS_VALUE); + + int8_t doSort = 0; + int8_t doMerge = 0; + // scan ------- + TSKEY *aKey = (TSKEY *)aColData[0].pData; + for (int32_t iVal = 1; iVal < aColData[0].nVal; ++iVal) { + if (aKey[iVal] > aKey[iVal - 1]) { + continue; + } else if (aKey[iVal] < aKey[iVal - 1]) { + doSort = 1; + break; + } else { + doMerge = 1; + } + } + + // sort ------- + if (doSort) { + tColDataSort(aColData, nColData); + } + + if (doMerge != 1) { + for (int32_t iVal = 1; iVal < aColData[0].nVal; ++iVal) { + if (aKey[iVal] == aKey[iVal - 1]) { + doMerge = 1; + break; + } + } + } + + // merge ------- + if (doMerge) { + tColDataMerge(aColData, nColData); + } + +_exit: + return; +} + +int32_t tPutColData(uint8_t *pBuf, SColData *pColData) { + int32_t n = 0; + + n += tPutI16v(pBuf ? pBuf + n : NULL, pColData->cid); + n += tPutI8(pBuf ? pBuf + n : NULL, pColData->type); + n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nVal); + n += tPutI8(pBuf ? pBuf + n : NULL, pColData->flag); + + // bitmap + switch (pColData->flag) { + case (HAS_NULL | HAS_NONE): + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT1_SIZE(pColData->nVal)); + n += BIT1_SIZE(pColData->nVal); + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT2_SIZE(pColData->nVal)); + n += BIT2_SIZE(pColData->nVal); + break; + default: + break; + } + + // value + if (pColData->flag & HAS_VALUE) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + if (pBuf) memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2); + n += (pColData->nVal << 2); + + n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nData); + if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); + n += pColData->nData; + } else { + if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); + n += pColData->nData; + } + } + + return n; +} + +int32_t tGetColData(uint8_t *pBuf, SColData *pColData) { + int32_t n = 0; + + n += tGetI16v(pBuf + n, &pColData->cid); + n += tGetI8(pBuf + n, &pColData->type); + n += tGetI32v(pBuf + n, &pColData->nVal); + n += tGetI8(pBuf + n, &pColData->flag); + + // bitmap + switch (pColData->flag) { + case (HAS_NULL | HAS_NONE): + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + pColData->pBitMap = pBuf + n; + n += BIT1_SIZE(pColData->nVal); + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + pColData->pBitMap = pBuf + n; + n += BIT2_SIZE(pColData->nVal); + break; + default: + break; + } + + // value + if (pColData->flag & HAS_VALUE) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + pColData->aOffset = (int32_t *)(pBuf + n); + n += (pColData->nVal << 2); + + n += tGetI32v(pBuf + n, &pColData->nData); + pColData->pData = pBuf + n; + n += pColData->nData; + } else { + pColData->pData = pBuf + n; + pColData->nData = TYPE_BYTES[pColData->type] * pColData->nVal; + n += pColData->nData; + } + } + + return n; +} + #define CALC_SUM_MAX_MIN(SUM, MAX, MIN, VAL) \ do { \ (SUM) += (VAL); \ diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 95625e8d9360946fbcf1432f804ec7841e006dfc..dde7d50c32a09d49add0829d16d0151f17e71570 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -5424,6 +5424,8 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS if (tEncodeI32(&encoder, pField->bytes) < 0) return -1; if (tEncodeCStr(&encoder, pField->name) < 0) return -1; } + if (tEncodeI8(&encoder, pReq->createStb) < 0) return -1; + if (tEncodeU64(&encoder, pReq->targetStbUid) < 0) return -1; tEndEncode(&encoder); @@ -5484,6 +5486,8 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea } } } + if (tDecodeI8(&decoder, &pReq->createStb) < 0) return -1; + if (tDecodeU64(&decoder, &pReq->targetStbUid) < 0) return -1; tEndDecode(&decoder); @@ -5627,30 +5631,6 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) { return 0; } -STSchema *tdGetSTSChemaFromSSChema(SSchema *pSchema, int32_t nCols, int32_t sver) { - STSchemaBuilder schemaBuilder = {0}; - if (tdInitTSchemaBuilder(&schemaBuilder, sver) < 0) { - return NULL; - } - - for (int i = 0; i < nCols; i++) { - SSchema *schema = pSchema + i; - if (tdAddColToSchema(&schemaBuilder, schema->type, schema->flags, schema->colId, schema->bytes) < 0) { - tdDestroyTSchemaBuilder(&schemaBuilder); - return NULL; - } - } - - STSchema *pNSchema = tdGetSchemaFromBuilder(&schemaBuilder); - if (pNSchema == NULL) { - tdDestroyTSchemaBuilder(&schemaBuilder); - return NULL; - } - - tdDestroyTSchemaBuilder(&schemaBuilder); - return pNSchema; -} - int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) { if (tStartEncode(pCoder) < 0) return -1; @@ -5728,6 +5708,25 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) { return 0; } +void tDestroySVCreateTbReq(SVCreateTbReq *pReq, int32_t flags) { + if (pReq == NULL) return; + + if (flags & TSDB_MSG_FLG_ENCODE) { + // TODO + } else if (flags & TSDB_MSG_FLG_DECODE) { + if (pReq->comment) { + pReq->comment = NULL; + taosMemoryFree(pReq->comment); + } + + if (pReq->type == TSDB_CHILD_TABLE) { + if (pReq->ctb.tagName) taosArrayDestroy(pReq->ctb.tagName); + } else if (pReq->type == TSDB_NORMAL_TABLE) { + if (pReq->ntb.schemaRow.pSchema) taosMemoryFree(pReq->ntb.schemaRow.pSchema); + } + } +} + int tEncodeSVCreateTbBatchReq(SEncoder *pCoder, const SVCreateTbBatchReq *pReq) { int32_t nReq = taosArrayGetSize(pReq->pArray); @@ -6719,3 +6718,328 @@ int32_t tDecodeSBatchDeleteReq(SDecoder *pDecoder, SBatchDeleteReq *pReq) { } return 0; } + +static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubmitTbData) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32v(pCoder, pSubmitTbData->flags) < 0) return -1; + + // auto create table + if (pSubmitTbData->flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { + ASSERT(pSubmitTbData->pCreateTbReq); + if (tEncodeSVCreateTbReq(pCoder, pSubmitTbData->pCreateTbReq) < 0) return -1; + } + + // submit data + if (tEncodeI64(pCoder, pSubmitTbData->suid) < 0) return -1; + if (tEncodeI64(pCoder, pSubmitTbData->uid) < 0) return -1; + if (tEncodeI32v(pCoder, pSubmitTbData->sver) < 0) return -1; + + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + uint64_t nColData = TARRAY_SIZE(pSubmitTbData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pSubmitTbData->aCol); + + if (tEncodeU64v(pCoder, nColData) < 0) return -1; + + for (uint64_t i = 0; i < nColData; i++) { + pCoder->pos += tPutColData(pCoder->data ? pCoder->data + pCoder->pos : NULL, &aColData[i]); + } + } else { + if (tEncodeU64v(pCoder, TARRAY_SIZE(pSubmitTbData->aRowP)) < 0) return -1; + + SRow **rows = (SRow **)TARRAY_DATA(pSubmitTbData->aRowP); + for (int32_t iRow = 0; iRow < TARRAY_SIZE(pSubmitTbData->aRowP); ++iRow) { + if (pCoder->data) memcpy(pCoder->data + pCoder->pos, rows[iRow], rows[iRow]->len); + pCoder->pos += rows[iRow]->len; + } + } + + tEndEncode(pCoder); + return 0; +} + +static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbData) { + int32_t code = 0; + + if (tStartDecode(pCoder) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (tDecodeI32v(pCoder, &pSubmitTbData->flags) < 0) return -1; + + if (pSubmitTbData->flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { + pSubmitTbData->pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (pSubmitTbData->pCreateTbReq == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + if (tDecodeSVCreateTbReq(pCoder, pSubmitTbData->pCreateTbReq) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + } + + // submit data + if (tDecodeI64(pCoder, &pSubmitTbData->suid) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + if (tDecodeI64(pCoder, &pSubmitTbData->uid) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + if (tDecodeI32v(pCoder, &pSubmitTbData->sver) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + uint64_t nColData; + + if (tDecodeU64v(pCoder, &nColData) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + pSubmitTbData->aCol = taosArrayInit(nColData, sizeof(SColData)); + if (pSubmitTbData->aCol == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t i = 0; i < nColData; ++i) { + pCoder->pos += tGetColData(pCoder->data + pCoder->pos, taosArrayReserve(pSubmitTbData->aCol, 1)); + } + } else { + uint64_t nRow; + if (tDecodeU64v(pCoder, &nRow) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + pSubmitTbData->aRowP = taosArrayInit(nRow, sizeof(SRow *)); + if (pSubmitTbData->aRowP == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t iRow = 0; iRow < nRow; ++iRow) { + SRow **ppRow = taosArrayReserve(pSubmitTbData->aRowP, 1); + + *ppRow = (SRow *)(pCoder->data + pCoder->pos); + pCoder->pos += (*ppRow)->len; + } + } + + tEndDecode(pCoder); + +_exit: + if (code) { + // TODO: clear + } + return 0; +} + +int32_t tEncodeSSubmitReq2(SEncoder *pCoder, const SSubmitReq2 *pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeU64v(pCoder, taosArrayGetSize(pReq->aSubmitTbData)) < 0) return -1; + for (uint64_t i = 0; i < taosArrayGetSize(pReq->aSubmitTbData); i++) { + if (tEncodeSSubmitTbData(pCoder, taosArrayGet(pReq->aSubmitTbData, i)) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSSubmitReq2(SDecoder *pCoder, SSubmitReq2 *pReq) { + int32_t code = 0; + + memset(pReq, 0, sizeof(*pReq)); + + // decode + if (tStartDecode(pCoder) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + uint64_t nSubmitTbData; + if (tDecodeU64v(pCoder, &nSubmitTbData) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + pReq->aSubmitTbData = taosArrayInit(nSubmitTbData, sizeof(SSubmitTbData)); + if (pReq->aSubmitTbData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (uint64_t i = 0; i < nSubmitTbData; i++) { + if (tDecodeSSubmitTbData(pCoder, taosArrayReserve(pReq->aSubmitTbData, 1)) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + } + + tEndDecode(pCoder); + +_exit: + if (code) { + if (pReq->aSubmitTbData) { + // todo + taosArrayDestroy(pReq->aSubmitTbData); + pReq->aSubmitTbData = NULL; + } + } + return code; +} + +void tDestroySSubmitTbData(SSubmitTbData *pTbData, int32_t flag) { + if (NULL == pTbData) { + return; + } + + if (flag == TSDB_MSG_FLG_ENCODE) { + if (pTbData->pCreateTbReq) { + tdDestroySVCreateTbReq(pTbData->pCreateTbReq); + taosMemoryFree(pTbData->pCreateTbReq); + } + + if (pTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + int32_t nColData = TARRAY_SIZE(pTbData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pTbData->aCol); + + for (int32_t i = 0; i < nColData; ++i) { + tColDataDestroy(&aColData[i]); + } + taosArrayDestroy(pTbData->aCol); + } else { + int32_t nRow = TARRAY_SIZE(pTbData->aRowP); + SRow **rows = (SRow **)TARRAY_DATA(pTbData->aRowP); + + for (int32_t i = 0; i < nRow; ++i) { + tRowDestroy(rows[i]); + } + taosArrayDestroy(pTbData->aRowP); + } + } else if (flag == TSDB_MSG_FLG_DECODE) { + if (pTbData->pCreateTbReq) { + tDestroySVCreateTbReq(pTbData->pCreateTbReq, TSDB_MSG_FLG_DECODE); + taosMemoryFree(pTbData->pCreateTbReq); + } + + if (pTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + taosArrayDestroy(pTbData->aCol); + } else { + taosArrayDestroy(pTbData->aRowP); + } + } +} + +void tDestroySSubmitReq2(SSubmitReq2 *pReq, int32_t flag) { + if (pReq->aSubmitTbData == NULL) return; + + int32_t nSubmitTbData = TARRAY_SIZE(pReq->aSubmitTbData); + SSubmitTbData *aSubmitTbData = (SSubmitTbData *)TARRAY_DATA(pReq->aSubmitTbData); + + for (int32_t i = 0; i < nSubmitTbData; i++) { + tDestroySSubmitTbData(&aSubmitTbData[i], flag); + } + taosArrayDestroy(pReq->aSubmitTbData); +} + +int32_t tEncodeSSubmitRsp2(SEncoder *pCoder, const SSubmitRsp2 *pRsp) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32v(pCoder, pRsp->affectedRows) < 0) return -1; + + if (tEncodeU64v(pCoder, taosArrayGetSize(pRsp->aCreateTbRsp)) < 0) return -1; + for (int32_t i = 0; i < taosArrayGetSize(pRsp->aCreateTbRsp); ++i) { + if (tEncodeSVCreateTbRsp(pCoder, taosArrayGet(pRsp->aCreateTbRsp, i)) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSSubmitRsp2(SDecoder *pCoder, SSubmitRsp2 *pRsp) { + int32_t code = 0; + + memset(pRsp, 0, sizeof(SSubmitRsp2)); + + // decode + if (tStartDecode(pCoder) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (tDecodeI32v(pCoder, &pRsp->affectedRows) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + uint64_t nCreateTbRsp; + if (tDecodeU64v(pCoder, &nCreateTbRsp) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (nCreateTbRsp) { + pRsp->aCreateTbRsp = taosArrayInit(nCreateTbRsp, sizeof(SVCreateTbRsp)); + if (pRsp->aCreateTbRsp == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t i = 0; i < nCreateTbRsp; ++i) { + SVCreateTbRsp *pCreateTbRsp = taosArrayReserve(pRsp->aCreateTbRsp, 1); + if (tDecodeSVCreateTbRsp(pCoder, pCreateTbRsp) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + } + } + + tEndDecode(pCoder); + +_exit: + if (code) { + if (pRsp->aCreateTbRsp) { + taosArrayDestroyEx(pRsp->aCreateTbRsp, NULL /* todo */); + } + } + return code; +} + +void tDestroySSubmitRsp2(SSubmitRsp2 *pRsp, int32_t flag) { + if (NULL == pRsp) { + return; + } + + if (flag & TSDB_MSG_FLG_ENCODE) { + if (pRsp->aCreateTbRsp) { + int32_t nCreateTbRsp = TARRAY_SIZE(pRsp->aCreateTbRsp); + SVCreateTbRsp *aCreateTbRsp = TARRAY_DATA(pRsp->aCreateTbRsp); + for (int32_t i = 0; i < nCreateTbRsp; ++i) { + if (aCreateTbRsp[i].pMeta) { + taosMemoryFree(aCreateTbRsp[i].pMeta); + } + } + taosArrayDestroy(pRsp->aCreateTbRsp); + } + } else if (flag & TSDB_MSG_FLG_DECODE) { + if (pRsp->aCreateTbRsp) { + int32_t nCreateTbRsp = TARRAY_SIZE(pRsp->aCreateTbRsp); + SVCreateTbRsp *aCreateTbRsp = TARRAY_DATA(pRsp->aCreateTbRsp); + for (int32_t i = 0; i < nCreateTbRsp; ++i) { + if (aCreateTbRsp[i].pMeta) { + taosMemoryFree(aCreateTbRsp[i].pMeta); + } + } + taosArrayDestroy(pRsp->aCreateTbRsp); + } + } +} diff --git a/source/common/src/tname.c b/source/common/src/tname.c index f21938ed29632817d84d6908af119a7fc1c73ffe..e5ed7a3728cebe78ae9cc24304e6e8ce2e94c7ef 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -90,10 +90,8 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName) { pName->type = TSDB_TABLE_NAME_T; pName->acctId = acctId; - memset(pName->dbname, 0, TSDB_DB_NAME_LEN); - strncpy(pName->dbname, pDbName, TSDB_DB_NAME_LEN - 1); - memset(pName->tname, 0, TSDB_TABLE_NAME_LEN); - strncpy(pName->tname, pTableName, TSDB_TABLE_NAME_LEN - 1); + snprintf(pName->dbname, sizeof(pName->dbname), "%s", pDbName); + snprintf(pName->tname, sizeof(pName->tname), "%s", pTableName); return pName; } @@ -162,9 +160,7 @@ int32_t tNameGetFullDbName(const SName* name, char* dst) { return 0; } -bool tNameIsEmpty(const SName* name) { - return name->type == 0 || name->acctId == 0; -} +bool tNameIsEmpty(const SName* name) { return name->type == 0 || name->acctId == 0; } const char* tNameGetTableName(const SName* name) { ASSERT(name != NULL && name->type == TSDB_TABLE_NAME_T); @@ -284,8 +280,8 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { } static int compareKv(const void* p1, const void* p2) { - SSmlKv* kv1 = *(SSmlKv**)p1; - SSmlKv* kv2 = *(SSmlKv**)p2; + SSmlKv* kv1 = (SSmlKv*)p1; + SSmlKv* kv2 = (SSmlKv*)p2; int32_t kvLen1 = kv1->keyLen; int32_t kvLen2 = kv2->keyLen; int32_t res = strncasecmp(kv1->key, kv2->key, TMIN(kvLen1, kvLen2)); @@ -302,11 +298,11 @@ static int compareKv(const void* p1, const void* p2) { void buildChildTableName(RandTableName* rName) { SStringBuilder sb = {0}; taosStringBuilderAppendStringLen(&sb, rName->stbFullName, rName->stbFullNameLen); - if(sb.buf == NULL) return; + if (sb.buf == NULL) return; taosArraySort(rName->tags, compareKv); for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) { taosStringBuilderAppendChar(&sb, ','); - SSmlKv* tagKv = taosArrayGetP(rName->tags, j); + SSmlKv* tagKv = taosArrayGet(rName->tags, j); taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); taosStringBuilderAppendChar(&sb, '='); if (IS_VAR_DATA_TYPE(tagKv->type)) { diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 969da4226fb3cf59e124432701d64a270f20c373..9d381ce15fa762b74cecf23eeec1fa6667a10b99 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -341,7 +341,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { } if (iColumn == 0) { - ASSERT(pColVal->cid == pTColumn->colId); + ASSERT(pColVal && pColVal->cid == pTColumn->colId); ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); ASSERT(pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID); } else { diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index b05ae602f8ee5299ed108756f73f648867d1f81a..4f8652d02cc0602583edff7f59354d9d093275fc 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -117,7 +117,7 @@ STSchema *genSTSchema(int16_t nCols) { } STSchema *pResult = NULL; - pResult = tdGetSTSChemaFromSSChema(pSchema, nCols, 1); + pResult = tBuildTSchema(pSchema, nCols, 1); taosMemoryFree(pSchema); return pResult; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 693fe97daaaa238f917132bc3c3282abba3c4dda..99ba9b9b3bcc1e0efe9d1a1bb987e01c89786374 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -343,13 +343,12 @@ static void vmCheckSyncTimeout(SVnodeMgmt *pMgmt) { int32_t numOfVnodes = 0; SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); - for (int32_t i = 0; i < numOfVnodes; ++i) { - SVnodeObj *pVnode = ppVnodes[i]; - vnodeSyncCheckTimeout(pVnode->pImpl); - vmReleaseVnode(pMgmt, pVnode); - } - if (ppVnodes != NULL) { + for (int32_t i = 0; i < numOfVnodes; ++i) { + SVnodeObj *pVnode = ppVnodes[i]; + vnodeSyncCheckTimeout(pVnode->pImpl); + vmReleaseVnode(pMgmt, pVnode); + } taosMemoryFree(ppVnodes); } } diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index 08330e025f2f496fdbe904450b2a9a659d6d0198..16931ab6dfa1a67f37792b19946c337461108062 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -109,9 +109,8 @@ static int32_t dmStartNodes(SDnode *pDnode) { } } - dInfo("TDengine initialized successfully"); - dmReportStartup("TDengine", "initialized successfully"); - + dInfo("The daemon initialized successfully"); + dmReportStartup("The daemon", "initialized successfully"); return 0; } @@ -143,7 +142,7 @@ int32_t dmRunDnode(SDnode *pDnode) { while (1) { if (pDnode->stop) { - dInfo("TDengine is about to stop"); + dInfo("The daemon is about to stop"); dmSetStatus(pDnode, DND_STAT_STOPPED); dmStopNodes(pDnode); dmCloseNodes(pDnode); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 3bbf4a4279ea5f90829aea0d5bb96a04366e3cb4..eeb4249217bbc08780939b10a2d7fb4b467226c3 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -133,7 +133,10 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; SMqConsumerRecoverMsg *pRecoverMsg = pMsg->pCont; SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pRecoverMsg->consumerId); - ASSERT(pConsumer); + if (pConsumer == NULL) { + mError("cannot find consumer %" PRId64 " when processing consumer recover msg", pRecoverMsg->consumerId); + return -1; + } mInfo("receive consumer recover msg, consumer id %" PRId64 ", status %s", pRecoverMsg->consumerId, mndConsumerStatusName(pConsumer->status)); @@ -381,8 +384,6 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { return -1; } - ASSERT(strcmp(req.cgroup, pConsumer->cgroup) == 0); - atomic_store_32(&pConsumer->hbStatus, 0); // 1. check consumer status @@ -428,9 +429,8 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { for (int32_t i = 0; i < numOfTopics; i++) { char *topic = taosArrayGetP(pConsumer->currentTopics, i); SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic); - // txn guarantees pSub is created - ASSERT(pSub); + taosRLockLatch(&pSub->lock); SMqSubTopicEp topicEp = {0}; @@ -438,7 +438,6 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { // 2.1 fetch topic schema SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); - ASSERT(pTopic); taosRLockLatch(&pTopic->lock); tstrncpy(topicEp.db, pTopic->db, TSDB_DB_FNAME_LEN); topicEp.schema.nCols = pTopic->schema.nCols; @@ -559,6 +558,10 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { goto SUBSCRIBE_OVER; } + if (mndCheckTopicPrivilege(pMnode, pMsg->info.conn.user, MND_OPER_SUBSCRIBE, pTopic) != 0) { + goto SUBSCRIBE_OVER; + } + mndReleaseTopic(pMnode, pTopic); } @@ -776,8 +779,8 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, taosWLockLatch(&pOldConsumer->lock); if (pNewConsumer->updateType == CONSUMER_UPDATE__MODIFY) { - ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); - ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0); + /*A(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);*/ + /*A(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);*/ if (taosArrayGetSize(pNewConsumer->rebNewTopics) == 0 && taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0) { pOldConsumer->status = MQ_CONSUMER_STATUS__READY; @@ -799,8 +802,8 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; } } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) { - ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); - ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0); + /*A(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);*/ + /*A(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);*/ int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics); /*pOldConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *));*/ @@ -813,8 +816,8 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, pOldConsumer->status = MQ_CONSUMER_STATUS__LOST; } else if (pNewConsumer->updateType == CONSUMER_UPDATE__RECOVER) { - ASSERT(taosArrayGetSize(pOldConsumer->currentTopics) == 0); - ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); + /*A(taosArrayGetSize(pOldConsumer->currentTopics) == 0);*/ + /*A(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);*/ int32_t sz = taosArrayGetSize(pOldConsumer->assignedTopics); for (int32_t i = 0; i < sz; i++) { @@ -831,15 +834,15 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, pOldConsumer->rebalanceTime = pNewConsumer->upTime; } else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) { - ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1); - ASSERT(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0); + /*A(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1);*/ + /*A(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0);*/ char *addedTopic = strdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0)); // not exist in current topic -#if 1 +#if 0 for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->currentTopics); i++) { char *topic = taosArrayGetP(pOldConsumer->currentTopics, i); - ASSERT(strcmp(topic, addedTopic) != 0); + A(strcmp(topic, addedTopic) != 0); } #endif @@ -880,15 +883,15 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, atomic_add_fetch_32(&pOldConsumer->epoch, 1); } else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) { - ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0); - ASSERT(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 1); + /*A(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0);*/ + /*A(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 1);*/ char *removedTopic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0); // not exist in new topic -#if 1 +#if 0 for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) { char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i); - ASSERT(strcmp(topic, removedTopic) != 0); + A(strcmp(topic, removedTopic) != 0); } #endif @@ -914,7 +917,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, } } // must find the topic - ASSERT(i < sz); + /*A(i < sz);*/ // set status if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) { diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index a5f77513de9f6de359acf129612136af1762fa68..8983d73c7078d09ec82cefc7d54592f9457330ef 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -167,6 +167,10 @@ void tFreeStreamObj(SStreamObj *pStream) { taosArrayDestroy(pLevel); } taosArrayDestroy(pStream->tasks); + // tagSchema.pSchema + if (pStream->tagSchema.nCols > 0) { + taosMemoryFree(pStream->tagSchema.pSchema); + } } SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) { diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index af1a29def08eae42072fffc889cd494a95dea239..bdef8000bd3b54d3428b0dc6336d5e5ad66893e6 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -115,13 +115,11 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream if (pStream->fixedSinkVgId == 0) { SDbObj* pDb = mndAcquireDb(pMnode, pStream->targetDb); - ASSERT(pDb); if (pDb->cfg.numOfVgroups > 1) { isShuffle = true; pTask->outputType = TASK_OUTPUT__SHUFFLE_DISPATCH; pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) { - ASSERT(0); return -1; } } @@ -140,9 +138,7 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream for (int32_t j = 0; j < sinkLvSize; j++) { SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j); if (pLastLevelTask->nodeId == pVgInfo->vgId) { - ASSERT(pVgInfo->vgId > 0); pVgInfo->taskId = pLastLevelTask->taskId; - ASSERT(pVgInfo->taskId != 0); break; } } @@ -152,7 +148,6 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; SArray* pArray = taosArrayGetP(pStream->tasks, 0); // one sink only - ASSERT(taosArrayGetSize(pArray) == 1); SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0); pTask->fixedEpDispatcher.taskId = lastLevelTask->taskId; pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId; @@ -170,7 +165,6 @@ int32_t mndAssignTaskToVg(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, co plan->execNode.epSet = pTask->epSet; if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { - ASSERT(0); terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } @@ -195,7 +189,6 @@ int32_t mndAssignTaskToSnode(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, plan->execNode.epSet = pTask->epSet; if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { - ASSERT(0); terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } @@ -222,8 +215,6 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { void* pIter = NULL; SArray* tasks = taosArrayGetP(pStream->tasks, 0); - ASSERT(taosArrayGetSize(pStream->tasks) == 1); - while (1) { SVgObj* pVgroup = NULL; pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); @@ -257,7 +248,10 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { pTask->tbSink.stbUid = pStream->targetStbUid; memcpy(pTask->tbSink.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); - ASSERT(pTask->tbSink.pSchemaWrapper); + if (pTask->tbSink.pSchemaWrapper == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } } sdbRelease(pSdb, pVgroup); } @@ -265,7 +259,6 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { } int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, SStreamObj* pStream) { - ASSERT(pStream->fixedSinkVgId != 0); SArray* tasks = taosArrayGetP(pStream->tasks, 0); SStreamTask* pTask = tNewSStreamTask(pStream->uid); if (pTask == NULL) { @@ -275,8 +268,6 @@ int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, SStreamObj* pStream) { pTask->fillHistory = pStream->fillHistory; mndAddTaskToTaskSet(tasks, pTask); - ASSERT(pStream->fixedSinkVg.vgId == pStream->fixedSinkVgId); - pTask->nodeId = pStream->fixedSinkVgId; #if 0 SVgObj* pVgroup = mndAcquireVgroup(pMnode, pStream->fixedSinkVgId); @@ -311,13 +302,16 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { return -1; } int32_t planTotLevel = LIST_LENGTH(pPlan->pSubplans); - ASSERT(planTotLevel <= 2); + pStream->tasks = taosArrayInit(planTotLevel, sizeof(void*)); bool hasExtraSink = false; bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0; SDbObj* pDbObj = mndAcquireDb(pMnode, pStream->targetDb); - ASSERT(pDbObj != NULL); + if (pDbObj == NULL) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } bool multiTarget = pDbObj->cfg.numOfVgroups > 1; sdbRelease(pSdb, pDbObj); @@ -351,7 +345,10 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); - ASSERT(plan->subplanType == SUBPLAN_TYPE_MERGE); + if (plan->subplanType != SUBPLAN_TYPE_MERGE) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } pInnerTask = tNewSStreamTask(pStream->uid); if (pInnerTask == NULL) { @@ -409,7 +406,10 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 1); SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); - ASSERT(plan->subplanType == SUBPLAN_TYPE_SCAN); + if (plan->subplanType != SUBPLAN_TYPE_SCAN) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } void* pIter = NULL; while (1) { @@ -471,9 +471,15 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { taosArrayPush(pStream->tasks, &taskOneLevel); SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); - ASSERT(LIST_LENGTH(inner->pNodeList) == 1); + if (LIST_LENGTH(inner->pNodeList) != 1) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); - ASSERT(plan->subplanType == SUBPLAN_TYPE_SCAN); + if (plan->subplanType != SUBPLAN_TYPE_SCAN) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } void* pIter = NULL; while (1) { @@ -550,9 +556,6 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); } - ASSERT(pSub->unassignedVgs); - ASSERT(taosHashGetSize(pSub->consumerHash) == 0); - void* pIter = NULL; while (1) { pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); @@ -590,10 +593,6 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib sdbRelease(pSdb, pVgroup); } - ASSERT(pSub->unassignedVgs->size > 0); - - ASSERT(taosHashGetSize(pSub->consumerHash) == 0); - qDestroyQueryPlan(pPlan); return 0; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 141bb1df601fe7d1e3c27090170e37d6769d879f..fe0dc9e8578cd9afc46900e0530eb28788fa5c0b 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -202,11 +202,13 @@ static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) { _OVER: if (terrno != 0) { - mError("sma:%s, failed to decode from raw:%p since %s", pSma == NULL ? "null" : pSma->name, pRaw, terrstr()); - taosMemoryFreeClear(pSma->expr); - taosMemoryFreeClear(pSma->tagsFilter); - taosMemoryFreeClear(pSma->sql); - taosMemoryFreeClear(pSma->ast); + if (pSma != NULL) { + mError("sma:%s, failed to decode from raw:%p since %s", pSma->name, pRaw, terrstr()); + taosMemoryFreeClear(pSma->expr); + taosMemoryFreeClear(pSma->tagsFilter); + taosMemoryFreeClear(pSma->sql); + taosMemoryFreeClear(pSma->ast); + } taosMemoryFreeClear(pRow); return NULL; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 83ed6eea78531eb0749aa98b97b759af5fc829d6..6b54a36a6fc3b4135fb19f1bbe40ee13cdecf7c5 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -314,7 +314,11 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, } tstrncpy(pObj->targetDb, pTargetDb->name, TSDB_DB_FNAME_LEN); - pObj->targetStbUid = mndGenerateUid(pObj->targetSTbName, TSDB_TABLE_FNAME_LEN); + if (pCreate->createStb == STREAM_CREATE_STABLE_TRUE) { + pObj->targetStbUid = mndGenerateUid(pObj->targetSTbName, TSDB_TABLE_FNAME_LEN); + } else { + pObj->targetStbUid = pCreate->targetStbUid; + } pObj->targetDbUid = pTargetDb->uid; mndReleaseDb(pMnode, pTargetDb); @@ -326,16 +330,46 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, // deserialize ast if (nodesStringToNode(pObj->ast, &pAst) < 0) { - /*ASSERT(0);*/ goto FAIL; } // extract output schema from ast if (qExtractResultSchema(pAst, (int32_t *)&pObj->outputSchema.nCols, &pObj->outputSchema.pSchema) != 0) { - /*ASSERT(0);*/ goto FAIL; } + int32_t numOfNULL = taosArrayGetSize(pCreate->fillNullCols); + if(numOfNULL > 0) { + pObj->outputSchema.nCols += numOfNULL; + SSchema* pFullSchema = taosMemoryCalloc(pObj->outputSchema.nCols, sizeof(SSchema)); + if (!pFullSchema) { + goto FAIL; + } + + int32_t nullIndex = 0; + int32_t dataIndex = 0; + for (int16_t i = 0; i < pObj->outputSchema.nCols; i++) { + SColLocation* pos = taosArrayGet(pCreate->fillNullCols, nullIndex); + if (i < pos->slotId) { + pFullSchema[i].bytes = pObj->outputSchema.pSchema[dataIndex].bytes; + pFullSchema[i].colId = i + 1; // pObj->outputSchema.pSchema[dataIndex].colId; + pFullSchema[i].flags = pObj->outputSchema.pSchema[dataIndex].flags; + strcpy(pFullSchema[i].name, pObj->outputSchema.pSchema[dataIndex].name); + pFullSchema[i].type = pObj->outputSchema.pSchema[dataIndex].type; + dataIndex++; + } else { + pFullSchema[i].bytes = 0; + pFullSchema[i].colId = pos->colId; + pFullSchema[i].flags = COL_SET_NULL; + memset(pFullSchema[i].name, 0, TSDB_COL_NAME_LEN); + pFullSchema[i].type = pos->type; + nullIndex++; + } + } + taosMemoryFree(pObj->outputSchema.pSchema); + pObj->outputSchema.pSchema = pFullSchema; + } + SPlanContext cxt = { .pAstRoot = pAst, .topicQuery = false, @@ -347,13 +381,11 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, // using ast and param to build physical plan if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) { - /*ASSERT(0);*/ goto FAIL; } // save physcial plan if (nodesNodeToString((SNode *)pPlan, false, &pObj->physicalPlan, NULL) != 0) { - /*ASSERT(0);*/ goto FAIL; } @@ -361,7 +393,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, if (pCreate->numOfTags) { pObj->tagSchema.pSchema = taosMemoryCalloc(pCreate->numOfTags, sizeof(SSchema)); } - ASSERT(pCreate->numOfTags == taosArrayGetSize(pCreate->pTags)); + /*A(pCreate->numOfTags == taosArrayGetSize(pCreate->pTags));*/ for (int32_t i = 0; i < pCreate->numOfTags; i++) { SField *pField = taosArrayGet(pCreate->pTags, i); pObj->tagSchema.pSchema[i].colId = pObj->outputSchema.nCols + i + 1; @@ -378,9 +410,6 @@ FAIL: } int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) { - if (pTask->taskLevel == TASK_LEVEL__AGG) { - ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0); - } SEncoder encoder; tEncoderInit(&encoder, NULL, 0); tEncodeSStreamTask(&encoder, pTask); @@ -472,7 +501,6 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre SMCreateStbReq createReq = {0}; tstrncpy(createReq.name, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); createReq.numOfColumns = pStream->outputSchema.nCols; - createReq.numOfTags = 1; // group id createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField)); // build fields taosArraySetSize(createReq.pColumns, createReq.numOfColumns); @@ -483,14 +511,29 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre pField->type = pStream->outputSchema.pSchema[i].type; pField->bytes = pStream->outputSchema.pSchema[i].bytes; } - createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField)); - taosArraySetSize(createReq.pTags, 1); - // build tags - SField *pField = taosArrayGet(createReq.pTags, 0); - strcpy(pField->name, "group_id"); - pField->type = TSDB_DATA_TYPE_UBIGINT; - pField->flags = 0; - pField->bytes = 8; + + if (pStream->tagSchema.nCols == 0) { + createReq.numOfTags = 1; + createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField)); + taosArraySetSize(createReq.pTags, createReq.numOfTags); + // build tags + SField *pField = taosArrayGet(createReq.pTags, 0); + strcpy(pField->name, "group_id"); + pField->type = TSDB_DATA_TYPE_UBIGINT; + pField->flags = 0; + pField->bytes = 8; + } else { + createReq.numOfTags = pStream->tagSchema.nCols; + createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField)); + taosArraySetSize(createReq.pTags, createReq.numOfTags); + for (int32_t i = 0; i < createReq.numOfTags; i++) { + SField *pField = taosArrayGet(createReq.pTags, i); + pField->bytes = pStream->tagSchema.pSchema[i].bytes; + pField->flags = pStream->tagSchema.pSchema[i].flags; + pField->type = pStream->tagSchema.pSchema[i].type; + tstrncpy(pField->name, pStream->tagSchema.pSchema[i].name, TSDB_COL_NAME_LEN); + } + } if (mndCheckCreateStbReq(&createReq) != 0) { goto _OVER; @@ -545,8 +588,6 @@ _OVER: } static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { - ASSERT(pTask->nodeId != 0); - // vnode /*if (pTask->nodeId > 0) {*/ SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq)); @@ -646,7 +687,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; // create stb for stream - if (mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user) < 0) { + if (createStreamReq.createStb == STREAM_CREATE_STABLE_TRUE && mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user) < 0) { mError("trans:%d, failed to create stb for stream %s since %s", pTrans->id, createStreamReq.name, terrstr()); mndTransDrop(pTrans); goto _OVER; @@ -808,10 +849,9 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { int32_t sz = taosArrayGetSize(pLevel); for (int32_t j = 0; j < sz; j++) { SStreamTask *pTask = taosArrayGetP(pLevel, j); - ASSERT(pTask->nodeId > 0); + /*A(pTask->nodeId > 0);*/ SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->nodeId); if (pVgObj == NULL) { - ASSERT(0); taosRUnLockLatch(&pStream->lock); mndReleaseStream(pMnode, pStream); mndTransDrop(pTrans); @@ -871,7 +911,6 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { SMDropStreamReq dropReq = {0}; if (tDeserializeSMDropStreamReq(pReq->pCont, pReq->contLen, &dropReq) < 0) { - ASSERT(0); terrno = TSDB_CODE_INVALID_MSG; return -1; } diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index c429d49fef049c494c9d0665bb49c360826367f4..6b675586e47a9116c9043991c17fa58984d0a44f 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -271,9 +271,11 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { int32_t mndInitSync(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; taosThreadMutexInit(&pMgmt->lock, NULL); + taosThreadMutexLock(&pMgmt->lock); pMgmt->transId = 0; pMgmt->transSec = 0; pMgmt->transSeq = 0; + taosThreadMutexUnlock(&pMgmt->lock); SSyncInfo syncInfo = { .snapshotStrategy = SYNC_STRATEGY_STANDARD_SNAPSHOT, diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 48c35f3f0704780b73e9d3af9b3ab31e0c58da63..96dba24566d8a6f21ea19672d728b795ba0245d7 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -696,14 +696,9 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { sdbRelease(pSdb, pConsumer); } -#if 0 - if (pTopic->refConsumerCnt != 0) { - mndReleaseTopic(pMnode, pTopic); - terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED; - mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) { return -1; } -#endif STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq, "drop-topic"); if (pTrans == NULL) { diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 2550c68cfb408843de163b5ff6d78b1fcf726fdb..54ea9e7b2437b303088fcdd9d003eb1aa19af8e5 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -1441,10 +1441,10 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, { SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL) return -1; + if (pRaw == NULL) goto _OVER; if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { sdbFreeRaw(pRaw); - return -1; + goto _OVER; } (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index e5e7fea1cfdb12a46fc3126ce8fe5df85212acea..b1093e0f23e15ab5330028f0690d3e2aa124704d 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -178,6 +178,7 @@ void tsdbReaderClose(STsdbReader *pReader); bool tsdbNextDataBlock(STsdbReader *pReader); void tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow); int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave); +void tsdbReleaseDataBlock(STsdbReader *pReader); SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); @@ -224,11 +225,19 @@ typedef struct SSnapContext { } SSnapContext; typedef struct STqReader { - int64_t ver; - const SSubmitReq *pMsg; - SSubmitBlk *pBlock; - SSubmitMsgIter msgIter; - SSubmitBlkIter blkIter; + // const SSubmitReq *pMsg; + // SSubmitBlk *pBlock; + // SSubmitMsgIter msgIter; + // SSubmitBlkIter blkIter; + + int64_t ver; + SPackedData msg2; + + int8_t setMsg; + SSubmitReq2 submit; + int32_t nextBlk; + + int64_t lastBlkUid; SWalReader *pWalReader; @@ -253,11 +262,14 @@ int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqSeekVer(STqReader *pReader, int64_t ver); int32_t tqNextBlock(STqReader *pReader, SFetchRet *ret); -int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t ver); -bool tqNextDataBlock(STqReader *pReader); -bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); -int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader); -int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas); +int32_t tqReaderSetSubmitReq2(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); +// int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t ver); +bool tqNextDataBlock2(STqReader *pReader); +bool tqNextDataBlockFilterOut2(STqReader *pReader, SHashObj *filterOutUids); +int32_t tqRetrieveDataBlock2(SSDataBlock *pBlock, STqReader *pReader); +int32_t tqRetrieveTaosxBlock2(STqReader *pReader, SArray *blocks, SArray *schemas); +// int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader); +// int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas); int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg); diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 50e3c3a9c16e54a70d2b4f07a4640d87013a7776..007207562380f1e29ca7309025bf774dff27e333 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -44,7 +44,6 @@ typedef struct SRSmaInfoItem SRSmaInfoItem; typedef struct SRSmaFS SRSmaFS; typedef struct SQTaskFile SQTaskFile; typedef struct SQTaskFReader SQTaskFReader; -typedef struct SQTaskFWriter SQTaskFWriter; struct SSmaEnv { SRWLatch lock; @@ -85,22 +84,20 @@ struct STSmaStat { struct SQTaskFile { volatile int32_t nRef; - int32_t padding; + int8_t level; + int64_t suid; int64_t version; int64_t size; + int64_t mtime; }; struct SQTaskFReader { SSma *pSma; + int8_t level; + int64_t suid; int64_t version; TdFilePtr pReadH; }; -struct SQTaskFWriter { - SSma *pSma; - int64_t version; - TdFilePtr pWriteH; - char *fname; -}; struct SRSmaFS { SArray *aQTaskInf; // array of SQTaskFile @@ -214,85 +211,40 @@ static FORCE_INLINE void tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) { // rsma void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree); -int32_t tdRSmaFSOpen(SSma *pSma, int64_t version); +int32_t tdRSmaFSOpen(SSma *pSma, int64_t version, int8_t rollback); void tdRSmaFSClose(SRSmaFS *fs); -int32_t tdRSmaFSRef(SSma *pSma, SRSmaStat *pStat, int64_t version); -void tdRSmaFSUnRef(SSma *pSma, SRSmaStat *pStat, int64_t version); -int64_t tdRSmaFSMaxVer(SSma *pSma, SRSmaStat *pStat); -int32_t tdRSmaFSUpsertQTaskFile(SRSmaFS *pFS, SQTaskFile *qTaskFile); -int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer); +int32_t tdRSmaFSPrepareCommit(SSma *pSma, SRSmaFS *pFSNew); +int32_t tdRSmaFSCommit(SSma *pSma); +int32_t tdRSmaFSFinishCommit(SSma *pSma); +int32_t tdRSmaFSCopy(SSma *pSma, SRSmaFS *pFS); +int32_t tdRSmaFSTakeSnapshot(SSma *pSma, SRSmaFS *pFS); +int32_t tdRSmaFSRef(SSma *pSma, SRSmaFS *pFS); +void tdRSmaFSUnRef(SSma *pSma, SRSmaFS *pFS); +int32_t tdRSmaFSUpsertQTaskFile(SSma *pSma, SRSmaFS *pFS, SQTaskFile *qTaskFile, int32_t nSize); +int32_t tdRSmaFSRollback(SSma *pSma); +int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer, int8_t rollback); int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type); int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); -int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer); -void tdRSmaQTaskInfoGetFileName(int32_t vgId, int64_t version, char *outputName); -void tdRSmaQTaskInfoGetFullName(int32_t vgId, int64_t version, const char *path, char *outputName); +int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer, int8_t rollback); +void tdRSmaQTaskInfoGetFileName(int32_t vgId, int64_t suid, int8_t level, int64_t version, char *outputName); +void tdRSmaQTaskInfoGetFullName(int32_t vgId, int64_t suid, int8_t level, int64_t version, const char *path, + char *outputName); void tdRSmaQTaskInfoGetFullPath(int32_t vgId, int8_t level, const char *path, char *outputName); void tdRSmaQTaskInfoGetFullPathEx(int32_t vgId, tb_uid_t suid, int8_t level, const char *path, char *outputName); static FORCE_INLINE void tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { int32_t ref = T_REF_INC(pRSmaInfo); - smaDebug("vgId:%d, ref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); + smaTrace("vgId:%d, ref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); } static FORCE_INLINE void tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { int32_t ref = T_REF_DEC(pRSmaInfo); - smaDebug("vgId:%d, unref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); + smaTrace("vgId:%d, unref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); } -// smaFileUtil ================ - -#define TD_FILE_HEAD_SIZE 512 - -typedef struct STFInfo STFInfo; -typedef struct STFile STFile; - -struct STFInfo { - // common fields - uint32_t magic; - uint32_t ftype; - uint32_t fver; - int64_t fsize; -}; - -enum { - TD_FTYPE_RSMA_QTASKINFO = 0, -}; - -#if 0 -struct STFile { - uint8_t state; - STFInfo info; - char *fname; - TdFilePtr pFile; -}; - -#define TD_TFILE_PFILE(tf) ((tf)->pFile) -#define TD_TFILE_OPENED(tf) (TD_TFILE_PFILE(tf) != NULL) -#define TD_TFILE_FULL_NAME(tf) ((tf)->fname) -#define TD_TFILE_OPENED(tf) (TD_TFILE_PFILE(tf) != NULL) -#define TD_TFILE_CLOSED(tf) (!TD_TFILE_OPENED(tf)) -#define TD_TFILE_SET_CLOSED(f) (TD_TFILE_PFILE(f) = NULL) -#define TD_TFILE_SET_STATE(tf, s) ((tf)->state = (s)) - -int32_t tdInitTFile(STFile *pTFile, const char *dname, const char *fname); -int32_t tdCreateTFile(STFile *pTFile, bool updateHeader, int8_t fType); -int32_t tdOpenTFile(STFile *pTFile, int flags); -int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte); -int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence); -int64_t tdWriteTFile(STFile *pTFile, void *buf, int64_t nbyte); -int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset); -int64_t tdGetTFileSize(STFile *pTFile, int64_t *size); -int32_t tdRemoveTFile(STFile *pTFile); -int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo); -int32_t tdUpdateTFileHeader(STFile *pTFile); -void tdUpdateTFileMagic(STFile *pTFile, void *pCksm); -void tdCloseTFile(STFile *pTFile); -void tdDestroyTFile(STFile *pTFile); -#endif - -void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version, - char *outputName); -void tdGetVndDirName(int32_t vgId, const char *pdname, const char *dname, bool endWithSep, char *outputName); +void tdRSmaGetFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t suid, + int8_t level, int64_t version, char *outputName); +void tdRSmaGetDirName(int32_t vgId, const char *pdname, const char *dname, bool endWithSep, char *outputName); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 828341ddd8666bbb55c2cedce1b34c50f6b3362a..104e0945ba978815e38f17ecf107ff73c2f2905a 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -154,7 +154,8 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum); // tqExec -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp); +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp); +// int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp); int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision); int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp); int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry); @@ -181,8 +182,9 @@ int32_t tqOffsetDelete(STqOffsetStore* pStore, const char* subscribeKey) int32_t tqOffsetCommitFile(STqOffsetStore* pStore); // tqSink -// void tqSinkToTableMerge(SStreamTask* pTask, void* vnode, int64_t ver, void* data); -void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* data); +int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBlock* pDataBlock, + SBatchDeleteReq* deleteReq); +void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* data); // tqOffset char* tqOffsetBuildFName(const char* path, int32_t fVer); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 5a2e462c8cd69598ee8642c7109983c97c034294..8c8775548d21702a31d4adc7e9d7f1c6aa30fa4c 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -70,6 +70,9 @@ typedef struct SDiskData SDiskData; typedef struct SDiskDataBuilder SDiskDataBuilder; typedef struct SBlkInfo SBlkInfo; +#define TSDBROW_ROW_FMT ((int8_t)0x0) +#define TSDBROW_COL_FMT ((int8_t)0x1) + #define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_MAX_SUBBLOCKS 8 #define TSDB_FHDR_SIZE 512 @@ -102,26 +105,29 @@ static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) { // tsdbUtil.c ============================================================================================== // TSDBROW -#define TSDBROW_TS(ROW) (((ROW)->type == 0) ? (ROW)->pTSRow->ts : (ROW)->pBlockData->aTSKEY[(ROW)->iRow]) -#define TSDBROW_VERSION(ROW) (((ROW)->type == 0) ? (ROW)->version : (ROW)->pBlockData->aVersion[(ROW)->iRow]) -#define TSDBROW_SVERSION(ROW) TD_ROW_SVER((ROW)->pTSRow) -#define TSDBROW_KEY(ROW) ((TSDBKEY){.version = TSDBROW_VERSION(ROW), .ts = TSDBROW_TS(ROW)}) -#define tsdbRowFromTSRow(VERSION, TSROW) ((TSDBROW){.type = 0, .version = (VERSION), .pTSRow = (TSROW)}) -#define tsdbRowFromBlockData(BLOCKDATA, IROW) ((TSDBROW){.type = 1, .pBlockData = (BLOCKDATA), .iRow = (IROW)}) -void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); -// int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); +#define TSDBROW_TS(ROW) (((ROW)->type == TSDBROW_ROW_FMT) ? (ROW)->pTSRow->ts : (ROW)->pBlockData->aTSKEY[(ROW)->iRow]) +#define TSDBROW_VERSION(ROW) \ + (((ROW)->type == TSDBROW_ROW_FMT) ? (ROW)->version : (ROW)->pBlockData->aVersion[(ROW)->iRow]) +#define TSDBROW_SVERSION(ROW) ((ROW)->type == TSDBROW_ROW_FMT ? (ROW)->pTSRow->sver : -1) +#define TSDBROW_KEY(ROW) ((TSDBKEY){.version = TSDBROW_VERSION(ROW), .ts = TSDBROW_TS(ROW)}) +#define tsdbRowFromTSRow(VERSION, TSROW) ((TSDBROW){.type = TSDBROW_ROW_FMT, .version = (VERSION), .pTSRow = (TSROW)}) +#define tsdbRowFromBlockData(BLOCKDATA, IROW) \ + ((TSDBROW){.type = TSDBROW_COL_FMT, .pBlockData = (BLOCKDATA), .iRow = (IROW)}) + +void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); int32_t tsdbRowCmprFn(const void *p1, const void *p2); // STSDBRowIter -void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); +int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); +void tsdbRowClose(STSDBRowIter *pIter); SColVal *tsdbRowIterNext(STSDBRowIter *pIter); // SRowMerger -int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); -int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); +int32_t tsdbRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); +int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); -int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); -void tRowMergerClear(SRowMerger *pMerger); -int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow); -int32_t tRowMergerGetRow(SRowMerger *pMerger, STSRow **ppRow); +int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); +void tsdbRowMergerClear(SRowMerger *pMerger); +int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow); +int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow); // TABLEID int32_t tTABLEIDCmprFn(const void *p1, const void *p2); // TSDBKEY @@ -146,24 +152,22 @@ int32_t tGetBlockIdx(uint8_t *p, void *ph); int32_t tCmprBlockIdx(void const *lhs, void const *rhs); int32_t tCmprBlockL(void const *lhs, void const *rhs); // SBlockData -#define tBlockDataFirstRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, 0) -#define tBlockDataLastRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1) -#define tBlockDataFirstKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataFirstRow(PBLOCKDATA)) -#define tBlockDataLastKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA)) - -int32_t tBlockDataCreate(SBlockData *pBlockData); -void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear); -int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid); -void tBlockDataReset(SBlockData *pBlockData); -int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); -void tBlockDataClear(SBlockData *pBlockData); -SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx); -void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); -int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); -int32_t tBlockDataAddColData(SBlockData *pBlockData, SColData **ppColData); -int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[], - int32_t aBufN[]); -int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]); +#define tBlockDataFirstRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, 0) +#define tBlockDataLastRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1) +#define tBlockDataFirstKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataFirstRow(PBLOCKDATA)) +#define tBlockDataLastKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA)) +#define tBlockDataGetColDataByIdx(PBLOCKDATA, IDX) (&(PBLOCKDATA)->aColData[IDX]) + +int32_t tBlockDataCreate(SBlockData *pBlockData); +void tBlockDataDestroy(SBlockData *pBlockData); +int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid); +void tBlockDataReset(SBlockData *pBlockData); +int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); +void tBlockDataClear(SBlockData *pBlockData); +void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); +int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[], + int32_t aBufN[]); +int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]); // SDiskDataHdr int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr); int32_t tGetDiskDataHdr(uint8_t *p, void *ph); @@ -205,10 +209,10 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in // tsdbMemTable ============================================================================================== // SMemTable int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); -void tsdbMemTableDestroy(SMemTable *pMemTable); +void tsdbMemTableDestroy(SMemTable *pMemTable, bool proactive); STbData *tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid); -void tsdbRefMemTable(SMemTable *pMemTable); -void tsdbUnrefMemTable(SMemTable *pMemTable); +int32_t tsdbRefMemTable(SMemTable *pMemTable, SQueryNode *pQNode); +int32_t tsdbUnrefMemTable(SMemTable *pMemTable, SQueryNode *pNode, bool proactive); SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable); // STbDataIter int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter); @@ -286,8 +290,8 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader); int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData); int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); // tsdbRead.c ============================================================================================== -int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap, const char *id); -void tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap, const char *id); +int32_t tsdbTakeReadSnap(STsdbReader *pReader, _query_reseek_func_t reseek, STsdbReadSnap **ppSnap); +void tsdbUntakeReadSnap(STsdbReader *pReader, STsdbReadSnap *pSnap, bool proactive); // tsdbMerge.c ============================================================================================== int32_t tsdbMerge(STsdb *pTsdb); @@ -335,10 +339,13 @@ struct SVersionRange { typedef struct SMemSkipListNode SMemSkipListNode; struct SMemSkipListNode { int8_t level; + int8_t flag; // TSDBROW_ROW_FMT for row format, TSDBROW_COL_FMT for col format + int32_t iRow; int64_t version; - STSRow *pTSRow; + void *pData; SMemSkipListNode *forwards[0]; }; + typedef struct SMemSkipList { int64_t size; uint32_t seed; @@ -364,6 +371,8 @@ struct SMemTable { STsdb *pTsdb; SVBufPool *pPool; volatile int32_t nRef; + int64_t minVer; + int64_t maxVer; TSKEY minKey; TSKEY maxKey; int64_t nRow; @@ -376,11 +385,11 @@ struct SMemTable { }; struct TSDBROW { - int8_t type; // 0 for row from tsRow, 1 for row from block data + int8_t type; // TSDBROW_ROW_FMT for row from tsRow, TSDBROW_COL_FMT for row from block data union { struct { int64_t version; - STSRow *pTSRow; + SRow *pTSRow; }; struct { SBlockData *pBlockData; @@ -466,14 +475,14 @@ struct SSttBlk { // (SBlockData){.suid = suid, .uid = 0}: block data for N child tables int .last file // (SBlockData){.suid = 0, .uid = uid}: block data for 1 normal table int .last/.data file struct SBlockData { - int64_t suid; // 0 means normal table block data, otherwise child table block data - int64_t uid; // 0 means block data in .last file, otherwise in .data file - int32_t nRow; // number of rows - int64_t *aUid; // uids of each row, only exist in block data in .last file (uid == 0) - int64_t *aVersion; // versions of each row - TSKEY *aTSKEY; // timestamp of each row - int32_t nColData; - SArray *aColData; // SArray + int64_t suid; // 0 means normal table block data, otherwise child table block data + int64_t uid; // 0 means block data in .last file, otherwise in .data file + int32_t nRow; // number of rows + int64_t *aUid; // uids of each row, only exist in block data in .last file (uid == 0) + int64_t *aVersion; // versions of each row + TSKEY *aTSKEY; // timestamp of each row + int32_t nColData; + SColData *aColData; }; struct TABLEID { @@ -565,10 +574,14 @@ struct SDFileSet { }; struct STSDBRowIter { - TSDBROW *pRow; - STSchema *pTSchema; - SColVal colVal; - int32_t i; + TSDBROW *pRow; + union { + SRowIter *pIter; + struct { + int32_t iColData; + SColVal cv; + }; + }; }; struct SRowMerger { STSchema *pTSchema; @@ -594,9 +607,11 @@ struct SDelFWriter { }; struct STsdbReadSnap { - SMemTable *pMem; - SMemTable *pIMem; - STsdbFS fs; + SMemTable *pMem; + SQueryNode *pNode; + SMemTable *pIMem; + SQueryNode *pINode; + STsdbFS fs; }; struct SDataFWriter { @@ -715,6 +730,9 @@ void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); // tsdbCache ============================================================================================== typedef struct SCacheRowsReader { + STsdb *pTsdb; + SVersionRange verRange; + TdThreadMutex readerMutex; SVnode *pVnode; STSchema *pSchema; uint64_t uid; @@ -739,8 +757,8 @@ typedef struct { int32_t tsdbOpenCache(STsdb *pTsdb); void tsdbCloseCache(STsdb *pTsdb); -int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb); -int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, STSRow *row, bool dup); +int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, TSDBROW *row, STsdb *pTsdb); +int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, TSDBROW *row, bool dup); int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, LRUHandle **h); int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, LRUHandle **h); int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h); @@ -752,6 +770,8 @@ int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); size_t tsdbCacheGetCapacity(SVnode *pVnode); +// int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema); + // ========== inline functions ========== static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { TSDBKEY *pKey1 = (TSDBKEY *)p1; @@ -793,8 +813,13 @@ static FORCE_INLINE TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) { } pIter->pRow = &pIter->row; - pIter->pRow->version = pIter->pNode->version; - pIter->pRow->pTSRow = pIter->pNode->pTSRow; + if (pIter->pNode->flag == TSDBROW_ROW_FMT) { + pIter->row = tsdbRowFromTSRow(pIter->pNode->version, pIter->pNode->pData); + } else if (pIter->pNode->flag == TSDBROW_COL_FMT) { + pIter->row = tsdbRowFromBlockData(pIter->pNode->pData, pIter->pNode->iRow); + } else { + ASSERT(0); + } return pIter->pRow; } diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index e075d8a6ca0af9da2a0ca7f12eeb834599fce3e5..e0d0e9408b1dde6edf29b895a4c3c916105de646 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -61,10 +61,19 @@ struct SVBufPoolNode { }; struct SVBufPool { - SVBufPool* next; + SVBufPool* freeNext; + SVBufPool* recycleNext; + SVBufPool* recyclePrev; + + // query handle list + TdThreadMutex mutex; + int32_t nQuery; + SQueryNode qList; + SVnode* pVnode; - TdThreadSpinlock* lock; + int32_t id; volatile int32_t nRef; + TdThreadSpinlock* lock; int64_t size; uint8_t* ptr; SVBufPoolNode* pTail; @@ -74,6 +83,8 @@ struct SVBufPool { int32_t vnodeOpenBufPool(SVnode* pVnode); int32_t vnodeCloseBufPool(SVnode* pVnode); void vnodeBufPoolReset(SVBufPool* pPool); +void vnodeBufPoolAddToFreeList(SVBufPool* pPool); +int32_t vnodeBufPoolRecycle(SVBufPool* pPool); // vnodeQuery.c int32_t vnodeQueryOpen(SVnode* pVnode); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 75367883f1d82edf349bbccbee5552dcf299a003..9c5f47b731e2393aca1d21099fbfec6373d97cd0 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -76,6 +76,7 @@ typedef struct SRSmaSnapReader SRSmaSnapReader; typedef struct SRSmaSnapWriter SRSmaSnapWriter; typedef struct SSnapDataHdr SSnapDataHdr; typedef struct SCommitInfo SCommitInfo; +typedef struct SQueryNode SQueryNode; #define VNODE_META_DIR "meta" #define VNODE_TSDB_DIR "tsdb" @@ -87,16 +88,29 @@ typedef struct SCommitInfo SCommitInfo; #define VNODE_RSMA1_DIR "rsma1" #define VNODE_RSMA2_DIR "rsma2" +#define VNODE_BUFPOOL_SEGMENTS 3 + #define VND_INFO_FNAME "vnode.json" // vnd.h +typedef int32_t (*_query_reseek_func_t)(void* pQHandle); +struct SQueryNode { + SQueryNode* pNext; + SQueryNode** ppNext; + void* pQHandle; + _query_reseek_func_t reseek; +}; + void* vnodeBufPoolMalloc(SVBufPool* pPool, int size); void* vnodeBufPoolMallocAligned(SVBufPool* pPool, int size); void vnodeBufPoolFree(SVBufPool* pPool, void* p); void vnodeBufPoolRef(SVBufPool* pPool); -void vnodeBufPoolUnRef(SVBufPool* pPool); +void vnodeBufPoolUnRef(SVBufPool* pPool, bool proactive); int vnodeDecodeInfo(uint8_t* pData, SVnodeInfo* pInfo); +int32_t vnodeBufPoolRegisterQuery(SVBufPool* pPool, SQueryNode* pQNode); +void vnodeBufPoolDeregisterQuery(SVBufPool* pPool, SQueryNode* pQNode, bool proactive); + // meta typedef struct SMCtbCursor SMCtbCursor; typedef struct SMStbCursor SMStbCursor; @@ -162,10 +176,9 @@ int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo); int32_t tsdbFinishCommit(STsdb* pTsdb); int32_t tsdbRollbackCommit(STsdb* pTsdb); int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now); -int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg); -int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); -int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, - SSubmitBlkRsp* pRsp); +int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg); +int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp); +int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows); int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey); int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg); @@ -190,7 +203,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msg int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t version, char* msg, int32_t msgLen); -int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* data, int64_t ver); +int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit); int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver); int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec); @@ -203,19 +216,12 @@ int32_t tqProcessTaskRecoverFinishReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRecoverFinishRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqCheckLogInWal(STQ* pTq, int64_t version); -SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pSchema, - SSchemaWrapper* pTagSchemaWrapper, bool createTb, int64_t suid, const char* stbFullName, - SBatchDeleteReq* pDeleteReq); - // sma int32_t smaInit(); void smaCleanUp(); int32_t smaOpen(SVnode* pVnode, int8_t rollback); int32_t smaClose(SSma* pSma); int32_t smaBegin(SSma* pSma); -int32_t smaSyncPreCommit(SSma* pSma); -int32_t smaSyncCommit(SSma* pSma); -int32_t smaSyncPostCommit(SSma* pSma); int32_t smaPrepareAsyncCommit(SSma* pSma); int32_t smaCommit(SSma* pSma, SCommitInfo* pInfo); int32_t smaFinishCommit(SSma* pSma); @@ -226,11 +232,10 @@ int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); int32_t tdProcessRSmaCreate(SSma* pSma, SVCreateStbReq* pReq); -int32_t tdProcessRSmaSubmit(SSma* pSma, void* pMsg, int32_t inputType); +int32_t tdProcessRSmaSubmit(SSma* pSma, int64_t version, void* pReq, void* pMsg, int32_t len, int32_t inputType); int32_t tdProcessRSmaDrop(SSma* pSma, SVDropStbReq* pReq); int32_t tdFetchTbUidList(SSma* pSma, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid); int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore, bool isAdd); -void tdUidStoreDestory(STbUidStore* pStore); void* tdUidStoreFree(STbUidStore* pStore); // SMetaSnapReader ======================================== @@ -277,6 +282,7 @@ int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData); // SRSmaSnapWriter ======================================== int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWriter** ppWriter); int32_t rsmaSnapWrite(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData); +int32_t rsmaSnapWriterPrepareClose(SRSmaSnapWriter* pWriter); int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback); typedef struct { @@ -338,16 +344,24 @@ typedef struct SVCommitSched { } SVCommitSched; struct SVnode { - char* path; - SVnodeCfg config; - SVState state; - SVStatis statis; - STfs* pTfs; - SMsgCb msgCb; + char* path; + SVnodeCfg config; + SVState state; + SVStatis statis; + STfs* pTfs; + SMsgCb msgCb; + + // Buffer Pool TdThreadMutex mutex; TdThreadCond poolNotEmpty; - SVBufPool* pPool; + SVBufPool* aBufPool[VNODE_BUFPOOL_SEGMENTS]; + SVBufPool* freeList; SVBufPool* inUse; + SVBufPool* onCommit; + SVBufPool* recycleHead; + SVBufPool* recycleTail; + SVBufPool* onRecycle; + SMeta* pMeta; SSma* pSma; STsdb* pTsdb; @@ -426,6 +440,7 @@ enum { struct SSnapDataHdr { int8_t type; + int8_t flag; int64_t index; int64_t size; uint8_t data[]; diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index d68658b0d9489bae47ad8787cd8ce0d4de1f3438..85d8f031fb3f4ab32c806cebec583c5b21656c11 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -55,8 +55,8 @@ struct SMetaCache { // query cache struct STagFilterResCache { TdThreadMutex lock; - SHashObj* pTableEntry; - SLRUCache* pUidResCache; + SHashObj* pTableEntry; + SLRUCache* pUidResCache; } sTagFilterResCache; }; @@ -210,7 +210,7 @@ _exit: int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo) { int32_t code = 0; - // ASSERT(metaIsWLocked(pMeta)); + // meta is wlocked for calling this func. // search SMetaCache* pCache = pMeta->pCache; @@ -221,7 +221,10 @@ int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo) { } if (*ppEntry) { // update - ASSERT(pInfo->suid == (*ppEntry)->info.suid); + if (pInfo->suid != (*ppEntry)->info.suid) { + metaError("meta/cache: suid should be same as the one in cache."); + return TSDB_CODE_FAILED; + } if (pInfo->version > (*ppEntry)->info.version) { (*ppEntry)->info.version = pInfo->version; (*ppEntry)->info.skmVer = pInfo->skmVer; @@ -340,7 +343,7 @@ _exit: int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo) { int32_t code = 0; - // ASSERT(metaIsWLocked(pMeta)); + // meta is wlocked for calling this func. // search SMetaCache* pCache = pMeta->pCache; @@ -449,7 +452,11 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK // do some book mark work after acquiring the filter result from cache STagFilterResEntry** pEntry = taosHashGet(pTableMap, &suid, sizeof(uint64_t)); - ASSERT(pEntry != NULL); + if (NULL == pEntry) { + metaError("meta/cache: pEntry should not be NULL."); + return TSDB_CODE_FAILED; + } + *acquireRes = 1; const char* p = taosLRUCacheValue(pCache, pHandle); @@ -494,7 +501,7 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK taosMemoryFree(*p1); } - atomic_store_32(&(*pEntry)->qTimes, 0); // reset the query times + atomic_store_32(&(*pEntry)->qTimes, 0); // reset the query times taosArrayDestroy(pInvalidRes); taosThreadMutexUnlock(pLock); @@ -550,16 +557,19 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int buf[0] = suid; memcpy(&buf[1], pKey, keyLen); - ASSERT(sizeof(uint64_t) + keyLen == 24); + if (sizeof(uint64_t) + keyLen != 24) { + metaError("meta/cache: incorrect keyLen:%" PRId32 " length.", keyLen); + return TSDB_CODE_FAILED; + } // add to cache. - taosLRUCacheInsert(pCache, buf, sizeof(uint64_t) + keyLen, pPayload, payloadLen, freePayload, NULL, - TAOS_LRU_PRIORITY_LOW); + int32_t ret = taosLRUCacheInsert(pCache, buf, sizeof(uint64_t) + keyLen, pPayload, payloadLen, freePayload, NULL, + TAOS_LRU_PRIORITY_LOW); taosThreadMutexUnlock(pLock); - metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", TD_VID(pMeta->pVnode), suid, - (int32_t)taosLRUCacheGetUsage(pCache), taosHashGetSize(pTableEntry)); + metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d, ret:%d", TD_VID(pMeta->pVnode), + suid, (int32_t)taosLRUCacheGetUsage(pCache), taosHashGetSize(pTableEntry), ret); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index 60df66fc01e745642bcf74dcfe9b10d751edaa3f..e50931ac0618252eea58afc0d1a734d252a05d1e 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -51,7 +51,9 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { } else if (pME->type == TSDB_TSMA_TABLE) { if (tEncodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1; } else { - ASSERT(0); + metaError("meta/entry: invalide table type: %" PRId8 " encode failed.", pME->type); + + return -1; } tEndEncode(pCoder); diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 867b481bcc104561703105c4aed5bd9c75fd8455..609ffc58c3d4ed00c3b5db729179a3559b36e93d 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -359,7 +359,10 @@ static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL return -1; } - ASSERT(pTagIdxKey1->type == pTagIdxKey2->type); + if (pTagIdxKey1->type != pTagIdxKey2->type) { + metaError("meta/open: incorrect tag idx type."); + return TSDB_CODE_FAILED; + } // check NULL, NULL is always the smallest if (pTagIdxKey1->isNull && !pTagIdxKey2->isNull) { diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index cfdb4ab8d1d18aee42ef0528d64d72581c25630d..def5dc22b3818d97656b1f51a46951e64051c381 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -19,7 +19,7 @@ void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) { memset(pReader, 0, sizeof(*pReader)); pReader->flags = flags; pReader->pMeta = pMeta; - if (!(flags & META_READER_NOLOCK)) { + if (pReader->pMeta && !(flags & META_READER_NOLOCK)) { metaRLock(pMeta); } } @@ -152,7 +152,7 @@ bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid) { } int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { - SMeta *pMeta = pReader->pMeta; + SMeta *pMeta = pReader->pMeta; int64_t version1; // query uid.idx @@ -239,7 +239,6 @@ int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName) { return 0; } - int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) { int code = 0; SMetaReader mr = {0}; @@ -612,23 +611,14 @@ tb_uid_t metaStbCursorNext(SMStbCursor *pStbCur) { } STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) { - // SMetaReader mr = {0}; STSchema *pTSchema = NULL; SSchemaWrapper *pSW = NULL; - STSchemaBuilder sb = {0}; - SSchema *pSchema; + SSchema *pSchema = NULL; pSW = metaGetTableSchema(pMeta, uid, sver, lock); if (!pSW) return NULL; - tdInitTSchemaBuilder(&sb, pSW->version); - for (int i = 0; i < pSW->nCols; i++) { - pSchema = pSW->pSchema + i; - tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes); - } - pTSchema = tdGetSchemaFromBuilder(&sb); - - tdDestroyTSchemaBuilder(&sb); + pTSchema = tBuildTSchema(pSW->pSchema, pSW->nCols, pSW->version); taosMemoryFree(pSW->pSchema); taosMemoryFree(pSW); @@ -662,7 +652,13 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv goto _exit; } - ASSERT(c); + if (c == 0) { + metaULock(pMeta); + tdbTbcClose(pSkmDbC); + code = TSDB_CODE_FAILED; + metaError("meta/query: incorrect c: %" PRId32 ".", c); + goto _exit; + } if (c < 0) { tdbTbcMoveToPrev(pSkmDbC); @@ -686,7 +682,11 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv } } - ASSERT(sver > 0); + if (sver <= 0) { + metaError("meta/query: incorrect sver: %" PRId32 ".", sver); + code = TSDB_CODE_FAILED; + goto _exit; + } skmDbKey.uid = suid ? suid : uid; skmDbKey.sver = sver; @@ -709,21 +709,11 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv tdbFree(pData); // convert - STSchemaBuilder sb = {0}; - - tdInitTSchemaBuilder(&sb, pSchemaWrapper->version); - for (int i = 0; i < pSchemaWrapper->nCols; i++) { - SSchema *pSchema = pSchemaWrapper->pSchema + i; - tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes); - } - - STSchema *pTSchema = tdGetSchemaFromBuilder(&sb); + STSchema *pTSchema = tBuildTSchema(pSchemaWrapper->pSchema, pSchemaWrapper->nCols, pSchemaWrapper->version); if (pTSchema == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; } - tdDestroyTSchemaBuilder(&sb); - *ppTSchema = pTSchema; taosMemoryFree(pSchemaWrapper->pSchema); @@ -756,9 +746,7 @@ int64_t metaGetTimeSeriesNum(SMeta *pMeta) { return pMeta->pVnode->config.vndStats.numOfTimeSeries + pMeta->pVnode->config.vndStats.numOfNTimeSeries; } -int64_t metaGetNtbNum(SMeta *pMeta) { - return pMeta->pVnode->config.vndStats.numOfNTables; -} +int64_t metaGetNtbNum(SMeta *pMeta) { return pMeta->pVnode->config.vndStats.numOfNTables; } typedef struct { SMeta *pMeta; diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 8b023b7bc50292dbdd90b52fbeb83eab5d625429..6e505dfde512e53607e7410dbf5f31c61f5ffdd6 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -100,7 +100,10 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { break; } - ASSERT(pData && nData); + if (!pData || !nData) { + metaError("meta/snap: invalide nData: %" PRId32 " meta snap read failed.", nData); + goto _exit; + } *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + nData); if (*ppData == NULL) { @@ -358,7 +361,11 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t for (int i = 0; i < taosArrayGetSize(ctx->idList); i++) { int64_t* uid = taosArrayGet(ctx->idList, i); SIdInfo* idData = (SIdInfo*)taosHashGet(ctx->idVersion, uid, sizeof(int64_t)); - ASSERT(idData); + if (!idData) { + metaError("meta/snap: null idData"); + return TSDB_CODE_FAILED; + } + idData->index = i; metaDebug("tmqsnap init idVersion uid:%" PRIi64 " version:%" PRIi64 " index:%d", *uid, idData->version, idData->index); @@ -475,7 +482,10 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in int64_t* uidTmp = taosArrayGet(ctx->idList, ctx->index); ctx->index++; SIdInfo* idInfo = (SIdInfo*)taosHashGet(ctx->idVersion, uidTmp, sizeof(tb_uid_t)); - ASSERT(idInfo); + if (!idInfo) { + metaError("meta/snap: null idInfo"); + return TSDB_CODE_FAILED; + } *uid = *uidTmp; ret = MoveToPosition(ctx, idInfo->version, *uidTmp); @@ -509,7 +519,11 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in (ctx->subType == TOPIC_SUB_TYPE__TABLE && me.type == TSDB_CHILD_TABLE && me.ctbEntry.suid == ctx->suid)) { STableInfoForChildTable* data = (STableInfoForChildTable*)taosHashGet(ctx->suidInfo, &me.ctbEntry.suid, sizeof(tb_uid_t)); - ASSERT(data); + if (!data) { + metaError("meta/snap: null data"); + return TSDB_CODE_FAILED; + } + SVCreateTbReq req = {0}; req.type = TSDB_CHILD_TABLE; @@ -530,7 +544,8 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in } else { SArray* pTagVals = NULL; if (tTagToValArray((const STag*)p, &pTagVals) != 0) { - ASSERT(0); + metaError("meta/snap: tag to val array failed."); + return TSDB_CODE_FAILED; } int16_t nCols = taosArrayGetSize(pTagVals); for (int j = 0; j < nCols; ++j) { @@ -574,7 +589,8 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in ret = buildNormalChildTableInfo(&req, pBuf, contLen); *type = TDMT_VND_CREATE_TABLE; } else { - ASSERT(0); + metaError("meta/snap: invalid topic sub type: %" PRId8 " get meta from snap failed.", ctx->subType); + ret = -1; } tDecoderClear(&dc); @@ -595,7 +611,10 @@ SMetaTableInfo getUidfromSnapShot(SSnapContext* ctx) { int64_t* uidTmp = taosArrayGet(ctx->idList, ctx->index); ctx->index++; SIdInfo* idInfo = (SIdInfo*)taosHashGet(ctx->idVersion, uidTmp, sizeof(tb_uid_t)); - ASSERT(idInfo); + if (!idInfo) { + metaError("meta/snap: null idInfo"); + return result; + } int32_t ret = MoveToPosition(ctx, idInfo->version, *uidTmp); if (ret != 0) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 60a27bec102d78983747aa89fa1dd11c9f0c10f3..78e0643e1076c04529e04c98e347016aeec0076b 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -46,7 +46,7 @@ static void metaGetEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) { pInfo->suid = 0; pInfo->skmVer = pEntry->ntbEntry.schemaRow.version; } else { - ASSERT(0); + metaError("meta/table: invalide table type: %" PRId8 " get entry info failed.", pEntry->type); } } @@ -342,10 +342,23 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c); - ASSERT(ret == 0 && c == 0); + if (!(ret == 0 && c == 0)) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + + terrno = TSDB_CODE_TDB_STB_NOT_EXIST; + metaError("meta/table: invalide ret: %" PRId32 " or c: %" PRId32 "alter stb failed.", ret, c); + return -1; + } ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); - ASSERT(ret == 0); + if (ret < 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + + terrno = TSDB_CODE_TDB_STB_NOT_EXIST; + return -1; + } oStbEntry.pBuf = taosMemoryMalloc(nData); memcpy(oStbEntry.pBuf, pData, nData); @@ -558,7 +571,8 @@ static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME) { ctime = pME->ntbEntry.ctime; ttlDays = pME->ntbEntry.ttlDays; } else { - ASSERT(0); + metaError("meta/table: invalide table type: %" PRId8 " build ttl idx key failed.", pME->type); + return; } if (ttlDays <= 0) return; @@ -773,7 +787,11 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); - ASSERT(c == 0); + if (c != 0) { + tdbTbcClose(pUidIdxc); + metaError("meta/table: invalide c: %" PRId32 " alt tb column failed.", c); + return -1; + } tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); oversion = ((SUidIdxVal *)pData)[0].version; @@ -783,7 +801,13 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); - ASSERT(c == 0); + if (c != 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + metaError("meta/table: invalide c: %" PRId32 " alt tb column failed.", c); + return -1; + } + tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); // get table entry @@ -792,7 +816,13 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl memcpy(entry.pBuf, pData, nData); tDecoderInit(&dc, entry.pBuf, nData); ret = metaDecodeEntry(&dc, &entry); - ASSERT(ret == 0); + if (ret != 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + tDecoderClear(&dc); + metaError("meta/table: invalide ret: %" PRId32 " alt tb column failed.", ret); + return -1; + } if (entry.type != TSDB_NORMAL_TABLE) { terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; @@ -812,7 +842,11 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl if (iCol >= pSchema->nCols) break; pColumn = &pSchema->pSchema[iCol]; - ASSERT(pAlterTbReq->colName); + if (NULL == pAlterTbReq->colName) { + metaError("meta/table: null pAlterTbReq->colName"); + return -1; + } + if (strcmp(pColumn->name, pAlterTbReq->colName) == 0) break; iCol++; } @@ -964,7 +998,12 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); - ASSERT(c == 0); + if (c != 0) { + tdbTbcClose(pUidIdxc); + terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); + return -1; + } tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); oversion = ((SUidIdxVal *)pData)[0].version; @@ -977,7 +1016,14 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA /* get ctbEntry */ tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); - ASSERT(c == 0); + if (c != 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); + return -1; + } + tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); ctbEntry.pBuf = taosMemoryMalloc(nData); @@ -1075,7 +1121,11 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA metaUpdateTagIdx(pMeta, &ctbEntry); } - ASSERT(ctbEntry.ctbEntry.pTags); + if (NULL == ctbEntry.ctbEntry.pTags) { + metaError("meta/table: null tags, update tag val failed."); + goto _err; + } + SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, ((STag *)(ctbEntry.ctbEntry.pTags))->len, pMeta->txn); @@ -1130,7 +1180,11 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); - ASSERT(c == 0); + if (c != 0) { + tdbTbcClose(pUidIdxc); + metaError("meta/table: invalide c: %" PRId32 " update tb options failed.", c); + return -1; + } tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); oversion = ((SUidIdxVal *)pData)[0].version; @@ -1140,7 +1194,13 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); - ASSERT(c == 0); + if (c != 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + metaError("meta/table: invalide c: %" PRId32 " update tb options failed.", c); + return -1; + } + tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); // get table entry @@ -1149,7 +1209,13 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p memcpy(entry.pBuf, pData, nData); tDecoderInit(&dc, entry.pBuf, nData); ret = metaDecodeEntry(&dc, &entry); - ASSERT(ret == 0); + if (ret != 0) { + tDecoderClear(&dc); + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + metaError("meta/table: invalide ret: %" PRId32 " alt tb options failed.", ret); + return -1; + } entry.version = version; metaWLock(pMeta); @@ -1408,7 +1474,8 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { } else if (pME->type == TSDB_NORMAL_TABLE) { pSW = &pME->ntbEntry.schemaRow; } else { - ASSERT(0); + metaError("meta/table: invalide table type: %" PRId8 " save skm db failed.", pME->type); + return TSDB_CODE_FAILED; } skmDbKey.uid = pME->uid; diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 20db35e5b561b559bbf5a0145b95bae0ec714d08..38f04bb8e8bea7f12e7fb128775d5a80f54bb51a 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -17,42 +17,11 @@ extern SSmaMgmt smaMgmt; -#if 0 -static int32_t tdProcessRSmaSyncPreCommitImpl(SSma *pSma); -static int32_t tdProcessRSmaSyncCommitImpl(SSma *pSma); -static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma); -#endif static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma); static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma, SCommitInfo *pInfo); static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma); static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat); -#if 0 -/** - * @brief Only applicable to Rollup SMA - * - * @param pSma - * @return int32_t - */ -int32_t smaSyncPreCommit(SSma *pSma) { return tdProcessRSmaSyncPreCommitImpl(pSma); } - -/** - * @brief Only applicable to Rollup SMA - * - * @param pSma - * @return int32_t - */ -int32_t smaSyncCommit(SSma *pSma) { return tdProcessRSmaSyncCommitImpl(pSma); } - -/** - * @brief Only applicable to Rollup SMA - * - * @param pSma - * @return int32_t - */ -int32_t smaSyncPostCommit(SSma *pSma) { return tdProcessRSmaSyncPostCommitImpl(pSma); } -#endif - /** * @brief async commit, only applicable to Rollup SMA * @@ -128,166 +97,24 @@ _exit: int32_t smaFinishCommit(SSma *pSma) { int32_t code = 0; + int32_t lino = 0; SVnode *pVnode = pSma->pVnode; + code = tdRSmaFSFinishCommit(pSma); + TSDB_CHECK_CODE(code, lino, _exit); + if (VND_RSMA1(pVnode) && (code = tsdbFinishCommit(VND_RSMA1(pVnode))) < 0) { - smaError("vgId:%d, failed to finish commit tsdb rsma1 since %s", TD_VID(pVnode), tstrerror(code)); - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } if (VND_RSMA2(pVnode) && (code = tsdbFinishCommit(VND_RSMA2(pVnode))) < 0) { - smaError("vgId:%d, failed to finish commit tsdb rsma2 since %s", TD_VID(pVnode), tstrerror(code)); - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } _exit: - terrno = code; - return code; -} - -#if 0 -/** - * @brief pre-commit for rollup sma(sync commit). - * 1) set trigger stat of rsma timer TASK_TRIGGER_STAT_PAUSED. - * 2) wait for all triggered fetch tasks to finish - * 3) perform persist task for qTaskInfo - * - * @param pSma - * @return int32_t - */ -static int32_t tdProcessRSmaSyncPreCommitImpl(SSma *pSma) { - SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); - if (!pSmaEnv) { - return TSDB_CODE_SUCCESS; - } - - SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); - SRSmaStat *pRSmaStat = SMA_STAT_RSMA(pStat); - - // step 1: set rsma stat paused - atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED); - - // step 2: wait for all triggered fetch tasks to finish - int32_t nLoops = 0; - while (1) { - if (T_REF_VAL_GET(pStat) == 0) { - smaDebug("vgId:%d, rsma fetch tasks are all finished", SMA_VID(pSma)); - break; - } else { - smaDebug("vgId:%d, rsma fetch tasks are not all finished yet", SMA_VID(pSma)); - } - ++nLoops; - if (nLoops > 1000) { - sched_yield(); - nLoops = 0; - } + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); } - - // step 3: perform persist task for qTaskInfo - pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied; - tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)); - - smaDebug("vgId:%d, rsma pre commit success", SMA_VID(pSma)); - - return TSDB_CODE_SUCCESS; -} - -/** - * @brief commit for rollup sma - * - * @param pSma - * @return int32_t - */ -static int32_t tdProcessRSmaSyncCommitImpl(SSma *pSma) { -#if 0 - SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); - if (!pSmaEnv) { - return TSDB_CODE_SUCCESS; - } -#endif - return TSDB_CODE_SUCCESS; -} -#endif - -// SQTaskFile ====================================================== - -/** - * @brief At most time, there is only one qtaskinfo file committed latest in aTaskFile. Sometimes, there would be - * multiple qtaskinfo files supporting snapshot replication. - * - * @param pSma - * @param pStat - * @return int32_t - */ -static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pStat) { - SVnode *pVnode = pSma->pVnode; - SRSmaFS *pFS = RSMA_FS(pStat); - int64_t committed = pStat->commitAppliedVer; - int64_t fsMaxVer = -1; - char qTaskInfoFullName[TSDB_FILENAME_LEN]; - - taosWLockLatch(RSMA_FS_LOCK(pStat)); - - for (int32_t i = 0; i < taosArrayGetSize(pFS->aQTaskInf);) { - SQTaskFile *pTaskF = taosArrayGet(pFS->aQTaskInf, i); - int32_t oldVal = atomic_fetch_sub_32(&pTaskF->nRef, 1); - if ((oldVal <= 1) && (pTaskF->version < committed)) { - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->version, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); - if (taosRemoveFile(qTaskInfoFullName) < 0) { - smaWarn("vgId:%d, cleanup qinf, committed %" PRIi64 ", failed to remove %s since %s", TD_VID(pVnode), committed, - qTaskInfoFullName, tstrerror(TAOS_SYSTEM_ERROR(errno))); - } else { - smaDebug("vgId:%d, cleanup qinf, committed %" PRIi64 ", success to remove %s", TD_VID(pVnode), committed, - qTaskInfoFullName); - } - taosArrayRemove(pFS->aQTaskInf, i); - continue; - } - ++i; - } - - if (taosArrayGetSize(pFS->aQTaskInf) > 0) { - fsMaxVer = ((SQTaskFile *)taosArrayGetLast(pFS->aQTaskInf))->version; - } - - if (fsMaxVer < committed) { - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), committed, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); - if (taosCheckExistFile(qTaskInfoFullName)) { - SQTaskFile qFile = {.nRef = 1, .padding = 0, .version = committed, .size = 0}; - if (!taosArrayPush(pFS->aQTaskInf, &qFile)) { - taosWUnLockLatch(RSMA_FS_LOCK(pStat)); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - } - } else { - smaDebug("vgId:%d, update qinf, no need as committed %" PRIi64 " not larger than fsMaxVer %" PRIi64, TD_VID(pVnode), - committed, fsMaxVer); - } - - taosWUnLockLatch(RSMA_FS_LOCK(pStat)); - return TSDB_CODE_SUCCESS; -} - -#if 0 -/** - * @brief post-commit for rollup sma - * 1) clean up the outdated qtaskinfo files - * - * @param pSma - * @return int32_t - */ -static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma) { - SVnode *pVnode = pSma->pVnode; - if (!VND_IS_RSMA(pVnode)) { - return TSDB_CODE_SUCCESS; - } - - SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSma); - - tdUpdateQTaskInfoFiles(pSma, pRSmaStat); - - return TSDB_CODE_SUCCESS; + return code; } -#endif /** * @brief Rsma async commit implementation(only do some necessary light weighted task) @@ -298,9 +125,12 @@ static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma) { * @return int32_t */ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { + int32_t code = 0; + int32_t lino = 0; + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); if (!pEnv) { - return TSDB_CODE_SUCCESS; + return code; } SSmaStat *pStat = SMA_ENV_STAT(pEnv); @@ -317,7 +147,11 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { } } pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied; - ASSERT(pRSmaStat->commitAppliedVer > 0); + if (ASSERTS(pRSmaStat->commitAppliedVer >= 0, "commit applied version %" PRIi64 " < 0", + pRSmaStat->commitAppliedVer)) { + code = TSDB_CODE_APP_ERROR; + TSDB_CHECK_CODE(code, lino, _exit); + } // step 2: wait for all triggered fetch tasks to finish nLoops = 0; @@ -351,9 +185,9 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { } } smaInfo("vgId:%d, rsma commit, all items are consumed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); - if (tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)) < 0) { - return TSDB_CODE_FAILED; - } + code = tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)); + TSDB_CHECK_CODE(code, lino, _exit); + smaInfo("vgId:%d, rsma commit, operator state committed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); #if 0 // consuming task of qTaskInfo clone @@ -361,8 +195,6 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { // lock taosWLockLatch(SMA_ENV_LOCK(pEnv)); - ASSERT(RSMA_INFO_HASH(pRSmaStat)); - void *pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL); while (pIter) { @@ -380,10 +212,20 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { // all rsma results are written completely STsdb *pTsdb = NULL; - if ((pTsdb = VND_RSMA1(pSma->pVnode))) tsdbPrepareCommit(pTsdb); - if ((pTsdb = VND_RSMA2(pSma->pVnode))) tsdbPrepareCommit(pTsdb); + if ((pTsdb = VND_RSMA1(pSma->pVnode))) { + code = tsdbPrepareCommit(pTsdb); + TSDB_CHECK_CODE(code, lino, _exit); + } + if ((pTsdb = VND_RSMA2(pSma->pVnode))) { + code = tsdbPrepareCommit(pTsdb); + TSDB_CHECK_CODE(code, lino, _exit); + } - return TSDB_CODE_SUCCESS; +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pSma), __func__, lino, tstrerror(code)); + } + return code; } /** @@ -394,30 +236,27 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { */ static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma, SCommitInfo *pInfo) { int32_t code = 0; + int32_t lino = 0; SVnode *pVnode = pSma->pVnode; + SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); if (!pSmaEnv) { goto _exit; } -#if 0 - SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pSmaEnv); + + code = tdRSmaFSCommit(pSma); + TSDB_CHECK_CODE(code, lino, _exit); - // perform persist task for qTaskInfo operator - if (tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)) < 0) { - return TSDB_CODE_FAILED; - } -#endif + code = tsdbCommit(VND_RSMA1(pVnode), pInfo); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbCommit(VND_RSMA2(pVnode), pInfo); + TSDB_CHECK_CODE(code, lino, _exit); - if ((code = tsdbCommit(VND_RSMA1(pVnode), pInfo)) < 0) { - smaError("vgId:%d, failed to commit tsdb rsma1 since %s", TD_VID(pVnode), tstrerror(code)); - goto _exit; - } - if ((code = tsdbCommit(VND_RSMA2(pVnode), pInfo)) < 0) { - smaError("vgId:%d, failed to commit tsdb rsma2 since %s", TD_VID(pVnode), tstrerror(code)); - goto _exit; - } _exit: - terrno = code; + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } return code; } @@ -477,8 +316,6 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); } - tdUpdateQTaskInfoFiles(pSma, pRSmaStat); - atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 0); return TSDB_CODE_SUCCESS; diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index a272f5fc97883943e95d8ee7feb1c60ec90d49cb..a980b653a9c9d7e2e3f71c2502464e0fe68ff986 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -131,7 +131,7 @@ static int32_t tdNewSmaEnv(SSma *pSma, int8_t smaType, SSmaEnv **ppEnv) { (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&SMA_TSMA_ENV(pSma), *ppEnv) : atomic_store_ptr(&SMA_RSMA_ENV(pSma), *ppEnv); - if (tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType, pSma) != TSDB_CODE_SUCCESS) { + if ((terrno = tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType, pSma)) != TSDB_CODE_SUCCESS) { tdFreeSmaEnv(pEnv); *ppEnv = NULL; (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&SMA_TSMA_ENV(pSma), NULL) @@ -193,10 +193,16 @@ static void tRSmaInfoHashFreeNode(void *data) { } static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pSma) { - ASSERT(pSmaStat != NULL); + int32_t code = 0; + int32_t lino = 0; + + if (ASSERTS(pSmaStat != NULL, "pSmaStat is NULL")) { + terrno = TSDB_CODE_RSMA_INVALID_ENV; + TSDB_CHECK_CODE(code, lino, _exit); + } if (*pSmaStat) { // no lock - return TSDB_CODE_SUCCESS; + return code; // success, return directly } /** @@ -207,8 +213,8 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS if (!(*pSmaStat)) { *pSmaStat = (SSmaStat *)taosMemoryCalloc(1, sizeof(SSmaStat) + sizeof(TdThread) * tsNumOfVnodeRsmaThreads); if (!(*pSmaStat)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } if (smaType == TSDB_SMA_TYPE_ROLLUP) { @@ -224,7 +230,8 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS if (refId < 0) { smaError("vgId:%d, taosAddRef refId:%" PRIi64 " to rsetId rsetId:%d max:%d failed since:%s", SMA_VID(pSma), refId, smaMgmt.rsetId, SMA_MGMT_REF_NUM, tstrerror(terrno)); - return TSDB_CODE_FAILED; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } else { smaDebug("vgId:%d, taosAddRef refId:%" PRIi64 " to rsetId rsetId:%d max:%d succeed", SMA_VID(pSma), refId, smaMgmt.rsetId, SMA_MGMT_REF_NUM); @@ -235,25 +242,30 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS RSMA_INFO_HASH(pRSmaStat) = taosHashInit( RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (!RSMA_INFO_HASH(pRSmaStat)) { - return TSDB_CODE_FAILED; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } taosHashSetFreeFp(RSMA_INFO_HASH(pRSmaStat), tRSmaInfoHashFreeNode); if (tdRsmaStartExecutor(pSma) < 0) { - return TSDB_CODE_FAILED; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } - if (!(RSMA_FS(pRSmaStat)->aQTaskInf = taosArrayInit(1, sizeof(SQTaskFile)))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } + taosInitRWLatch(RSMA_FS_LOCK(pRSmaStat)); } else if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { // TODO } else { - ASSERT(0); + ASSERTS(0, "unknown smaType:%" PRIi8, smaType); + code = TSDB_CODE_APP_ERROR; + TSDB_CHECK_CODE(code, lino, _exit); } } - return TSDB_CODE_SUCCESS; +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pSma), __func__, lino, tstrerror(code)); + } + return code; } static void tdDestroyTSmaStat(STSmaStat *pStat) { @@ -342,7 +354,10 @@ static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { smaDebug("vgId:%d, remove refId:%" PRIi64 " from rsmaRef:%" PRIi32 " succeed", vid, refId, smaMgmt.rsetId); } } else { - ASSERT(0); + ASSERTS(0, "unknown smaType:%" PRIi8, smaType); + terrno = TSDB_CODE_APP_ERROR; + smaError("%s failed at line %d since %s", __func__, __LINE__, terrstr()); + return -1; } } return 0; @@ -351,7 +366,7 @@ static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { int32_t tdLockSma(SSma *pSma) { int code = taosThreadMutexLock(&pSma->mutex); if (code != 0) { - smaError("vgId:%d, failed to lock td since %s", SMA_VID(pSma), strerror(errno)); + smaError("vgId:%d, failed to lock since %s", SMA_VID(pSma), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); return -1; } @@ -360,12 +375,17 @@ int32_t tdLockSma(SSma *pSma) { } int32_t tdUnLockSma(SSma *pSma) { - ASSERT(SMA_LOCKED(pSma)); + if (ASSERTS(SMA_LOCKED(pSma), "pSma %p is not locked:%d", pSma, pSma->locked)) { + terrno = TSDB_CODE_APP_ERROR; + smaError("vgId:%d, failed to unlock since %s", SMA_VID(pSma), tstrerror(terrno)); + return -1; + } + pSma->locked = false; int code = taosThreadMutexUnlock(&pSma->mutex); if (code != 0) { - smaError("vgId:%d, failed to unlock td since %s", SMA_VID(pSma), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); + smaError("vgId:%d, failed to unlock since %s", SMA_VID(pSma), strerror(errno)); return -1; } return 0; diff --git a/source/dnode/vnode/src/sma/smaFS.c b/source/dnode/vnode/src/sma/smaFS.c index 8db36be741f3187834d4b2f2eca4fa302ca524fe..ef872d055e5eec598af61bb9f410f6361b01c024 100644 --- a/source/dnode/vnode/src/sma/smaFS.c +++ b/source/dnode/vnode/src/sma/smaFS.c @@ -17,250 +17,636 @@ // ================================================================================================= -static int32_t tdFetchQTaskInfoFiles(SSma *pSma, int64_t version, SArray **output); +// static int32_t tdFetchQTaskInfoFiles(SSma *pSma, int64_t version, SArray **output); static int32_t tdQTaskInfCmprFn1(const void *p1, const void *p2); -static int32_t tdQTaskInfCmprFn2(const void *p1, const void *p2); -/** - * @brief Open RSma FS from qTaskInfo files - * - * @param pSma - * @param version - * @return int32_t - */ -int32_t tdRSmaFSOpen(SSma *pSma, int64_t version) { - SVnode *pVnode = pSma->pVnode; - int64_t commitID = pVnode->state.commitID; - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = NULL; - SArray *output = NULL; - terrno = TSDB_CODE_SUCCESS; +static FORCE_INLINE int32_t tPutQTaskF(uint8_t *p, SQTaskFile *pFile) { + int32_t n = 0; - if (!pEnv) { - return TSDB_CODE_SUCCESS; - } + n += tPutI8(p ? p + n : p, pFile->level); + n += tPutI64v(p ? p + n : p, pFile->size); + n += tPutI64v(p ? p + n : p, pFile->suid); + n += tPutI64v(p ? p + n : p, pFile->version); + n += tPutI64v(p ? p + n : p, pFile->mtime); + + return n; +} - if (tdFetchQTaskInfoFiles(pSma, version, &output) < 0) { - goto _end; +static int32_t tdRSmaFSToBinary(uint8_t *p, SRSmaFS *pFS) { + int32_t n = 0; + uint32_t size = taosArrayGetSize(pFS->aQTaskInf); + + // version + n += tPutI8(p ? p + n : p, 0); + + // SArray + n += tPutU32v(p ? p + n : p, size); + for (uint32_t i = 0; i < size; ++i) { + n += tPutQTaskF(p ? p + n : p, taosArrayGet(pFS->aQTaskInf, i)); } - pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + return n; +} + +int32_t tdRSmaGetQTaskF(uint8_t *p, SQTaskFile *pFile) { + int32_t n = 0; + + n += tGetI8(p + n, &pFile->level); + n += tGetI64v(p + n, &pFile->size); + n += tGetI64v(p + n, &pFile->suid); + n += tGetI64v(p + n, &pFile->version); + n += tGetI64v(p + n, &pFile->mtime); + + return n; +} + +static int32_t tsdbBinaryToFS(uint8_t *pData, int64_t nData, SRSmaFS *pFS) { + int32_t code = 0; + int32_t n = 0; + int8_t version = 0; + + // version + n += tGetI8(pData + n, &version); + + // SArray + taosArrayClear(pFS->aQTaskInf); + uint32_t size = 0; + n += tGetU32v(pData + n, &size); + for (uint32_t i = 0; i < size; ++i) { + SQTaskFile qTaskF = {0}; + + int32_t nt = tdRSmaGetQTaskF(pData + n, &qTaskF); + if (nt < 0) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } - for (int32_t i = 0; i < taosArrayGetSize(output); ++i) { - int32_t vid = 0; - int64_t version = -1; - sscanf((const char *)taosArrayGetP(output, i), "v%dqinf.v%" PRIi64, &vid, &version); - SQTaskFile qTaskFile = {.version = version, .nRef = 1}; - if ((terrno = tdRSmaFSUpsertQTaskFile(RSMA_FS(pStat), &qTaskFile)) < 0) { - goto _end; + n += nt; + if (taosArrayPush(pFS->aQTaskInf, &qTaskF) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; } - smaInfo("vgId:%d, open fs, version:%" PRIi64 ", ref:%d", TD_VID(pVnode), qTaskFile.version, qTaskFile.nRef); } -_end: - for (int32_t i = 0; i < taosArrayGetSize(output); ++i) { - void *ptr = taosArrayGetP(output, i); - taosMemoryFreeClear(ptr); + if (ASSERTS(n + sizeof(TSCKSUM) == nData, "n:%d + sizeof(TSCKSUM):%d != nData:%d", n, (int32_t)sizeof(TSCKSUM), + nData)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + +_exit: + return code; +} + +static int32_t tdRSmaSaveFSToFile(SRSmaFS *pFS, const char *fname) { + int32_t code = 0; + int32_t lino = 0; + + // encode to binary + int32_t size = tdRSmaFSToBinary(NULL, pFS) + sizeof(TSCKSUM); + uint8_t *pData = taosMemoryMalloc(size); + if (pData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + tdRSmaFSToBinary(pData, pFS); + taosCalcChecksumAppend(0, pData, size); + + // save to file + TdFilePtr pFD = taosCreateFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (pFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + int64_t n = taosWriteFile(pFD, pData, size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + taosCloseFile(&pFD); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (taosFsyncFile(pFD) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + taosCloseFile(&pFD); + TSDB_CHECK_CODE(code, lino, _exit); + } + + taosCloseFile(&pFD); + +_exit: + if (pData) taosMemoryFree(pData); + if (code) { + smaError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); + } + return code; +} + +static int32_t tdRSmaFSCreate(SRSmaFS *pFS, int32_t size) { + int32_t code = 0; + + pFS->aQTaskInf = taosArrayInit(size, sizeof(SQTaskFile)); + if (pFS->aQTaskInf == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; } - taosArrayDestroy(output); - if (terrno != TSDB_CODE_SUCCESS) { - smaError("vgId:%d, open rsma fs failed since %s", TD_VID(pVnode), terrstr()); - return TSDB_CODE_FAILED; +_exit: + return code; +} + +static void tdRSmaGetCurrentFName(SSma *pSma, char *current, char *current_t) { + SVnode *pVnode = pSma->pVnode; + if (pVnode->pTfs) { + if (current) { + snprintf(current, TSDB_FILENAME_LEN - 1, "%s%svnode%svnode%d%srsma%sPRESENT", tfsGetPrimaryPath(pVnode->pTfs), + TD_DIRSEP, TD_DIRSEP, TD_VID(pVnode), TD_DIRSEP, TD_DIRSEP); + } + if (current_t) { + snprintf(current_t, TSDB_FILENAME_LEN - 1, "%s%svnode%svnode%d%srsma%sPRESENT.t", tfsGetPrimaryPath(pVnode->pTfs), + TD_DIRSEP, TD_DIRSEP, TD_VID(pVnode), TD_DIRSEP, TD_DIRSEP); + } + } else { +#if 0 + if (current) { + snprintf(current, TSDB_FILENAME_LEN - 1, "%s%sPRESENT", pTsdb->path, TD_DIRSEP); + } + if (current_t) { + snprintf(current_t, TSDB_FILENAME_LEN - 1, "%s%sPRESENT.t", pTsdb->path, TD_DIRSEP); + } +#endif } - return TSDB_CODE_SUCCESS; } -void tdRSmaFSClose(SRSmaFS *fs) { taosArrayDestroy(fs->aQTaskInf); } +static int32_t tdRSmaLoadFSFromFile(const char *fname, SRSmaFS *pFS) { + int32_t code = 0; + int32_t lino = 0; + uint8_t *pData = NULL; + + // load binary + TdFilePtr pFD = taosOpenFile(fname, TD_FILE_READ); + if (pFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + int64_t size; + if (taosFStatFile(pFD, &size, NULL) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + taosCloseFile(&pFD); + TSDB_CHECK_CODE(code, lino, _exit); + } + + pData = taosMemoryMalloc(size); + if (pData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + taosCloseFile(&pFD); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (taosReadFile(pFD, pData, size) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + taosCloseFile(&pFD); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (!taosCheckChecksumWhole(pData, size)) { + code = TSDB_CODE_FILE_CORRUPTED; + taosCloseFile(&pFD); + TSDB_CHECK_CODE(code, lino, _exit); + } + + taosCloseFile(&pFD); + + // decode binary + code = tsdbBinaryToFS(pData, size, pFS); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (pData) taosMemoryFree(pData); + if (code) { + smaError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); + } + return code; +} static int32_t tdQTaskInfCmprFn1(const void *p1, const void *p2) { - if (*(int64_t *)p1 < ((SQTaskFile *)p2)->version) { + const SQTaskFile *q1 = (const SQTaskFile *)p1; + const SQTaskFile *q2 = (const SQTaskFile *)p2; + + if (q1->suid < q2->suid) { + return -1; + } else if (q1->suid > q2->suid) { + return 1; + } + + if (q1->level < q2->level) { return -1; - } else if (*(int64_t *)p1 > ((SQTaskFile *)p2)->version) { + } else if (q1->level > q2->level) { + return 1; + } + + if (q1->version < q2->version) { + return -2; + } else if (q1->version > q2->version) { return 1; } + return 0; } -int32_t tdRSmaFSRef(SSma *pSma, SRSmaStat *pStat, int64_t version) { - SArray *aQTaskInf = RSMA_FS(pStat)->aQTaskInf; - SQTaskFile *pTaskF = NULL; - int32_t oldVal = 0; +static int32_t tdRSmaFSApplyChange(SSma *pSma, SRSmaFS *pFSNew) { + int32_t code = 0; + int32_t lino = 0; + int32_t nRef = 0; + SVnode *pVnode = pSma->pVnode; + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + SRSmaFS *pFSOld = RSMA_FS(pStat); + int64_t version = pStat->commitAppliedVer; + char fname[TSDB_FILENAME_LEN] = {0}; + + // SQTaskFile + int32_t nNew = taosArrayGetSize(pFSNew->aQTaskInf); + int32_t iNew = 0; + while (iNew < nNew) { + SQTaskFile *pQTaskFNew = TARRAY_GET_ELEM(pFSNew->aQTaskInf, iNew++); + + int32_t idx = taosArraySearchIdx(pFSOld->aQTaskInf, pQTaskFNew, tdQTaskInfCmprFn1, TD_GE); + + if (idx < 0) { + idx = taosArrayGetSize(pFSOld->aQTaskInf); + pQTaskFNew->nRef = 1; + } else { + SQTaskFile *pTaskF = TARRAY_GET_ELEM(pFSOld->aQTaskInf, idx); + int32_t c1 = tdQTaskInfCmprFn1(pQTaskFNew, pTaskF); + if (c1 == 0) { + // utilize the item in pFSOld->qQTaskInf, instead of pFSNew + continue; + } else if (c1 < 0) { + // NOTHING TODO + } else { + code = TSDB_CODE_RSMA_FS_UPDATE; + TSDB_CHECK_CODE(code, lino, _exit); + } + } - taosRLockLatch(RSMA_FS_LOCK(pStat)); - if ((pTaskF = taosArraySearch(aQTaskInf, &version, tdQTaskInfCmprFn1, TD_EQ))) { - oldVal = atomic_fetch_add_32(&pTaskF->nRef, 1); - ASSERT(oldVal > 0); + if (taosArrayInsert(pFSOld->aQTaskInf, idx, pQTaskFNew) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + // remove previous version + while (--idx >= 0) { + SQTaskFile *preTaskF = TARRAY_GET_ELEM(pFSOld->aQTaskInf, idx); + int32_t c2 = tdQTaskInfCmprFn1(preTaskF, pQTaskFNew); + if (c2 == 0) { + code = TSDB_CODE_RSMA_FS_UPDATE; + TSDB_CHECK_CODE(code, lino, _exit); + } else if (c2 != -2) { + break; + } + + nRef = atomic_sub_fetch_32(&preTaskF->nRef, 1); + if (nRef <= 0) { + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), preTaskF->suid, preTaskF->level, preTaskF->version, + tfsGetPrimaryPath(pVnode->pTfs), fname); + (void)taosRemoveFile(fname); + taosArrayRemove(pFSOld->aQTaskInf, idx); + } + } } - taosRUnLockLatch(RSMA_FS_LOCK(pStat)); - return oldVal; + +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } + return code; } -int64_t tdRSmaFSMaxVer(SSma *pSma, SRSmaStat *pStat) { - SArray *aQTaskInf = RSMA_FS(pStat)->aQTaskInf; - int64_t version = -1; +static int32_t tdRSmaFSScanAndTryFix(SSma *pSma) { + int32_t code = 0; +#if 0 + int32_t lino = 0; + SVnode *pVnode = pSma->pVnode; + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + SRSmaFS *pFS = RSMA_FS(pStat); + char fname[TSDB_FILENAME_LEN] = {0}; + char fnameVer[TSDB_FILENAME_LEN] = {0}; + + // SArray + int32_t size = taosArrayGetSize(pFS->aQTaskInf); + for (int32_t i = 0; i < size; ++i) { + SQTaskFile *pTaskF = (SQTaskFile *)taosArrayGet(pFS->aQTaskInf, i); + + // main.tdb ========= + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, pTaskF->level, pTaskF->version, + tfsGetPrimaryPath(pVnode->pTfs), fnameVer); + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, pTaskF->level, -1, tfsGetPrimaryPath(pVnode->pTfs), fname); + + if (taosCheckExistFile(fnameVer)) { + if (taosRenameFile(fnameVer, fname) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + smaDebug("vgId:%d, %s:%d succeed to to rename %s to %s", TD_VID(pVnode), __func__, lino, fnameVer, fname); + } else if (taosCheckExistFile(fname)) { + if (taosRemoveFile(fname) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + smaDebug("vgId:%d, %s:%d succeed to to remove %s", TD_VID(pVnode), __func__, lino, fname); + } + } - taosRLockLatch(RSMA_FS_LOCK(pStat)); - if (taosArrayGetSize(aQTaskInf) > 0) { - version = ((SQTaskFile *)taosArrayGetLast(aQTaskInf))->version; + { + // remove those invalid files (todo) + // main.tdb-journal.5 // TDB should handle its clear for kill -9 } - taosRUnLockLatch(RSMA_FS_LOCK(pStat)); - return version; + +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } +#endif + return code; } -void tdRSmaFSUnRef(SSma *pSma, SRSmaStat *pStat, int64_t version) { - SVnode *pVnode = pSma->pVnode; - SArray *aQTaskInf = RSMA_FS(pStat)->aQTaskInf; - char qTaskFullName[TSDB_FILENAME_LEN]; - SQTaskFile *pTaskF = NULL; - int32_t idx = -1; +// EXPOSED APIS ==================================================================================== - taosWLockLatch(RSMA_FS_LOCK(pStat)); - if ((idx = taosArraySearchIdx(aQTaskInf, &version, tdQTaskInfCmprFn1, TD_EQ)) >= 0) { - ASSERT(idx < taosArrayGetSize(aQTaskInf)); - pTaskF = taosArrayGet(aQTaskInf, idx); - if (atomic_sub_fetch_32(&pTaskF->nRef, 1) <= 0) { - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->version, tfsGetPrimaryPath(pVnode->pTfs), qTaskFullName); - if (taosRemoveFile(qTaskFullName) < 0) { - smaWarn("vgId:%d, failed to remove %s since %s", TD_VID(pVnode), qTaskFullName, - tstrerror(TAOS_SYSTEM_ERROR(errno))); +int32_t tdRSmaFSOpen(SSma *pSma, int64_t version, int8_t rollback) { + int32_t code = 0; + int32_t lino = 0; + SVnode *pVnode = pSma->pVnode; + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + + // open handle + code = tdRSmaFSCreate(RSMA_FS(pStat), 0); + TSDB_CHECK_CODE(code, lino, _exit); + + // open impl + char current[TSDB_FILENAME_LEN] = {0}; + char current_t[TSDB_FILENAME_LEN] = {0}; + tdRSmaGetCurrentFName(pSma, current, current_t); + + if (taosCheckExistFile(current)) { + code = tdRSmaLoadFSFromFile(current, RSMA_FS(pStat)); + TSDB_CHECK_CODE(code, lino, _exit); + + if (taosCheckExistFile(current_t)) { + if (rollback) { + code = tdRSmaFSRollback(pSma); + TSDB_CHECK_CODE(code, lino, _exit); } else { - smaDebug("vgId:%d, success to remove %s", TD_VID(pVnode), qTaskFullName); + code = tdRSmaFSCommit(pSma); + TSDB_CHECK_CODE(code, lino, _exit); } - taosArrayRemove(aQTaskInf, idx); } + } else { + // 1st time open with empty current/qTaskInfoFile + code = tdRSmaSaveFSToFile(RSMA_FS(pStat), current); + TSDB_CHECK_CODE(code, lino, _exit); } - taosWUnLockLatch(RSMA_FS_LOCK(pStat)); + + // scan and try fix(remove main.db/main.db.xxx and use the one with version) + code = tdRSmaFSScanAndTryFix(pSma); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } + return code; } -/** - * @brief Fetch qtaskfiles LE than version - * - * @param pSma - * @param version - * @param output - * @return int32_t - */ -static int32_t tdFetchQTaskInfoFiles(SSma *pSma, int64_t version, SArray **output) { - SVnode *pVnode = pSma->pVnode; - TdDirPtr pDir = NULL; - TdDirEntryPtr pDirEntry = NULL; - char dir[TSDB_FILENAME_LEN]; - const char *pattern = "v[0-9]+qinf\\.v([0-9]+)?$"; - regex_t regex; - int code = 0; - - terrno = TSDB_CODE_SUCCESS; - - tdGetVndDirName(TD_VID(pVnode), tfsGetPrimaryPath(pVnode->pTfs), VNODE_RSMA_DIR, true, dir); - - if (!taosCheckExistFile(dir)) { - smaDebug("vgId:%d, fetch qtask files, no need as dir %s not exist", TD_VID(pVnode), dir); - return TSDB_CODE_SUCCESS; - } - - // Resource allocation and init - if ((code = regcomp(®ex, pattern, REG_EXTENDED)) != 0) { - terrno = TSDB_CODE_RSMA_REGEX_MATCH; - char errbuf[128]; - regerror(code, ®ex, errbuf, sizeof(errbuf)); - smaWarn("vgId:%d, fetch qtask files, regcomp for %s failed since %s", TD_VID(pVnode), dir, errbuf); - return TSDB_CODE_FAILED; - } - - if (!(pDir = taosOpenDir(dir))) { - regfree(®ex); - terrno = TAOS_SYSTEM_ERROR(errno); - smaError("vgId:%d, fetch qtask files, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr()); - return TSDB_CODE_FAILED; - } - - int32_t dirLen = strlen(dir); - char *dirEnd = POINTER_SHIFT(dir, dirLen); - regmatch_t regMatch[2]; - while ((pDirEntry = taosReadDir(pDir))) { - char *entryName = taosGetDirEntryName(pDirEntry); - if (!entryName) { - continue; - } +void tdRSmaFSClose(SRSmaFS *pFS) { pFS->aQTaskInf = taosArrayDestroy(pFS->aQTaskInf); } - code = regexec(®ex, entryName, 2, regMatch, 0); +int32_t tdRSmaFSPrepareCommit(SSma *pSma, SRSmaFS *pFSNew) { + int32_t code = 0; + int32_t lino = 0; + char tfname[TSDB_FILENAME_LEN]; - if (code == 0) { - // match - smaInfo("vgId:%d, fetch qtask files, max ver:%" PRIi64 ", %s found", TD_VID(pVnode), version, entryName); + tdRSmaGetCurrentFName(pSma, NULL, tfname); - int64_t ver = -1; - sscanf((const char *)POINTER_SHIFT(entryName, regMatch[1].rm_so), "%" PRIi64, &ver); - if ((ver <= version) && (ver > -1)) { - if (!(*output)) { - if (!(*output = taosArrayInit(1, POINTER_BYTES))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _end; - } - } - char *entryDup = strdup(entryName); - if (!entryDup) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _end; - } - if (!taosArrayPush(*output, &entryDup)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _end; - } - } else { - } - } else if (code == REG_NOMATCH) { - // not match - smaTrace("vgId:%d, fetch qtask files, not match %s", TD_VID(pVnode), entryName); - continue; - } else { - // has other error - char errbuf[128]; - regerror(code, ®ex, errbuf, sizeof(errbuf)); - smaWarn("vgId:%d, fetch qtask files, regexec failed since %s", TD_VID(pVnode), errbuf); - terrno = TSDB_CODE_RSMA_REGEX_MATCH; - goto _end; - } + // generate PRESENT.t + code = tdRSmaSaveFSToFile(pFSNew, tfname); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pSma->pVnode), __func__, lino, tstrerror(code)); } -_end: - taosCloseDir(&pDir); - regfree(®ex); - return terrno == 0 ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED; + return code; } -static int32_t tdQTaskFileCmprFn2(const void *p1, const void *p2) { - if (((SQTaskFile *)p1)->version < ((SQTaskFile *)p2)->version) { - return -1; - } else if (((SQTaskFile *)p1)->version > ((SQTaskFile *)p2)->version) { - return 1; +int32_t tdRSmaFSCommit(SSma *pSma) { + int32_t code = 0; + int32_t lino = 0; + SRSmaFS fs = {0}; + + char current[TSDB_FILENAME_LEN] = {0}; + char current_t[TSDB_FILENAME_LEN] = {0}; + tdRSmaGetCurrentFName(pSma, current, current_t); + + if (!taosCheckExistFile(current_t)) { + goto _exit; } - return 0; + // rename the file + if (taosRenameFile(current_t, current) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // load the new FS + code = tdRSmaFSCreate(&fs, 1); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tdRSmaLoadFSFromFile(current, &fs); + TSDB_CHECK_CODE(code, lino, _exit); + + // apply file change + code = tdRSmaFSApplyChange(pSma, &fs); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + tdRSmaFSClose(&fs); + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pSma), __func__, lino, tstrerror(code)); + } + return code; } -int32_t tdRSmaFSUpsertQTaskFile(SRSmaFS *pFS, SQTaskFile *qTaskFile) { - int32_t code = 0; - int32_t idx = taosArraySearchIdx(pFS->aQTaskInf, qTaskFile, tdQTaskFileCmprFn2, TD_GE); +int32_t tdRSmaFSFinishCommit(SSma *pSma) { + int32_t code = 0; + int32_t lino = 0; + SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pSmaEnv); - if (idx < 0) { - idx = taosArrayGetSize(pFS->aQTaskInf); + taosWLockLatch(RSMA_FS_LOCK(pStat)); + code = tdRSmaFSCommit(pSma); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + taosWUnLockLatch(RSMA_FS_LOCK(pStat)); + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pSma), __func__, lino, tstrerror(code)); } else { - SQTaskFile *pTaskF = (SQTaskFile *)taosArrayGet(pFS->aQTaskInf, idx); - int32_t c = tdQTaskFileCmprFn2(pTaskF, qTaskFile); - if (c == 0) { - pTaskF->nRef = qTaskFile->nRef; - pTaskF->version = qTaskFile->version; - pTaskF->size = qTaskFile->size; + smaInfo("vgId:%d, rsmaFS finish commit", SMA_VID(pSma)); + } + return code; +} + +int32_t tdRSmaFSRollback(SSma *pSma) { + int32_t code = 0; + int32_t lino = 0; + + char current_t[TSDB_FILENAME_LEN] = {0}; + tdRSmaGetCurrentFName(pSma, NULL, current_t); + (void)taosRemoveFile(current_t); + +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pSma), __func__, lino, tstrerror(errno)); + } + return code; +} + +int32_t tdRSmaFSUpsertQTaskFile(SSma *pSma, SRSmaFS *pFS, SQTaskFile *qTaskFile, int32_t nSize) { + int32_t code = 0; + + for (int32_t i = 0; i < nSize; ++i) { + SQTaskFile *qTaskF = qTaskFile + i; + + int32_t idx = taosArraySearchIdx(pFS->aQTaskInf, qTaskF, tdQTaskInfCmprFn1, TD_GE); + + if (idx < 0) { + idx = taosArrayGetSize(pFS->aQTaskInf); + } else { + SQTaskFile *pTaskF = (SQTaskFile *)taosArrayGet(pFS->aQTaskInf, idx); + int32_t c = tdQTaskInfCmprFn1(pTaskF, qTaskF); + if (c == 0) { + if (pTaskF->size != qTaskF->size) { + code = TSDB_CODE_RSMA_FS_UPDATE; + smaError("vgId:%d, %s failed at line %d since %s, level:%" PRIi8 ", suid:%" PRIi64 ", version:%" PRIi64 + ", size:%" PRIi64 " != %" PRIi64, + SMA_VID(pSma), __func__, __LINE__, tstrerror(code), pTaskF->level, pTaskF->suid, pTaskF->version, + pTaskF->size, qTaskF->size); + goto _exit; + } + continue; + } + } + + if (!taosArrayInsert(pFS->aQTaskInf, idx, qTaskF)) { + code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } } - if (taosArrayInsert(pFS->aQTaskInf, idx, qTaskFile) == NULL) { +_exit: + return code; +} + +int32_t tdRSmaFSRef(SSma *pSma, SRSmaFS *pFS) { + int32_t code = 0; + int32_t lino = 0; + int32_t nRef = 0; + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + SRSmaFS *qFS = RSMA_FS(pStat); + int32_t size = taosArrayGetSize(qFS->aQTaskInf); + + pFS->aQTaskInf = taosArrayInit(size, sizeof(SQTaskFile)); + if (pFS->aQTaskInf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); + } + + for (int32_t i = 0; i < size; ++i) { + SQTaskFile *qTaskF = (SQTaskFile *)taosArrayGet(qFS->aQTaskInf, i); + nRef = atomic_fetch_add_32(&qTaskF->nRef, 1); + if (nRef <= 0) { + code = TSDB_CODE_RSMA_FS_REF; + TSDB_CHECK_CODE(code, lino, _exit); + } } + taosArraySetSize(pFS->aQTaskInf, size); + memcpy(pFS->aQTaskInf->pData, qFS->aQTaskInf->pData, size * sizeof(SQTaskFile)); + _exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s, nRef %d", TD_VID(pSma->pVnode), __func__, lino, tstrerror(code), + nRef); + } return code; -} \ No newline at end of file +} + +void tdRSmaFSUnRef(SSma *pSma, SRSmaFS *pFS) { + int32_t nRef = 0; + char fname[TSDB_FILENAME_LEN]; + SVnode *pVnode = pSma->pVnode; + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + int32_t size = taosArrayGetSize(pFS->aQTaskInf); + + for (int32_t i = 0; i < size; ++i) { + SQTaskFile *pTaskF = (SQTaskFile *)taosArrayGet(pFS->aQTaskInf, i); + + nRef = atomic_sub_fetch_32(&pTaskF->nRef, 1); + if (nRef == 0) { + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, pTaskF->level, pTaskF->version, + tfsGetPrimaryPath(pVnode->pTfs), fname); + if (taosRemoveFile(fname) < 0) { + smaWarn("vgId:%d, failed to remove %s since %s", TD_VID(pVnode), fname, tstrerror(TAOS_SYSTEM_ERROR(errno))); + } else { + smaDebug("vgId:%d, success to remove %s", TD_VID(pVnode), fname); + } + } else if (nRef < 0) { + smaWarn("vgId:%d, abnormal unref %s since %s", TD_VID(pVnode), fname, tstrerror(TSDB_CODE_RSMA_FS_REF)); + } + } + + taosArrayDestroy(pFS->aQTaskInf); +} + +int32_t tdRSmaFSTakeSnapshot(SSma *pSma, SRSmaFS *pFS) { + int32_t code = 0; + int32_t lino = 0; + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + + taosRLockLatch(RSMA_FS_LOCK(pStat)); + code = tdRSmaFSRef(pSma, pFS); + TSDB_CHECK_CODE(code, lino, _exit); +_exit: + taosRUnLockLatch(RSMA_FS_LOCK(pStat)); + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pSma->pVnode), __func__, lino, tstrerror(code)); + } + return code; +} + +int32_t tdRSmaFSCopy(SSma *pSma, SRSmaFS *pFS) { + int32_t code = 0; + int32_t lino = 0; + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + SRSmaFS *qFS = RSMA_FS(pStat); + int32_t size = taosArrayGetSize(qFS->aQTaskInf); + + code = tdRSmaFSCreate(pFS, size); + TSDB_CHECK_CODE(code, lino, _exit); + + taosArraySetSize(pFS->aQTaskInf, size); + memcpy(pFS->aQTaskInf->pData, qFS->aQTaskInf->pData, size * sizeof(SQTaskFile)); + +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pSma->pVnode), __func__, lino, tstrerror(code)); + } + return code; +} diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index 2a769b68fe0ff93caa48a6a22030ca02b2af7023..21c283994345a00badb3e5c12c80f1e3b86bdc6f 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -34,13 +34,16 @@ static int32_t rsmaRestore(SSma *pSma); SRetention *r = (SRetention *)VND_RETENTIONS(v) + l; \ if (!RETENTION_VALID(r)) { \ if (l == 0) { \ - goto _err; \ + code = TSDB_CODE_INVALID_PARA; \ + TSDB_CHECK_CODE(code, lino, _exit); \ } \ break; \ } \ - smaSetKeepCfg(v, &keepCfg, pCfg, TSDB_TYPE_RSMA_L##l); \ + code = smaSetKeepCfg(v, &keepCfg, pCfg, TSDB_TYPE_RSMA_L##l); \ + TSDB_CHECK_CODE(code, lino, _exit); \ if (tsdbOpen(v, &SMA_RSMA_TSDB##l(pSma), VNODE_RSMA##l##_DIR, &keepCfg, rollback) < 0) { \ - goto _err; \ + code = terrno; \ + TSDB_CHECK_CODE(code, lino, _exit); \ } \ } while (0) @@ -68,12 +71,10 @@ static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t p days = keepDuration; } - if (level == TSDB_RETENTION_L0) { - goto end; + if (level < TSDB_RETENTION_L1 || level > TSDB_RETENTION_L2) { + goto _exit; } - ASSERT(level >= TSDB_RETENTION_L1 && level <= TSDB_RETENTION_L2); - freqDuration = convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE); keepDuration = convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE); @@ -91,16 +92,18 @@ static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t p if (days < freqDuration) { days = freqDuration; } -end: +_exit: smaInfo("vgId:%d, evaluated duration for level %d is %d, raw val:%d", TD_VID(pVnode), level + 1, days, duration); return days; } int smaSetKeepCfg(SVnode *pVnode, STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type) { + terrno = 0; pKeepCfg->precision = pCfg->precision; switch (type) { case TSDB_TYPE_TSMA: - ASSERT(0); + ASSERTS(0, "undefined smaType:%d", (int32_t)type); + terrno = TSDB_CODE_APP_ERROR; break; case TSDB_TYPE_RSMA_L0: SMA_SET_KEEP_CFG(pVnode, 0); @@ -112,21 +115,22 @@ int smaSetKeepCfg(SVnode *pVnode, STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int ty SMA_SET_KEEP_CFG(pVnode, 2); break; default: - ASSERT(0); + ASSERTS(0, "unknown smaType:%d", (int32_t)type); + terrno = TSDB_CODE_APP_ERROR; break; } - return 0; + return terrno; } int32_t smaOpen(SVnode *pVnode, int8_t rollback) { + int32_t code = 0; + int32_t lino = 0; STsdbCfg *pCfg = &pVnode->config.tsdbCfg; - ASSERT(!pVnode->pSma); - SSma *pSma = taosMemoryCalloc(1, sizeof(SSma)); if (!pSma) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } pVnode->pSma = pSma; @@ -137,27 +141,28 @@ int32_t smaOpen(SVnode *pVnode, int8_t rollback) { if (VND_IS_RSMA(pVnode)) { STsdbKeepCfg keepCfg = {0}; - for (int i = 0; i < TSDB_RETENTION_MAX; ++i) { + for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) { if (i == TSDB_RETENTION_L0) { SMA_OPEN_RSMA_IMPL(pVnode, 0); } else if (i == TSDB_RETENTION_L1) { SMA_OPEN_RSMA_IMPL(pVnode, 1); } else if (i == TSDB_RETENTION_L2) { SMA_OPEN_RSMA_IMPL(pVnode, 2); - } else { - ASSERT(0); } } // restore the rsma - if (tdRSmaRestore(pSma, RSMA_RESTORE_REBOOT, pVnode->state.committed) < 0) { - goto _err; + if (tdRSmaRestore(pSma, RSMA_RESTORE_REBOOT, pVnode->state.committed, rollback) < 0) { + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } } - return 0; -_err: - return -1; +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } + return code; } int32_t smaClose(SSma *pSma) { @@ -181,8 +186,11 @@ int32_t smaClose(SSma *pSma) { * @param committedVer * @return int32_t */ -int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer) { - ASSERT(VND_IS_RSMA(pSma->pVnode)); +int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer, int8_t rollback) { + if (!VND_IS_RSMA(pSma->pVnode)) { + terrno = TSDB_CODE_RSMA_INVALID_ENV; + return TSDB_CODE_FAILED; + } - return tdRSmaProcessRestoreImpl(pSma, type, committedVer); + return tdRSmaProcessRestoreImpl(pSma, type, committedVer, rollback); } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index ba99e5515d8b0b15892da433c80305477ab98dd4..809e553ab665b9473e833759c7f0b278df62354e 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -21,17 +21,17 @@ #define RSMA_FETCH_ACTIVE_MAX (1000) // ms #define RSMA_FETCH_INTERVAL (5000) // ms +#define RSMA_NEED_FETCH(r) (RSMA_INFO_ITEM((r), 0)->fetchLevel || RSMA_INFO_ITEM((r), 1)->fetchLevel) + SSmaMgmt smaMgmt = { .inited = 0, .rsetId = -1, }; -#define TD_QTASKINFO_FNAME_PREFIX "qinf.v" - typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem; -typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter; static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); +static void tdUidStoreDestory(STbUidStore *pStore); static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, bool isAdd); static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo, int8_t idx); @@ -57,38 +57,6 @@ struct SRSmaQTaskInfoItem { void *qTaskInfo; }; -struct SRSmaQTaskInfoIter { - STFile *pTFile; - int64_t offset; - int64_t fsize; - int32_t nBytes; - int32_t nAlloc; - char *pBuf; - // ------------ - char *qBuf; // for iterator - int32_t nBufPos; -}; - -void tdRSmaQTaskInfoGetFileName(int32_t vgId, int64_t version, char *outputName) { - tdGetVndFileName(vgId, NULL, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, version, outputName); -} - -void tdRSmaQTaskInfoGetFullName(int32_t vgId, int64_t version, const char *path, char *outputName) { - tdGetVndFileName(vgId, path, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, version, outputName); -} - -void tdRSmaQTaskInfoGetFullPath(int32_t vgId, int8_t level, const char *path, char *outputName) { - tdGetVndDirName(vgId, path, VNODE_RSMA_DIR, true, outputName); - int32_t rsmaLen = strlen(outputName); - snprintf(outputName + rsmaLen, TSDB_FILENAME_LEN - rsmaLen, "%" PRIi8, level); -} - -void tdRSmaQTaskInfoGetFullPathEx(int32_t vgId, tb_uid_t suid, int8_t level, const char *path, char *outputName) { - tdGetVndDirName(vgId, path, VNODE_RSMA_DIR, true, outputName); - int32_t rsmaLen = strlen(outputName); - snprintf(outputName + rsmaLen, TSDB_FILENAME_LEN - rsmaLen, "%" PRIi64 "%s%" PRIi8, suid, TD_DIRSEP, level); -} - static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) { // Note: free/kill may in RC if (!taskHandle || !(*taskHandle)) return; @@ -161,12 +129,17 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { } static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) { - ASSERT(*pStore == NULL); + if (ASSERTS(*pStore == NULL, "*pStore:%p != NULL", *pStore)) { + terrno = TSDB_CODE_APP_ERROR; + return TSDB_CODE_FAILED; + } + *pStore = taosMemoryCalloc(1, sizeof(STbUidStore)); if (*pStore == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_FAILED; } + return TSDB_CODE_SUCCESS; } @@ -195,15 +168,14 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pRSmaInfo->taskInfo[i]) { - if (((terrno = qUpdateQualifiedTableId(pRSmaInfo->taskInfo[i], tbUids, isAdd)) < 0)) { + if ((terrno = qUpdateQualifiedTableId(pRSmaInfo->taskInfo[i], tbUids, isAdd)) < 0) { tdReleaseRSmaInfo(pSma, pRSmaInfo); smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " level %d since %s", SMA_VID(pSma), *suid, i, terrstr()); return TSDB_CODE_FAILED; - } else { - smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 " uid:%" PRIi64 " level %d", - SMA_VID(pSma), pRSmaInfo->taskInfo[0], *suid, *(int64_t *)taosArrayGet(tbUids, 0), i); } + smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 " uid:%" PRIi64 " level %d", + SMA_VID(pSma), pRSmaInfo->taskInfo[i], *suid, *(int64_t *)taosArrayGet(tbUids, 0), i); } } @@ -264,8 +236,6 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui return TSDB_CODE_SUCCESS; } - ASSERT(ppStore != NULL); - if (!(*ppStore)) { if (tdUidStoreInit(ppStore) < 0) { return TSDB_CODE_FAILED; @@ -332,12 +302,15 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat } pItem->level = idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2; - ASSERT(pItem->level > 0); + + if (ASSERTS(pItem->level > 0, "pItem level:%" PRIi8 " should > 0", pItem->level)) { + terrno = TSDB_CODE_APP_ERROR; + return TSDB_CODE_FAILED; + } SRSmaRef rsmaRef = {.refId = pStat->refId, .suid = pRSmaInfo->suid}; taosHashPut(smaMgmt.refHash, &pItem, POINTER_BYTES, &rsmaRef, sizeof(rsmaRef)); - pItem->fetchLevel = pItem->level; taosTmrReset(tdRSmaFetchTrigger, RSMA_FETCH_INTERVAL, pItem, smaMgmt.tmrHandle, &pItem->tmrId); smaInfo("vgId:%d, item:%p table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64 @@ -363,10 +336,12 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con return TSDB_CODE_SUCCESS; } +#if 0 if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_ROLLUP) != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TDB_INIT_FAILED; return TSDB_CODE_FAILED; } +#endif SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); @@ -374,13 +349,8 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (pRSmaInfo) { - // TODO: free original pRSmaInfo if exists abnormally - tdFreeRSmaInfo(pSma, *(SRSmaInfo **)pRSmaInfo, true); - if (taosHashRemove(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)) < 0) { - terrno = TSDB_CODE_RSMA_REMOVE_EXISTS; - goto _err; - } - smaWarn("vgId:%d, remove the rsma info already exists for table %s, %" PRIi64, SMA_VID(pSma), tbName, suid); + smaInfo("vgId:%d, rsma info already exists for table %s, %" PRIi64, SMA_VID(pSma), tbName, suid); + return TSDB_CODE_SUCCESS; } // from write queue: single thead @@ -449,8 +419,8 @@ int32_t tdProcessRSmaCreate(SSma *pSma, SVCreateStbReq *pReq) { } if (!VND_IS_RSMA(pVnode)) { - smaTrace("vgId:%d, not create rsma for stable %s %" PRIi64 " since vnd is not rsma", TD_VID(pVnode), pReq->name, - pReq->suid); + smaWarn("vgId:%d, not create rsma for stable %s %" PRIi64 " since vnd is not rsma", TD_VID(pVnode), pReq->name, + pReq->suid); return TSDB_CODE_SUCCESS; } @@ -494,9 +464,8 @@ int32_t tdProcessRSmaDrop(SSma *pSma, SVDropStbReq *pReq) { tdReleaseRSmaInfo(pSma, pRSmaInfo); - // save to file - // TODO - smaDebug("vgId:%d, drop rsma for table %" PRIi64 " succeed", TD_VID(pVnode), pReq->suid); + // no need to save to file as triggered by dropping stable + smaDebug("vgId:%d, drop rsma for stable %" PRIi64 " succeed", TD_VID(pVnode), pReq->suid); return TSDB_CODE_SUCCESS; } @@ -561,7 +530,7 @@ static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid) return TSDB_CODE_SUCCESS; } -void tdUidStoreDestory(STbUidStore *pStore) { +static void tdUidStoreDestory(STbUidStore *pStore) { if (pStore) { if (pStore->uidHash) { if (pStore->tbUids) { @@ -601,8 +570,8 @@ static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { return TSDB_CODE_FAILED; } - SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; - // TODO: spin lock for race conditiond + SSubmitReq2 *pSubmitReq = (SSubmitReq2 *)pReq; + // spin lock for race condition during insert data if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { return TSDB_CODE_FAILED; } @@ -610,29 +579,19 @@ static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { return TSDB_CODE_SUCCESS; } -static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SSubmitBlkIter blkIter = {0}; - STSRow *row = NULL; +static int32_t tdFetchSubmitReqSuids(SSubmitReq2 *pMsg, STbUidStore *pStore) { + SArray *pSubmitTbData = pMsg ? pMsg->aSubmitTbData : NULL; + int32_t size = taosArrayGetSize(pSubmitTbData); terrno = TSDB_CODE_SUCCESS; - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { - return -1; - } - while (true) { - if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) { + for (int32_t i = 0; i < size; ++i) { + SSubmitTbData *pData = TARRAY_GET_ELEM(pSubmitTbData, i); + if ((terrno = tdUidStorePut(pStore, pData->suid, NULL)) < 0) { return -1; } - - if (!pBlock) break; - tdUidStorePut(pStore, msgIter.suid, NULL); } - if (terrno != TSDB_CODE_SUCCESS) { - return -1; - } return 0; } @@ -708,14 +667,14 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma #endif for (int32_t i = 0; i < taosArrayGetSize(pResList); ++i) { SSDataBlock *output = taosArrayGetP(pResList, i); - smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%d", output->info.id.uid, output->info.id.groupId, - output->info.rows); + smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%d", output->info.id.uid, + output->info.id.groupId, output->info.rows); - STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); - SSubmitReq *pReq = NULL; + STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); + SSubmitReq2 *pReq = NULL; // TODO: the schema update should be handled later(TD-17965) - if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, SMA_VID(pSma), suid) < 0) { + if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, output->info.id.groupId, SMA_VID(pSma), suid) < 0) { smaError("vgId:%d, build submit req for rsma table suid:%" PRIu64 ", uid:%" PRIu64 ", level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, terrstr()); @@ -723,19 +682,21 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma } if (pReq && tdProcessSubmitReq(sinkTsdb, output->info.version, pReq) < 0) { - taosMemoryFreeClear(pReq); + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); smaError("vgId:%d, process submit req for rsma suid:%" PRIu64 ", uid:%" PRIu64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, terrstr()); goto _err; } - smaDebug("vgId:%d, process submit req for rsma suid:%" PRIu64 ",uid:%" PRIu64 ", level %" PRIi8 " ver %" PRIi64 - " len %" PRIu32, - SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, output->info.version, - htonl(pReq->header.contLen)); + smaDebug("vgId:%d, process submit req for rsma suid:%" PRIu64 ",uid:%" PRIu64 ", level %" PRIi8 " ver %" PRIi64, + SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, output->info.version); - taosMemoryFreeClear(pReq); + if (pReq) { + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); + } } } @@ -753,22 +714,29 @@ _err: * @brief Copy msg to rsmaQueueBuffer for batch process * * @param pSma + * @param version * @param pMsg + * @param len * @param inputType * @param pInfo * @param suid * @return int32_t */ -static int32_t tdExecuteRSmaImplAsync(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfo *pInfo, - tb_uid_t suid) { - const SSubmitReq *pReq = (const SSubmitReq *)pMsg; +static int32_t tdExecuteRSmaImplAsync(SSma *pSma, int64_t version, const void *pMsg, int32_t len, int32_t inputType, + SRSmaInfo *pInfo, tb_uid_t suid) { + int32_t size = sizeof(int32_t) + sizeof(int64_t) + len; + void *qItem = taosAllocateQitem(size, DEF_QITEM, 0); - void *qItem = taosAllocateQitem(pReq->header.contLen, DEF_QITEM, 0); if (!qItem) { return TSDB_CODE_FAILED; } - memcpy(qItem, pMsg, pReq->header.contLen); + void *pItem = qItem; + + *(int32_t *)pItem = len; + pItem = POINTER_SHIFT(pItem, sizeof(int32_t)); + *(int64_t *)pItem = version; + memcpy(POINTER_SHIFT(pItem, sizeof(int64_t)), pMsg, len); taosWriteQitem(pInfo->queue, qItem); @@ -840,12 +808,13 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, return TSDB_CODE_SUCCESS; } if (!pInfo->pTSchema) { + terrno = TSDB_CODE_INVALID_PTR; smaWarn("vgId:%d, no schema to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, pInfo->suid); return TSDB_CODE_FAILED; } - smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level, - RSMA_INFO_QTASK(pInfo, idx), pInfo->suid); + smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64 " nMsg:%d", SMA_VID(pSma), level, + RSMA_INFO_QTASK(pInfo, idx), pInfo->suid, msgSize); #if 0 for (int32_t i = 0; i < msgSize; ++i) { @@ -854,7 +823,7 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, tdRsmaPrintSubmitReq(pSma, pReq); } #endif - if (qSetSMAInput(qTaskInfo, pMsg, msgSize, inputType) < 0) { + if ((terrno = qSetSMAInput(qTaskInfo, pMsg, msgSize, inputType)) < 0) { smaError("vgId:%d, rsma %" PRIi8 " qSetStreamInput failed since %s", SMA_VID(pSma), level, tstrerror(terrno)); return TSDB_CODE_FAILED; } @@ -867,44 +836,51 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t srcTaskInfo, SRSmaParam *param, tb_uid_t suid, int8_t idx) { + int32_t code = 0; + int32_t lino = 0; SVnode *pVnode = pSma->pVnode; char *pOutput = NULL; int32_t len = 0; - if ((terrno = qSerializeTaskStatus(srcTaskInfo, &pOutput, &len)) < 0) { - smaError("vgId:%d, rsma clone, table %" PRIi64 " serialize qTaskInfo failed since %s", TD_VID(pVnode), suid, - terrstr()); - goto _err; + if (!srcTaskInfo) { + code = TSDB_CODE_INVALID_PTR; + smaWarn("vgId:%d, rsma clone, table %" PRIi64 ", no need since srcTaskInfo is NULL", TD_VID(pVnode), suid); + TSDB_CHECK_CODE(code, lino, _exit); } + code = qSerializeTaskStatus(srcTaskInfo, &pOutput, &len); + TSDB_CHECK_CODE(code, lino, _exit); + SReadHandle handle = { .meta = pVnode->pMeta, .vnode = pVnode, .initTqReader = 1, }; - ASSERT(!dstTaskInfo); + + if (ASSERTS(!dstTaskInfo, "dstTaskInfo:%p is not NULL", dstTaskInfo)) { + code = TSDB_CODE_APP_ERROR; + TSDB_CHECK_CODE(code, lino, _exit); + } + dstTaskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle); if (!dstTaskInfo) { - terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE; - goto _err; + code = TSDB_CODE_RSMA_QTASKINFO_CREATE; + TSDB_CHECK_CODE(code, lino, _exit); } - if (qDeserializeTaskStatus(dstTaskInfo, pOutput, len) < 0) { - smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " failed since %s", TD_VID(pVnode), suid, - terrstr()); - goto _err; - } + code = qDeserializeTaskStatus(dstTaskInfo, pOutput, len); + TSDB_CHECK_CODE(code, lino, _exit); smaDebug("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " succeed", TD_VID(pVnode), suid); +_exit: taosMemoryFreeClear(pOutput); - return TSDB_CODE_SUCCESS; -_err: - taosMemoryFreeClear(pOutput); - tdRSmaQTaskInfoFree(dstTaskInfo, TD_VID(pVnode), idx + 1); - smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " failed since %s", TD_VID(pVnode), suid, - terrstr()); - return TSDB_CODE_FAILED; + if (code) { + tdRSmaQTaskInfoFree(dstTaskInfo, TD_VID(pVnode), idx + 1); + smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " failed since %s", TD_VID(pVnode), suid, + terrstr()); + } + return code; } /** @@ -915,43 +891,53 @@ _err: * @return int32_t */ static int32_t tdRSmaInfoClone(SSma *pSma, SRSmaInfo *pInfo) { + int32_t code = 0; + int32_t lino = 0; SRSmaParam *param = NULL; + SMetaReader mr = {0}; + if (!pInfo) { return TSDB_CODE_SUCCESS; } - SMetaReader mr = {0}; metaReaderInit(&mr, SMA_META(pSma), 0); smaDebug("vgId:%d, rsma clone qTaskInfo for suid:%" PRIi64, SMA_VID(pSma), pInfo->suid); if (metaGetTableEntryByUidCache(&mr, pInfo->suid) < 0) { - smaError("vgId:%d, rsma clone, failed to get table meta for %" PRIi64 " since %s", SMA_VID(pSma), pInfo->suid, - terrstr()); - goto _err; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (mr.me.type != TSDB_SUPER_TABLE) { + code = TSDB_CODE_RSMA_INVALID_SCHEMA; + TSDB_CHECK_CODE(code, lino, _exit); } - ASSERT(mr.me.type == TSDB_SUPER_TABLE); - ASSERT(mr.me.uid == pInfo->suid); + if (mr.me.uid != pInfo->suid) { + code = TSDB_CODE_RSMA_INVALID_SCHEMA; + TSDB_CHECK_CODE(code, lino, _exit); + } + if (TABLE_IS_ROLLUP(mr.me.flags)) { param = &mr.me.stbEntry.rsmaParam; for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (!pInfo->iTaskInfo[i]) { continue; } - if (tdCloneQTaskInfo(pSma, pInfo->taskInfo[i], pInfo->iTaskInfo[i], param, pInfo->suid, i) < 0) { - goto _err; - } + code = tdCloneQTaskInfo(pSma, pInfo->taskInfo[i], pInfo->iTaskInfo[i], param, pInfo->suid, i); + TSDB_CHECK_CODE(code, lino, _exit); } smaDebug("vgId:%d, rsma clone env success for %" PRIi64, SMA_VID(pSma), pInfo->suid); } else { - terrno = TSDB_CODE_RSMA_INVALID_SCHEMA; - goto _err; + code = TSDB_CODE_RSMA_INVALID_SCHEMA; + TSDB_CHECK_CODE(code, lino, _exit); } +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s, suid:%" PRIi64 ", flags:%" PRIi8 ",type:%" PRIi8 ", uid:%" PRIi64, + SMA_VID(pSma), __func__, lino, tstrerror(code), pInfo->suid, mr.me.flags, mr.me.type, mr.me.uid); + } metaReaderClear(&mr); - return TSDB_CODE_SUCCESS; -_err: - metaReaderClear(&mr); - smaError("vgId:%d, rsma clone env failed for %" PRIi64 " since %s", SMA_VID(pSma), pInfo->suid, terrstr()); - return TSDB_CODE_FAILED; + return code; } /** @@ -962,10 +948,14 @@ _err: * @return SRSmaInfo* */ static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) { + int32_t code = 0; + int32_t lino = 0; SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pStat = NULL; SRSmaInfo *pRSmaInfo = NULL; + terrno = 0; + if (!pEnv) { terrno = TSDB_CODE_RSMA_INVALID_ENV; return NULL; @@ -985,14 +975,17 @@ static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) { return NULL; } if (!pRSmaInfo->taskInfo[0]) { - if (tdRSmaInfoClone(pSma, pRSmaInfo) < 0) { + if ((terrno = tdRSmaInfoClone(pSma, pRSmaInfo)) < 0) { taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); return NULL; } } tdRefRSmaInfo(pSma, pRSmaInfo); taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); - ASSERT(pRSmaInfo->suid == suid); + if (ASSERTS(pRSmaInfo->suid == suid, "suid:%" PRIi64 " != %" PRIi64, pRSmaInfo->suid, suid)) { + terrno = TSDB_CODE_APP_ERROR; + return NULL; + } return pRSmaInfo; } taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); @@ -1010,12 +1003,14 @@ static FORCE_INLINE void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { * @brief async mode * * @param pSma + * @param version * @param pMsg * @param inputType * @param suid * @return int32_t */ -static int32_t tdExecuteRSmaAsync(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) { +static int32_t tdExecuteRSmaAsync(SSma *pSma, int64_t version, const void *pMsg, int32_t len, int32_t inputType, + tb_uid_t suid) { SRSmaInfo *pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, suid); if (!pRSmaInfo) { smaDebug("vgId:%d, execute rsma, no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); @@ -1023,7 +1018,7 @@ static int32_t tdExecuteRSmaAsync(SSma *pSma, const void *pMsg, int32_t inputTyp } if (inputType == STREAM_INPUT__DATA_SUBMIT) { - if (tdExecuteRSmaImplAsync(pSma, pMsg, inputType, pRSmaInfo, suid) < 0) { + if (tdExecuteRSmaImplAsync(pSma, version, pMsg, len, inputType, pRSmaInfo, suid) < 0) { tdReleaseRSmaInfo(pSma, pRSmaInfo); return TSDB_CODE_FAILED; } @@ -1038,40 +1033,43 @@ static int32_t tdExecuteRSmaAsync(SSma *pSma, const void *pMsg, int32_t inputTyp } } } else { - ASSERT(0); + terrno = TSDB_CODE_APP_ERROR; + tdReleaseRSmaInfo(pSma, pRSmaInfo); + smaError("vgId:%d, execute rsma, failed for suid:%" PRIu64 " since %s, type:%d", SMA_VID(pSma), suid, + tstrerror(terrno), inputType); + return TSDB_CODE_FAILED; } tdReleaseRSmaInfo(pSma, pRSmaInfo); return TSDB_CODE_SUCCESS; } -int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { +int32_t tdProcessRSmaSubmit(SSma *pSma, int64_t version, void *pReq, void *pMsg, int32_t len, int32_t inputType) { SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); if (!pEnv) { // only applicable when rsma env exists return TSDB_CODE_SUCCESS; } + STbUidStore uidStore = {0}; - SRetention *pRetention = SMA_RETENTION(pSma); - if (!RETENTION_VALID(pRetention + 1)) { - // return directly if retention level 1 is invalid - return TSDB_CODE_SUCCESS; - } if (inputType == STREAM_INPUT__DATA_SUBMIT) { - if (tdFetchSubmitReqSuids(pMsg, &uidStore) < 0) { + if (tdFetchSubmitReqSuids(pReq, &uidStore) < 0) { + smaError("vgId:%d, failed to process rsma submit fetch suid since: %s", SMA_VID(pSma), terrstr()); goto _err; } if (uidStore.suid != 0) { - if (tdExecuteRSmaAsync(pSma, pMsg, inputType, uidStore.suid) < 0) { + if (tdExecuteRSmaAsync(pSma, version, pMsg, len, inputType, uidStore.suid) < 0) { + smaError("vgId:%d, failed to process rsma submit exec 1 since: %s", SMA_VID(pSma), terrstr()); goto _err; } void *pIter = NULL; while ((pIter = taosHashIterate(uidStore.uidHash, pIter))) { tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - if (tdExecuteRSmaAsync(pSma, pMsg, inputType, *pTbSuid) < 0) { + if (tdExecuteRSmaAsync(pSma, version, pMsg, len, inputType, *pTbSuid) < 0) { + smaError("vgId:%d, failed to process rsma submit exec 2 since: %s", SMA_VID(pSma), terrstr()); goto _err; } } @@ -1081,7 +1079,6 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { return TSDB_CODE_SUCCESS; _err: tdUidStoreDestory(&uidStore); - smaError("vgId:%d, failed to process rsma submit since: %s", SMA_VID(pSma), terrstr()); return TSDB_CODE_FAILED; } @@ -1093,19 +1090,22 @@ _err: * @return int32_t */ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { + int32_t code = 0; + int32_t lino = 0; SVnode *pVnode = pSma->pVnode; SArray *suidList = NULL; STbUidStore uidStore = {0}; SMetaReader mr = {0}; + tb_uid_t suid = 0; if (!(suidList = taosArrayInit(1, sizeof(tb_uid_t)))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } if (vnodeGetStbIdList(pSma->pVnode, 0, suidList) < 0) { - smaError("vgId:%d, failed to restore rsma env since get stb id list error: %s", TD_VID(pVnode), terrstr()); - goto _err; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } int64_t arrSize = taosArrayGetSize(suidList); @@ -1122,19 +1122,26 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { int64_t nRsmaTables = 0; metaReaderInit(&mr, SMA_META(pSma), 0); if (!(uidStore.tbUids = taosArrayInit(1024, sizeof(tb_uid_t)))) { - goto _err; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } + for (int64_t i = 0; i < arrSize; ++i) { - tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); + suid = *(tb_uid_t *)taosArrayGet(suidList, i); smaDebug("vgId:%d, rsma restore, suid is %" PRIi64, TD_VID(pVnode), suid); if (metaGetTableEntryByUidCache(&mr, suid) < 0) { - smaError("vgId:%d, rsma restore, failed to get table meta for %" PRIi64 " since %s", TD_VID(pVnode), suid, - terrstr()); - goto _err; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } tDecoderClear(&mr.coder); - ASSERT(mr.me.type == TSDB_SUPER_TABLE); - ASSERT(mr.me.uid == suid); + if (mr.me.type != TSDB_SUPER_TABLE) { + code = TSDB_CODE_RSMA_INVALID_SCHEMA; + TSDB_CHECK_CODE(code, lino, _exit); + } + if (mr.me.uid != suid) { + code = TSDB_CODE_RSMA_INVALID_SCHEMA; + TSDB_CHECK_CODE(code, lino, _exit); + } if (TABLE_IS_ROLLUP(mr.me.flags)) { ++nRsmaTables; SRSmaParam *param = &mr.me.stbEntry.rsmaParam; @@ -1144,22 +1151,20 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { TD_VID(pVnode), suid, i, param->maxdelay[i], param->watermark[i], param->qmsgLen[i]); } if (tdRSmaProcessCreateImpl(pSma, &mr.me.stbEntry.rsmaParam, suid, mr.me.name) < 0) { - smaError("vgId:%d, rsma restore env failed for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr()); - goto _err; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } // reload all ctbUids for suid uidStore.suid = suid; if (vnodeGetCtbIdList(pVnode, suid, uidStore.tbUids) < 0) { - smaError("vgId:%d, rsma restore, get ctb idlist failed for %" PRIi64 " since %s", TD_VID(pVnode), suid, - terrstr()); - goto _err; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } if (tdUpdateTbUidList(pVnode->pSma, &uidStore, true) < 0) { - smaError("vgId:%d, rsma restore, update tb uid list failed for %" PRIi64 " since %s", TD_VID(pVnode), suid, - terrstr()); - goto _err; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } taosArrayClear(uidStore.tbUids); @@ -1168,36 +1173,36 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { } } - metaReaderClear(&mr); - taosArrayDestroy(suidList); - tdUidStoreDestory(&uidStore); - if (nTables) { *nTables = nRsmaTables; } - - return TSDB_CODE_SUCCESS; -_err: +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s, suid:%" PRIi64 ", type:%" PRIi8 ", uid:%" PRIi64, TD_VID(pVnode), + __func__, lino, tstrerror(code), suid, mr.me.type, mr.me.uid); + } metaReaderClear(&mr); taosArrayDestroy(suidList); tdUidStoreDestory(&uidStore); - - return TSDB_CODE_FAILED; + return code; } /** - * @brief reload ts data from checkpoint - * - * @param pSma - * @return int32_t + * N.B. the data would be restored from the unified WAL replay procedure */ -static int32_t tdRSmaRestoreTSDataReload(SSma *pSma) { - // NOTHING TODO: the data would be restored from the unified WAL replay procedure - return TSDB_CODE_SUCCESS; -} +int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer, int8_t rollback) { + // step 1: init env + if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_ROLLUP) != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TDB_INIT_FAILED; + return TSDB_CODE_FAILED; + } -int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer) { - // step 1: iterate all stables to restore the rsma env + // step 2: open SRSmaFS for qTaskFiles + if (tdRSmaFSOpen(pSma, qtaskFileVer, rollback) < 0) { + goto _err; + } + + // step 3: iterate all stables to restore the rsma env int64_t nTables = 0; if (tdRSmaRestoreQTaskInfoInit(pSma, &nTables) < 0) { goto _err; @@ -1207,16 +1212,6 @@ int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer) return TSDB_CODE_SUCCESS; } - // step 2: reload ts data from checkpoint - if (tdRSmaRestoreTSDataReload(pSma) < 0) { - goto _err; - } - - // step 3: open SRSmaFS for qTaskFiles - if (tdRSmaFSOpen(pSma, qtaskFileVer) < 0) { - goto _err; - } - smaInfo("vgId:%d, restore rsma task %" PRIi8 " from qtaskf %" PRIi64 " succeed", SMA_VID(pSma), type, qtaskFileVer); return TSDB_CODE_SUCCESS; _err: @@ -1226,19 +1221,26 @@ _err: } int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { - SSma *pSma = pRSmaStat->pSma; - SVnode *pVnode = pSma->pVnode; - int32_t vid = SMA_VID(pSma); + int32_t code = 0; + int32_t lino = 0; + SSma *pSma = pRSmaStat->pSma; + SVnode *pVnode = pSma->pVnode; + SArray *qTaskFArray = NULL; + int64_t version = pRSmaStat->commitAppliedVer; + TdFilePtr pOutFD = NULL; + TdFilePtr pInFD = NULL; + char fname[TSDB_FILENAME_LEN]; + char fnameVer[TSDB_FILENAME_LEN]; + SRSmaFS fs = {0}; if (taosHashGetSize(pInfoHash) <= 0) { return TSDB_CODE_SUCCESS; } - int64_t fsMaxVer = tdRSmaFSMaxVer(pSma, pRSmaStat); - if (pRSmaStat->commitAppliedVer <= fsMaxVer) { - smaDebug("vgId:%d, rsma persist, no need as applied %" PRIi64 " not larger than fsMaxVer %" PRIi64, vid, - pRSmaStat->commitAppliedVer, fsMaxVer); - return TSDB_CODE_SUCCESS; + qTaskFArray = taosArrayInit(taosHashGetSize(pInfoHash) << 1, sizeof(SQTaskFile)); + if (!qTaskFArray) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } void *infoHash = NULL; @@ -1253,19 +1255,79 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pRSmaInfo, i); if (pItem && pItem->pStreamState) { if (streamStateCommit(pItem->pStreamState) < 0) { - terrno = TSDB_CODE_RSMA_STREAM_STATE_COMMIT; - goto _err; + code = TSDB_CODE_RSMA_STREAM_STATE_COMMIT; + TSDB_CHECK_CODE(code, lino, _exit); + } + smaDebug("vgId:%d, rsma persist, stream state commit success, table %" PRIi64 ", level %d", TD_VID(pVnode), + pRSmaInfo->suid, i + 1); + + // qTaskInfo file + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pRSmaInfo->suid, i + 1, -1, tfsGetPrimaryPath(pVnode->pTfs), fname); + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pRSmaInfo->suid, i + 1, version, tfsGetPrimaryPath(pVnode->pTfs), + fnameVer); + if (taosCheckExistFile(fnameVer)) { + smaWarn("vgId:%d, rsma persist, duplicate file %s exist", TD_VID(pVnode), fnameVer); + } + + pOutFD = taosCreateFile(fnameVer, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (pOutFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); } - smaDebug("vgId:%d, rsma persist, stream state commit success, table %" PRIi64 " level %d", vid, pRSmaInfo->suid, - i + 1); + pInFD = taosOpenFile(fname, TD_FILE_READ); + if (pInFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + int64_t size = 0; + uint32_t mtime = 0; + if (taosFStatFile(pInFD, &size, &mtime) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + int64_t offset = 0; + if (taosFSendFile(pOutFD, pInFD, &offset, size) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + smaError("vgId:%d, rsma persist, send qtaskinfo file %s to %s failed since %s", TD_VID(pVnode), fname, + fnameVer, tstrerror(code)); + TSDB_CHECK_CODE(code, lino, _exit); + } + taosCloseFile(&pOutFD); + taosCloseFile(&pInFD); + + SQTaskFile qTaskF = { + .nRef = 1, .level = i + 1, .suid = pRSmaInfo->suid, .version = version, .size = size, .mtime = mtime}; + + taosArrayPush(qTaskFArray, &qTaskF); } } } - return TSDB_CODE_SUCCESS; -_err: - smaError("vgId:%d, rsma persist failed since %s", vid, terrstr()); - return TSDB_CODE_FAILED; + // prepare + code = tdRSmaFSCopy(pSma, &fs); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tdRSmaFSUpsertQTaskFile(pSma, &fs, qTaskFArray->pData, taosArrayGetSize(qTaskFArray)); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tdRSmaFSPrepareCommit(pSma, &fs); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + + taosArrayDestroy(fs.aQTaskInf); + taosArrayDestroy(qTaskFArray); + + if (code) { + if (pOutFD) taosCloseFile(&pOutFD); + if (pInFD) taosCloseFile(&pInFD); + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } + + terrno = code; + return code; } /** @@ -1343,21 +1405,18 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { // async process pItem->fetchLevel = pItem->level; #if 0 + // debugging codes SRSmaInfo *qInfo = tdAcquireRSmaInfoBySuid(pSma, pRSmaInfo->suid); SRSmaInfoItem *qItem = RSMA_INFO_ITEM(qInfo, pItem->level - 1); - ASSERT(qItem->level == pItem->level); - ASSERT(qItem->fetchLevel == pItem->fetchLevel); + make sure(qItem->level == pItem->level); + make sure(qItem->fetchLevel == pItem->fetchLevel); #endif if (atomic_load_8(&pRSmaInfo->assigned) == 0) { tsem_post(&(pStat->notEmpty)); } } break; - case TASK_TRIGGER_STAT_PAUSED: { - smaDebug("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid:%" PRIi64 " since stat is paused", - SMA_VID(pSma), pItem->level, pRSmaInfo->suid); - } break; case TASK_TRIGGER_STAT_INACTIVE: { - smaDebug("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid:%" PRIi64 " since stat is inactive", + smaDebug("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid:%" PRIi64 " since stat is inactive ", SMA_VID(pSma), pItem->level, pRSmaInfo->suid); } break; case TASK_TRIGGER_STAT_INIT: { @@ -1365,8 +1424,9 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { SMA_VID(pSma), pItem->level, pRSmaInfo->suid); } break; default: { - smaDebug("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid:%" PRIi64 " since stat is unknown", - SMA_VID(pSma), pItem->level, pRSmaInfo->suid); + smaDebug("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid:%" PRIi64 " since stat:%" PRIi8 + " is unknown", + SMA_VID(pSma), pItem->level, pRSmaInfo->suid, fetchTriggerStat); } break; } @@ -1378,7 +1438,8 @@ _end: static void tdFreeRSmaSubmitItems(SArray *pItems) { for (int32_t i = 0; i < taosArrayGetSize(pItems); ++i) { - taosFreeQitem(*(void **)taosArrayGet(pItems, i)); + SPackedData *packData = taosArrayGet(pItems, i); + taosFreeQitem(POINTER_SHIFT(packData->msgStr, -sizeof(int32_t) - sizeof(int64_t))); } taosArrayClear(pItems); } @@ -1447,7 +1508,11 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA void *msg = NULL; taosGetQitem(qall, (void **)&msg); if (msg) { - if (!taosArrayPush(pSubmitArr, &msg)) { + SPackedData packData = {.msgLen = *(int32_t *)msg, + .ver = *(int64_t *)POINTER_SHIFT(msg, sizeof(int32_t)), + .msgStr = POINTER_SHIFT(msg, sizeof(int32_t) + sizeof(int64_t))}; + + if (!taosArrayPush(pSubmitArr, &packData)) { tdFreeRSmaSubmitItems(pSubmitArr); goto _err; } @@ -1460,7 +1525,6 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA if (size > 0) { for (int32_t i = 1; i <= TSDB_RETENTION_L2; ++i) { if (tdExecuteRSmaImpl(pSma, pSubmitArr->pData, size, STREAM_INPUT__MERGED_SUBMIT, pInfo, type, i) < 0) { - tdFreeRSmaSubmitItems(pSubmitArr); goto _err; } } @@ -1468,6 +1532,9 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA } return TSDB_CODE_SUCCESS; _err: + smaError("vgId:%d, batch exec for suid:%" PRIi64 " execType:%d size:%d failed since %s", SMA_VID(pSma), pInfo->suid, + type, (int32_t)taosArrayGetSize(pSubmitArr), terrstr()); + tdFreeRSmaSubmitItems(pSubmitArr); while (1) { void *msg = NULL; taosGetQitem(qall, (void **)&msg); @@ -1489,6 +1556,8 @@ _err: */ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { + int32_t code = 0; + int32_t lino = 0; SVnode *pVnode = pSma->pVnode; SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); @@ -1497,14 +1566,14 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { bool isFetchAll = false; if (!pRSmaStat || !(infoHash = RSMA_INFO_HASH(pRSmaStat))) { - terrno = TSDB_CODE_RSMA_INVALID_STAT; - goto _err; + code = TSDB_CODE_RSMA_INVALID_STAT; + TSDB_CHECK_CODE(code, lino, _exit); } if (!(pSubmitArr = - taosArrayInit(TMIN(RSMA_SUBMIT_BATCH_SIZE, atomic_load_64(&pRSmaStat->nBufItems)), POINTER_BYTES))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + taosArrayInit(TMIN(RSMA_SUBMIT_BATCH_SIZE, atomic_load_64(&pRSmaStat->nBufItems)), sizeof(SPackedData)))) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } while (true) { @@ -1514,8 +1583,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { while ((pIter = taosHashIterate(infoHash, pIter))) { SRSmaInfo *pInfo = *(SRSmaInfo **)pIter; if (atomic_val_compare_exchange_8(&pInfo->assigned, 0, 1) == 0) { - if ((taosQueueItemSize(pInfo->queue) > 0) || RSMA_INFO_ITEM(pInfo, 0)->fetchLevel || - RSMA_INFO_ITEM(pInfo, 1)->fetchLevel) { + if ((taosQueueItemSize(pInfo->queue) > 0) || RSMA_NEED_FETCH(pInfo)) { int32_t batchCnt = -1; int32_t batchMax = taosHashGetSize(infoHash) / tsNumOfVnodeRsmaThreads; bool occupied = (batchMax <= 1); @@ -1531,13 +1599,24 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { smaDebug("vgId:%d, batchSize:%d, execType:%" PRIi32, SMA_VID(pSma), qallItemSize, type); } - if (RSMA_INFO_ITEM(pInfo, 0)->fetchLevel || RSMA_INFO_ITEM(pInfo, 1)->fetchLevel) { + if (RSMA_NEED_FETCH(pInfo)) { int8_t oldStat = atomic_val_compare_exchange_8(RSMA_COMMIT_STAT(pRSmaStat), 0, 2); if (oldStat == 0 || ((oldStat == 2) && atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)) < TASK_TRIGGER_STAT_PAUSED)) { int32_t oldVal = atomic_fetch_add_32(&pRSmaStat->nFetchAll, 1); - ASSERT(oldVal >= 0); - tdRSmaFetchAllResult(pSma, pInfo); + + if (ASSERTS(oldVal >= 0, "oldVal of nFetchAll: %d < 0", oldVal)) { + code = TSDB_CODE_APP_ERROR; + TSDB_CHECK_CODE(code, lino, _exit); + } + + int8_t curStat = atomic_load_8(RSMA_COMMIT_STAT(pRSmaStat)); + if (curStat == 1) { + smaDebug("vgId:%d, fetch all not exec as commit stat is %" PRIi8, SMA_VID(pSma), curStat); + } else { + tdRSmaFetchAllResult(pSma, pInfo); + } + if (0 == atomic_sub_fetch_32(&pRSmaStat->nFetchAll, 1)) { atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 0); } @@ -1547,17 +1626,9 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { if (qallItemSize > 0) { atomic_fetch_sub_64(&pRSmaStat->nBufItems, qallItemSize); continue; - } else if (RSMA_INFO_ITEM(pInfo, 0)->fetchLevel || RSMA_INFO_ITEM(pInfo, 1)->fetchLevel) { - if (atomic_load_8(RSMA_COMMIT_STAT(pRSmaStat)) == 0) { - continue; - } - for (int32_t j = 0; j < TSDB_RETENTION_L2; ++j) { - SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, j); - if (pItem->fetchLevel) { - pItem->fetchLevel = 0; - taosTmrReset(tdRSmaFetchTrigger, RSMA_FETCH_INTERVAL, pItem, smaMgmt.tmrHandle, &pItem->tmrId); - } - } + } + if (RSMA_NEED_FETCH(pInfo)) { + continue; } break; @@ -1567,7 +1638,9 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { } } } else { - ASSERT(0); + ASSERTS(0, "unknown rsma exec type:%d", (int32_t)type); + code = TSDB_CODE_APP_ERROR; + TSDB_CHECK_CODE(code, lino, _exit); } if (atomic_load_64(&pRSmaStat->nBufItems) <= 0) { @@ -1586,10 +1659,10 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { } // end of while(true) -_end: +_exit: taosArrayDestroy(pSubmitArr); - return TSDB_CODE_SUCCESS; -_err: - taosArrayDestroy(pSubmitArr); - return TSDB_CODE_FAILED; + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } + return code; } diff --git a/source/dnode/vnode/src/sma/smaSnapshot.c b/source/dnode/vnode/src/sma/smaSnapshot.c index 34f884f9f95776be10f1416fdf50f99a8fafce6e..de3d93395aa3790dcd231b461470ee35462abb46 100644 --- a/source/dnode/vnode/src/sma/smaSnapshot.c +++ b/source/dnode/vnode/src/sma/smaSnapshot.c @@ -17,14 +17,13 @@ static int32_t rsmaSnapReadQTaskInfo(SRSmaSnapReader* pReader, uint8_t** ppData); static int32_t rsmaSnapWriteQTaskInfo(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData); -static int32_t rsmaQTaskInfSnapReaderOpen(SRSmaSnapReader* pReader, int64_t version); -static int32_t rsmaQTaskInfSnapReaderClose(SQTaskFReader** ppReader); // SRSmaSnapReader ======================================== struct SRSmaSnapReader { SSma* pSma; int64_t sver; int64_t ever; + SRSmaFS fs; // for data file int8_t rsmaDataDone[TSDB_RETENTION_L2]; @@ -32,19 +31,23 @@ struct SRSmaSnapReader { // for qtaskinfo file int8_t qTaskDone; + int32_t fsIter; SQTaskFReader* pQTaskFReader; }; int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapReader** ppReader) { int32_t code = 0; + int32_t lino = 0; SVnode* pVnode = pSma->pVnode; SRSmaSnapReader* pReader = NULL; + SSmaEnv* pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat* pStat = (SRSmaStat*)SMA_ENV_STAT(pEnv); // alloc pReader = (SRSmaSnapReader*)taosMemoryCalloc(1, sizeof(*pReader)); if (pReader == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pReader->pSma = pSma; pReader->sver = sver; @@ -55,171 +58,147 @@ int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapRead if (pSma->pRSmaTsdb[i]) { code = tsdbSnapReaderOpen(pSma->pRSmaTsdb[i], sver, ever, i == 0 ? SNAP_DATA_RSMA1 : SNAP_DATA_RSMA2, &pReader->pDataReader[i]); - if (code < 0) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _exit); } } // open qtaskinfo - if ((code = rsmaQTaskInfSnapReaderOpen(pReader, ever)) < 0) { - goto _err; + taosRLockLatch(RSMA_FS_LOCK(pStat)); + code = tdRSmaFSRef(pSma, &pReader->fs); + taosRUnLockLatch(RSMA_FS_LOCK(pStat)); + TSDB_CHECK_CODE(code, lino, _exit); + + if (taosArrayGetSize(pReader->fs.aQTaskInf) > 0) { + pReader->pQTaskFReader = taosMemoryCalloc(1, sizeof(SQTaskFReader)); + if (!pReader->pQTaskFReader) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + pReader->pQTaskFReader->pSma = pSma; + pReader->pQTaskFReader->version = pReader->ever; } *ppReader = pReader; - - return TSDB_CODE_SUCCESS; -_err: - if (pReader) rsmaSnapReaderClose(&pReader); - *ppReader = NULL; - smaError("vgId:%d, vnode snapshot rsma reader open failed since %s", TD_VID(pVnode), tstrerror(code)); - return TSDB_CODE_FAILED; -} - -static int32_t rsmaQTaskInfSnapReaderOpen(SRSmaSnapReader* pReader, int64_t version) { - int32_t code = 0; - SSma* pSma = pReader->pSma; - SVnode* pVnode = pSma->pVnode; - SSmaEnv* pEnv = NULL; - SRSmaStat* pStat = NULL; - - if (!(pEnv = SMA_RSMA_ENV(pSma))) { - smaInfo("vgId:%d, vnode snapshot rsma reader for qtaskinfo version %" PRIi64 " not need as env is NULL", - TD_VID(pVnode), version); - return TSDB_CODE_SUCCESS; - } - - pStat = (SRSmaStat*)SMA_ENV_STAT(pEnv); - - int32_t ref = tdRSmaFSRef(pReader->pSma, pStat, version); - if (ref < 1) { - smaInfo("vgId:%d, vnode snapshot rsma reader for qtaskinfo version %" PRIi64 " not need as ref is %d", - TD_VID(pVnode), version, ref); - return TSDB_CODE_SUCCESS; - } - - char qTaskInfoFullName[TSDB_FILENAME_LEN]; - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), version, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); - - if (!taosCheckExistFile(qTaskInfoFullName)) { - tdRSmaFSUnRef(pSma, pStat, version); - smaInfo("vgId:%d, vnode snapshot rsma reader for qtaskinfo version %" PRIi64 " not need as %s not exist", - TD_VID(pVnode), version, qTaskInfoFullName); - return TSDB_CODE_SUCCESS; - } - - pReader->pQTaskFReader = taosMemoryCalloc(1, sizeof(SQTaskFReader)); - if (!pReader->pQTaskFReader) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _end; - } - - TdFilePtr fp = taosOpenFile(qTaskInfoFullName, TD_FILE_READ); - if (!fp) { - code = TAOS_SYSTEM_ERROR(errno); - taosMemoryFreeClear(pReader->pQTaskFReader); - goto _end; - } - - pReader->pQTaskFReader->pReadH = fp; - pReader->pQTaskFReader->pSma = pSma; - pReader->pQTaskFReader->version = pReader->ever; - -_end: - if (code < 0) { - tdRSmaFSUnRef(pSma, pStat, version); - smaError("vgId:%d, vnode snapshot rsma reader open %s succeed", TD_VID(pVnode), qTaskInfoFullName); - return TSDB_CODE_FAILED; - } - - smaInfo("vgId:%d, vnode snapshot rsma reader open %s succeed", TD_VID(pVnode), qTaskInfoFullName); - return TSDB_CODE_SUCCESS; -} - -static int32_t rsmaQTaskInfSnapReaderClose(SQTaskFReader** ppReader) { - if (!(*ppReader)) { - return TSDB_CODE_SUCCESS; +_exit: + if (code) { + if (pReader) rsmaSnapReaderClose(&pReader); + *ppReader = NULL; + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); } - - SSma* pSma = (*ppReader)->pSma; - SRSmaStat* pStat = SMA_RSMA_STAT(pSma); - int64_t version = (*ppReader)->version; - - taosCloseFile(&(*ppReader)->pReadH); - tdRSmaFSUnRef(pSma, pStat, version); - taosMemoryFreeClear(*ppReader); - smaInfo("vgId:%d, vnode snapshot rsma reader closed for qTaskInfo version %" PRIi64, SMA_VID(pSma), version); - - return TSDB_CODE_SUCCESS; + return code; } static int32_t rsmaSnapReadQTaskInfo(SRSmaSnapReader* pReader, uint8_t** ppBuf) { int32_t code = 0; - SSma* pSma = pReader->pSma; + int32_t lino = 0; + SVnode* pVnode = pReader->pSma->pVnode; + SQTaskFReader* qReader = pReader->pQTaskFReader; + SRSmaFS* pFS = &pReader->fs; int64_t n = 0; uint8_t* pBuf = NULL; - SQTaskFReader* qReader = pReader->pQTaskFReader; + int64_t version = pReader->ever; + char fname[TSDB_FILENAME_LEN]; if (!qReader) { *ppBuf = NULL; - smaInfo("vgId:%d, vnode snapshot rsma reader qtaskinfo, qTaskReader is NULL", SMA_VID(pSma)); - return 0; + smaInfo("vgId:%d, vnode snapshot rsma reader qtaskinfo, not needed since qTaskReader is NULL", TD_VID(pVnode)); + goto _exit; + } + + if (pReader->fsIter >= taosArrayGetSize(pFS->aQTaskInf)) { + *ppBuf = NULL; + smaInfo("vgId:%d, vnode snapshot rsma reader qtaskinfo, fsIter reach end", TD_VID(pVnode)); + goto _exit; + } + + while (pReader->fsIter < taosArrayGetSize(pFS->aQTaskInf)) { + SQTaskFile* qTaskF = taosArrayGet(pFS->aQTaskInf, pReader->fsIter++); + if (qTaskF->version != version) { + continue; + } + + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), qTaskF->suid, qTaskF->level, version, tfsGetPrimaryPath(pVnode->pTfs), + fname); + if (!taosCheckExistFile(fname)) { + smaError("vgId:%d, vnode snapshot rsma reader for qtaskinfo, table %" PRIi64 ", level %" PRIi8 + ", version %" PRIi64 " failed since %s not exist", + TD_VID(pVnode), qTaskF->suid, qTaskF->level, version, fname); + code = TSDB_CODE_RSMA_FS_SYNC; + TSDB_CHECK_CODE(code, lino, _exit); + } + + TdFilePtr fp = taosOpenFile(fname, TD_FILE_READ); + if (!fp) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + qReader->pReadH = fp; + qReader->level = qTaskF->level; + qReader->suid = qTaskF->suid; } if (!qReader->pReadH) { *ppBuf = NULL; - smaInfo("vgId:%d, vnode snapshot rsma reader qtaskinfo, readh is NULL", SMA_VID(pSma)); - return 0; + smaInfo("vgId:%d, vnode snapshot rsma reader qtaskinfo, not needed since readh is NULL", TD_VID(pVnode)); + goto _exit; } int64_t size = 0; if (taosFStatFile(qReader->pReadH, &size, NULL) < 0) { code = TAOS_SYSTEM_ERROR(errno); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } // seek if (taosLSeekFile(qReader->pReadH, 0, SEEK_SET) < 0) { code = TAOS_SYSTEM_ERROR(errno); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - ASSERT(!(*ppBuf)); - // alloc - *ppBuf = taosMemoryCalloc(1, sizeof(SSnapDataHdr) + size); + if (*ppBuf) { + *ppBuf = taosMemoryRealloc(*ppBuf, sizeof(SSnapDataHdr) + size); + } else { + *ppBuf = taosMemoryMalloc(sizeof(SSnapDataHdr) + size); + } if (!(*ppBuf)) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } // read n = taosReadFile(qReader->pReadH, POINTER_SHIFT(*ppBuf, sizeof(SSnapDataHdr)), size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } else if (n != size) { code = TSDB_CODE_FILE_CORRUPTED; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - smaInfo("vgId:%d, vnode snapshot rsma read qtaskinfo, size:%" PRIi64, SMA_VID(pSma), size); + smaInfo("vgId:%d, vnode snapshot rsma read qtaskinfo, version:%" PRIi64 ", size:%" PRIi64, TD_VID(pVnode), version, + size); SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppBuf); pHdr->type = SNAP_DATA_QTASK; + pHdr->flag = qReader->level; + pHdr->index = qReader->suid; pHdr->size = size; _exit: - smaInfo("vgId:%d, vnode snapshot rsma read qtaskinfo succeed", SMA_VID(pSma)); - return code; + if (qReader) taosCloseFile(&qReader->pReadH); -_err: - *ppBuf = NULL; - smaError("vgId:%d, vnode snapshot rsma read qtaskinfo failed since %s", SMA_VID(pSma), tstrerror(code)); + if (code) { + *ppBuf = NULL; + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } else { + smaInfo("vgId:%d, vnode snapshot rsma read qtaskinfo succeed", TD_VID(pVnode)); + } return code; } int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; + int32_t lino = 0; *ppData = NULL; @@ -233,14 +212,11 @@ int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData) { if (!pReader->rsmaDataDone[i]) { smaInfo("vgId:%d, vnode snapshot rsma read level %d not done", SMA_VID(pReader->pSma), i); code = tsdbSnapRead(pTsdbSnapReader, ppData); - if (code) { - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); + if (*ppData) { + goto _exit; } else { - if (*ppData) { - goto _exit; - } else { - pReader->rsmaDataDone[i] = 1; - } + pReader->rsmaDataDone[i] = 1; } } else { smaInfo("vgId:%d, vnode snapshot rsma read level %d is done", SMA_VID(pReader->pSma), i); @@ -251,22 +227,21 @@ int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData) { if (!pReader->qTaskDone) { smaInfo("vgId:%d, vnode snapshot rsma qtaskinfo not done", SMA_VID(pReader->pSma)); code = rsmaSnapReadQTaskInfo(pReader, ppData); - if (code) { - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); + if (*ppData) { + goto _exit; } else { pReader->qTaskDone = 1; - if (*ppData) { - goto _exit; - } } } _exit: - smaInfo("vgId:%d, vnode snapshot rsma read succeed", SMA_VID(pReader->pSma)); - return code; - -_err: - smaError("vgId:%d, vnode snapshot rsma read failed since %s", SMA_VID(pReader->pSma), tstrerror(code)); + if (code) { + smaError("vgId:%d, vnode snapshot rsma read failed since %s", SMA_VID(pReader->pSma), tstrerror(code)); + rsmaSnapReaderClose(&pReader); + } else { + smaInfo("vgId:%d, vnode snapshot rsma read succeed", SMA_VID(pReader->pSma)); + } return code; } @@ -274,14 +249,15 @@ int32_t rsmaSnapReaderClose(SRSmaSnapReader** ppReader) { int32_t code = 0; SRSmaSnapReader* pReader = *ppReader; + tdRSmaFSUnRef(pReader->pSma, &pReader->fs); + taosMemoryFreeClear(pReader->pQTaskFReader); + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pReader->pDataReader[i]) { tsdbSnapReaderClose(&pReader->pDataReader[i]); } } - rsmaQTaskInfSnapReaderClose(&pReader->pQTaskFReader); - smaInfo("vgId:%d, vnode snapshot rsma reader closed", SMA_VID(pReader->pSma)); taosMemoryFreeClear(*ppReader); @@ -293,28 +269,23 @@ struct SRSmaSnapWriter { SSma* pSma; int64_t sver; int64_t ever; - - // config - int64_t commitID; + SRSmaFS fs; // for data file STsdbSnapWriter* pDataWriter[TSDB_RETENTION_L2]; - - // for qtaskinfo file - SQTaskFReader* pQTaskFReader; - SQTaskFWriter* pQTaskFWriter; }; int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWriter** ppWriter) { int32_t code = 0; - SRSmaSnapWriter* pWriter = NULL; + int32_t lino = 0; SVnode* pVnode = pSma->pVnode; + SRSmaSnapWriter* pWriter = NULL; // alloc pWriter = (SRSmaSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter)); - if (pWriter == NULL) { + if (!pWriter) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pWriter->pSma = pSma; pWriter->sver = sver; @@ -324,100 +295,152 @@ int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWrit for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pSma->pRSmaTsdb[i]) { code = tsdbSnapWriterOpen(pSma->pRSmaTsdb[i], sver, ever, &pWriter->pDataWriter[i]); - if (code < 0) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _exit); } } // qtaskinfo - SQTaskFWriter* qWriter = (SQTaskFWriter*)taosMemoryCalloc(1, sizeof(SQTaskFWriter)); - qWriter->pSma = pSma; - - char qTaskInfoFullName[TSDB_FILENAME_LEN]; - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), 0, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); - TdFilePtr qTaskF = taosCreateFile(qTaskInfoFullName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (!qTaskF) { - taosMemoryFree(qWriter); - code = TAOS_SYSTEM_ERROR(errno); - smaError("vgId:%d, rsma snapshot writer open %s failed since %s", TD_VID(pSma->pVnode), qTaskInfoFullName, - tstrerror(code)); - goto _err; - } - qWriter->pWriteH = qTaskF; - int32_t fnameLen = strlen(qTaskInfoFullName) + 1; - qWriter->fname = taosMemoryCalloc(1, fnameLen); - strncpy(qWriter->fname, qTaskInfoFullName, fnameLen); - pWriter->pQTaskFWriter = qWriter; - smaDebug("vgId:%d, rsma snapshot writer open succeed for %s", TD_VID(pSma->pVnode), qTaskInfoFullName); + code = tdRSmaFSCopy(pSma, &pWriter->fs); + TSDB_CHECK_CODE(code, lino, _exit); // snapWriter *ppWriter = pWriter; - - smaInfo("vgId:%d, rsma snapshot writer open succeed", TD_VID(pSma->pVnode)); +_exit: + if (code) { + if (pWriter) rsmaSnapWriterClose(&pWriter, 0); + *ppWriter = NULL; + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } else { + smaInfo("vgId:%d, rsma snapshot writer open succeed", TD_VID(pSma->pVnode)); + } return code; +} -_err: - smaError("vgId:%d, rsma snapshot writer open failed since %s", TD_VID(pSma->pVnode), tstrerror(code)); - if (pWriter) rsmaSnapWriterClose(&pWriter, 0); - *ppWriter = NULL; +int32_t rsmaSnapWriterPrepareClose(SRSmaSnapWriter* pWriter) { + int32_t code = 0; + int32_t lino = 0; + + if (pWriter) { + code = tdRSmaFSPrepareCommit(pWriter->pSma, &pWriter->fs); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pWriter->pSma), __func__, lino, tstrerror(code)); + } return code; } int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback) { int32_t code = 0; + int32_t lino = 0; + SSma* pSma = NULL; + SVnode* pVnode = NULL; + SSmaEnv* pEnv = NULL; + SRSmaStat* pStat = NULL; SRSmaSnapWriter* pWriter = *ppWriter; - SVnode* pVnode = pWriter->pSma->pVnode; + const char* primaryPath = NULL; + char fname[TSDB_FILENAME_LEN] = {0}; + char fnameVer[TSDB_FILENAME_LEN] = {0}; + TdFilePtr pOutFD = NULL; + TdFilePtr pInFD = NULL; + + if (!pWriter) { + goto _exit; + } - if (rollback) { - // TODO: rsma1/rsma2 - // qtaskinfo - if (pWriter->pQTaskFWriter) { - if (taosRemoveFile(pWriter->pQTaskFWriter->fname) != 0) { - smaWarn("vgId:%d, vnode snapshot rsma writer failed to remove %s since %s", SMA_VID(pWriter->pSma), - pWriter->pQTaskFWriter->fname ? pWriter->pQTaskFWriter->fname : "NULL", - tstrerror(TAOS_SYSTEM_ERROR(errno))); - } + pSma = pWriter->pSma; + pVnode = pSma->pVnode; + pEnv = SMA_RSMA_ENV(pSma); + pStat = (SRSmaStat*)SMA_ENV_STAT(pEnv); + primaryPath = tfsGetPrimaryPath(pVnode->pTfs); + + // rsma1/rsma2 + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { + if (pWriter->pDataWriter[i]) { + code = tsdbSnapWriterClose(&pWriter->pDataWriter[i], rollback); + TSDB_CHECK_CODE(code, lino, _exit); } + } + + // qtaskinfo + if (rollback) { + tdRSmaFSRollback(pSma); + // remove qTaskFiles } else { - // rsma1/rsma2 - for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { - if (pWriter->pDataWriter[i]) { - code = tsdbSnapWriterClose(&pWriter->pDataWriter[i], rollback); - if (code) goto _err; + // sendFile from fname.Ver to fname + SRSmaFS* pFS = &pWriter->fs; + int32_t size = taosArrayGetSize(pFS->aQTaskInf); + for (int32_t i = 0; i < size; ++i) { + SQTaskFile* pTaskF = TARRAY_GET_ELEM(pFS->aQTaskInf, i); + if (pTaskF->version == pWriter->ever) { + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, pTaskF->level, pTaskF->version, primaryPath, fnameVer); + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, pTaskF->level, -1, primaryPath, fname); + + pInFD = taosOpenFile(fnameVer, TD_FILE_READ); + if (pInFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + pOutFD = taosCreateFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (pOutFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + int64_t size = 0; + if (taosFStatFile(pInFD, &size, NULL) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + int64_t offset = 0; + if (taosFSendFile(pOutFD, pInFD, &offset, size) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + smaError("vgId:%d, vnode snapshot rsma writer, send qtaskinfo file %s to %s failed since %s", TD_VID(pVnode), + fnameVer, fname, tstrerror(code)); + TSDB_CHECK_CODE(code, lino, _exit); + } + taosCloseFile(&pOutFD); + taosCloseFile(&pInFD); } } - // qtaskinfo - if (pWriter->pQTaskFWriter) { - char qTaskInfoFullName[TSDB_FILENAME_LEN]; - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pWriter->ever, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); - if (taosRenameFile(pWriter->pQTaskFWriter->fname, qTaskInfoFullName) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - smaInfo("vgId:%d, vnode snapshot rsma writer rename %s to %s", SMA_VID(pWriter->pSma), - pWriter->pQTaskFWriter->fname, qTaskInfoFullName); - // rsma restore - if ((code = tdRSmaRestore(pWriter->pSma, RSMA_RESTORE_SYNC, pWriter->ever)) < 0) { - goto _err; - } - smaInfo("vgId:%d, vnode snapshot rsma writer restore from %s succeed", SMA_VID(pWriter->pSma), qTaskInfoFullName); + // lock + taosWLockLatch(RSMA_FS_LOCK(pStat)); + code = tdRSmaFSCommit(pSma); + if (code) { + taosWUnLockLatch(RSMA_FS_LOCK(pStat)); + goto _exit; } + // unlock + taosWUnLockLatch(RSMA_FS_LOCK(pStat)); } - smaInfo("vgId:%d, vnode snapshot rsma writer close succeed", SMA_VID(pWriter->pSma)); - taosMemoryFree(pWriter); + // rsma restore + code = tdRSmaRestore(pWriter->pSma, RSMA_RESTORE_SYNC, pWriter->ever, rollback); + TSDB_CHECK_CODE(code, lino, _exit); + smaInfo("vgId:%d, vnode snapshot rsma writer restore from sync succeed", SMA_VID(pSma)); + +_exit: + if (pWriter) taosMemoryFree(pWriter); *ppWriter = NULL; - return code; + if (code) { + if (pOutFD) taosCloseFile(&pOutFD); + if (pInFD) taosCloseFile(&pInFD); + smaError("vgId:%d, vnode snapshot rsma writer close failed since %s", SMA_VID(pSma), tstrerror(code)); + } else { + smaInfo("vgId:%d, vnode snapshot rsma writer close succeed", pSma ? SMA_VID(pSma) : 0); + } -_err: - smaError("vgId:%d, vnode snapshot rsma writer close failed since %s", SMA_VID(pWriter->pSma), tstrerror(code)); return code; } int32_t rsmaSnapWrite(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; + int32_t lino = 0; SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; // rsma1/rsma2 @@ -430,42 +453,81 @@ int32_t rsmaSnapWrite(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) } else if (pHdr->type == SNAP_DATA_QTASK) { code = rsmaSnapWriteQTaskInfo(pWriter, pData, nData); } else { - ASSERT(0); + code = TSDB_CODE_RSMA_FS_SYNC; } - if (code < 0) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); _exit: - smaInfo("vgId:%d, rsma snapshot write for data type %" PRIi8 " succeed", SMA_VID(pWriter->pSma), pHdr->type); - return code; - -_err: - smaError("vgId:%d, rsma snapshot write for data type %" PRIi8 " failed since %s", SMA_VID(pWriter->pSma), pHdr->type, - tstrerror(code)); + if (code) { + smaError("vgId:%d, %s failed at line %d since %s, data type %" PRIi8, SMA_VID(pWriter->pSma), __func__, lino, + tstrerror(code), pHdr->type); + } else { + smaInfo("vgId:%d, rsma snapshot write for data type %" PRIi8 " succeed", SMA_VID(pWriter->pSma), pHdr->type); + } return code; } static int32_t rsmaSnapWriteQTaskInfo(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; - SQTaskFWriter* qWriter = pWriter->pQTaskFWriter; - - if (qWriter && qWriter->pWriteH) { - SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; - int64_t size = pHdr->size; - ASSERT(size == (nData - sizeof(SSnapDataHdr))); - int64_t contLen = taosWriteFile(qWriter->pWriteH, pHdr->data, size); - if (contLen != size) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - smaInfo("vgId:%d, vnode snapshot rsma write qtaskinfo %s succeed", SMA_VID(pWriter->pSma), qWriter->fname); + int32_t code = 0; + int32_t lino = 0; + SSma* pSma = pWriter->pSma; + SVnode* pVnode = pSma->pVnode; + char fname[TSDB_FILENAME_LEN]; + TdFilePtr fp = NULL; + SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; + + fname[0] = '\0'; + + if (pHdr->size != (nData - sizeof(SSnapDataHdr))) { + code = TSDB_CODE_RSMA_FS_SYNC; + TSDB_CHECK_CODE(code, lino, _exit); + } + + SQTaskFile qTaskFile = { + .nRef = 1, .level = pHdr->flag, .suid = pHdr->index, .version = pWriter->ever, .size = pHdr->size}; + + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pHdr->index, pHdr->flag, qTaskFile.version, + tfsGetPrimaryPath(pVnode->pTfs), fname); + + fp = taosCreateFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (!fp) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + int64_t contLen = taosWriteFile(fp, pHdr->data, pHdr->size); + if (contLen != pHdr->size) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + uint32_t mtime = 0; + if (taosFStatFile(fp, NULL, &mtime) != 0) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); } else { - smaInfo("vgId:%d, vnode snapshot rsma write qtaskinfo is not needed", SMA_VID(pWriter->pSma)); + qTaskFile.mtime = mtime; } + if (taosFsyncFile(fp) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + taosCloseFile(&fp); + + code = tdRSmaFSUpsertQTaskFile(pSma, &pWriter->fs, &qTaskFile, 1); + TSDB_CHECK_CODE(code, lino, _exit); + _exit: - return code; + if (code) { + if (fp) { + (void)taosRemoveFile(fname); + } + smaError("vgId:%d, %s failed at line %d since %s, file:%s", TD_VID(pVnode), __func__, lino, tstrerror(code), fname); + } else { + smaInfo("vgId:%d, vnode snapshot rsma write qtaskinfo %s succeed", TD_VID(pVnode), fname); + } -_err: - smaError("vgId:%d, vnode snapshot rsma write qtaskinfo failed since %s", SMA_VID(pWriter->pSma), tstrerror(code)); return code; } diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index a2c9484693a720243ccb205f0d311289c86f254e..65c3bf3095e223f78166ebbd7c8b0d161d51e43d 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -14,6 +14,7 @@ */ #include "sma.h" +#include "tq.h" #include "tsdb.h" #define SMA_STORAGE_MINUTES_MAX 86400 @@ -111,35 +112,242 @@ _err: * @return int32_t */ static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { - SSmaCfg *pCfg = (SSmaCfg *)pMsg; + int32_t code = 0; + int32_t lino = 0; + SSmaCfg *pCfg = (SSmaCfg *)pMsg; + SName stbFullName = {0}; + SVCreateStbReq pReq = {0}; if (TD_VID(pSma->pVnode) == pCfg->dstVgId) { // create tsma meta in dstVgId if (metaCreateTSma(SMA_META(pSma), version, pCfg) < 0) { - return -1; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } // create stable to save tsma result in dstVgId - SName stbFullName = {0}; tNameFromString(&stbFullName, pCfg->dstTbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - SVCreateStbReq pReq = {0}; pReq.name = (char *)tNameGetTableName(&stbFullName); pReq.suid = pCfg->dstTbUid; pReq.schemaRow = pCfg->schemaRow; pReq.schemaTag = pCfg->schemaTag; if (metaCreateSTable(SMA_META(pSma), version, &pReq) < 0) { - return -1; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } + } else { + code = terrno = TSDB_CODE_TSMA_INVALID_STAT; + TSDB_CHECK_CODE(code, lino, _exit); + } +_exit: + if (code) { + smaError("vgId:%d, failed at line %d to create sma index %s %" PRIi64 " on stb:%" PRIi64 ", dstSuid:%" PRIi64 + " dstTb:%s dstVg:%d", + SMA_VID(pSma), lino, pCfg->indexName, pCfg->indexUid, pCfg->tableUid, pCfg->dstTbUid, pReq.name, + pCfg->dstVgId); + } else { smaDebug("vgId:%d, success to create sma index %s %" PRIi64 " on stb:%" PRIi64 ", dstSuid:%" PRIi64 " dstTb:%s dstVg:%d", SMA_VID(pSma), pCfg->indexName, pCfg->indexUid, pCfg->tableUid, pCfg->dstTbUid, pReq.name, pCfg->dstVgId); - } else { - ASSERT(0); } - return 0; + return code; +} + +int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *pTSchema, + SSchemaWrapper *pTagSchemaWrapper, bool createTb, int64_t suid, const char *stbFullName, + SBatchDeleteReq *pDeleteReq, void **ppData, int32_t *pLen) { + void *pBuf = NULL; + int32_t len = 0; + SSubmitReq2 *pReq = NULL; + SArray *tagArray = NULL; + SArray *createTbArray = NULL; + SArray *pVals = NULL; + + int32_t sz = taosArrayGetSize(pBlocks); + + if (!(tagArray = taosArrayInit(1, sizeof(STagVal)))) { + goto _end; + } + + if (!(createTbArray = taosArrayInit(sz, POINTER_BYTES))) { + goto _end; + } + + if (!(pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { + goto _end; + } + + // create table req + if (createTb) { + for (int32_t i = 0; i < sz; ++i) { + SSDataBlock *pDataBlock = taosArrayGet(pBlocks, i); + SVCreateTbReq *pCreateTbReq = NULL; + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + taosArrayPush(createTbArray, &pCreateTbReq); + continue; + } + + if (!(pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq)))) { + goto _end; + }; + + // don't move to the end of loop as to destroy in the end of func when error occur + taosArrayPush(createTbArray, &pCreateTbReq); + + // set const + pCreateTbReq->flags = 0; + pCreateTbReq->type = TSDB_CHILD_TABLE; + pCreateTbReq->ctb.suid = suid; + + // set super table name + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + pCreateTbReq->ctb.stbName = strdup((char *)tNameGetTableName(&name)); // strdup(stbFullName); + + // set tag content + taosArrayClear(tagArray); + STagVal tagVal = { + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.id.groupId, + }; + taosArrayPush(tagArray, &tagVal); + pCreateTbReq->ctb.tagNum = taosArrayGetSize(tagArray); + + STag *pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + if (pTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + pCreateTbReq->ctb.pTag = (uint8_t *)pTag; + + // set tag name + SArray *tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = {0}; + strcpy(tagNameStr, "group_id"); + taosArrayPush(tagName, tagNameStr); + pCreateTbReq->ctb.tagName = tagName; + + // set table name + if (pDataBlock->info.parTbName[0]) { + pCreateTbReq->name = strdup(pDataBlock->info.parTbName); + } else { + pCreateTbReq->name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId); + } + } + } + + // SSubmitTbData req + for (int32_t i = 0; i < sz; ++i) { + SSDataBlock *pDataBlock = taosArrayGet(pBlocks, i); + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + pDeleteReq->suid = suid; + pDeleteReq->deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); + tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, pDeleteReq); + continue; + } + + int32_t rows = pDataBlock->info.rows; + + SSubmitTbData *pTbData = (SSubmitTbData *)taosMemoryCalloc(1, sizeof(SSubmitTbData)); + if (!pTbData) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + if (!(pTbData->aRowP = taosArrayInit(rows, sizeof(SRow *)))) { + taosMemoryFree(pTbData); + goto _end; + } + pTbData->suid = suid; + pTbData->uid = 0; // uid is assigned by vnode + pTbData->sver = pTSchema->version; + + if (createTb) { + pTbData->pCreateTbReq = taosArrayGetP(createTbArray, i); + if (pTbData->pCreateTbReq) pTbData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; + } + + if (!pVals && !(pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)))) { + taosArrayDestroy(pTbData->aRowP); + taosMemoryFree(pTbData); + goto _end; + } + + for (int32_t j = 0; j < rows; ++j) { + taosArrayClear(pVals); + for (int32_t k = 0; k < pTSchema->numOfCols; k++) { + const STColumn *pCol = &pTSchema->columns[k]; + SColumnInfoData *pColData = taosArrayGet(pDataBlock->pDataBlock, k); + if (colDataIsNull_s(pColData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + void *data = colDataGetData(pColData, j); + if (IS_STR_DATA_TYPE(pCol->type)) { + SValue sv = (SValue){.nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } else { + SValue sv; + memcpy(&sv.val, data, tDataTypes[pCol->type].bytes); + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } + } + } + SRow *pRow = NULL; + if ((terrno = tRowBuild(pVals, (STSchema *)pTSchema, &pRow)) < 0) { + tDestroySSubmitTbData(pTbData, TSDB_MSG_FLG_ENCODE); + goto _end; + } + ASSERT(pRow); + taosArrayPush(pTbData->aRowP, &pRow); + } + + taosArrayPush(pReq->aSubmitTbData, pTbData); + } + + // encode + tEncodeSize(tEncodeSSubmitReq2, pReq, len, terrno); + if (TSDB_CODE_SUCCESS == terrno) { + SEncoder encoder; + len += sizeof(SMsgHead); + pBuf = rpcMallocCont(len); + if (NULL == pBuf) { + goto _end; + } + ((SMsgHead *)pBuf)->vgId = TD_VID(pVnode); + ((SMsgHead *)pBuf)->contLen = htonl(len); + tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SMsgHead)), len - sizeof(SMsgHead)); + if (tEncodeSSubmitReq2(&encoder, pReq) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + /*vError("failed to encode submit req since %s", terrstr());*/ + } + tEncoderClear(&encoder); + } +_end: + taosArrayDestroy(tagArray); + taosArrayDestroy(pVals); + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + + if (terrno != 0) { + rpcFreeCont(pBuf); + taosArrayDestroy(pDeleteReq->deleteReqs); + return TSDB_CODE_FAILED; + } + if (ppData) *ppData = pBuf; + if (pLen) *pLen = len; + return TSDB_CODE_SUCCESS; } /** @@ -174,7 +382,7 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char STSmaStat *pTsmaStat = NULL; if (!pEnv || !(pStat = SMA_ENV_STAT(pEnv))) { - terrno = TSDB_CODE_TSMA_INVALID_STAT; + terrno = TSDB_CODE_TSMA_INVALID_ENV; return TSDB_CODE_FAILED; } @@ -204,27 +412,32 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char } SBatchDeleteReq deleteReq = {0}; - SSubmitReq *pSubmitReq = - tqBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true, - pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq); - // TODO deleteReq - taosArrayDestroy(deleteReq.deleteReqs); - + void *pSubmitReq = NULL; + int32_t contLen = 0; - if (!pSubmitReq) { - smaError("vgId:%d, failed to gen submit blk while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), + if (smaBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true, + pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq, &pSubmitReq, + &contLen) < 0) { + smaError("vgId:%d, failed to gen submit msg while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid, tstrerror(terrno)); goto _err; } + // TODO deleteReq + taosArrayDestroy(deleteReq.deleteReqs); #if 0 - ASSERT(!strncasecmp("td.tsma.rst.tb", pTsmaStat->pTSma->dstTbName, 14)); + if (!strncasecmp("td.tsma.rst.tb", pTsmaStat->pTSma->dstTbName, 14)) { + terrno = TSDB_CODE_APP_ERROR; + smaError("vgId:%d, tsma insert for smaIndex %" PRIi64 " failed since %s, %s", SMA_VID(pSma), indexUid, + pTsmaStat->pTSma->indexUid, tstrerror(terrno), pTsmaStat->pTSma->dstTbName); + goto _err; + } #endif SRpcMsg submitReqMsg = { .msgType = TDMT_VND_SUBMIT, .pCont = pSubmitReq, - .contLen = ntohl(pSubmitReq->length), + .contLen = contLen, }; if (tmsgPutToQueue(&pSma->pVnode->msgCb, WRITE_QUEUE, &submitReqMsg) < 0) { diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 4d09d690d684c68199f63165875216ea3685acad..7c538280e52ed127ff2815623a2afa00c126de7f 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -15,197 +15,72 @@ #include "sma.h" -// smaFileUtil ================ -#if 0 -#define TD_FILE_STATE_OK 0 -#define TD_FILE_STATE_BAD 1 +#define TD_QTASKINFO_FNAME_PREFIX "main.tdb" -#define TD_FILE_INIT_MAGIC 0xFFFFFFFF - -static int32_t tdEncodeTFInfo(void **buf, STFInfo *pInfo); -static void *tdDecodeTFInfo(void *buf, STFInfo *pInfo); - -static int32_t tdEncodeTFInfo(void **buf, STFInfo *pInfo) { - int32_t tlen = 0; - - tlen += taosEncodeFixedU32(buf, pInfo->magic); - tlen += taosEncodeFixedU32(buf, pInfo->ftype); - tlen += taosEncodeFixedU32(buf, pInfo->fver); - tlen += taosEncodeFixedI64(buf, pInfo->fsize); - - return tlen; -} - -static void *tdDecodeTFInfo(void *buf, STFInfo *pInfo) { - buf = taosDecodeFixedU32(buf, &(pInfo->magic)); - buf = taosDecodeFixedU32(buf, &(pInfo->ftype)); - buf = taosDecodeFixedU32(buf, &(pInfo->fver)); - buf = taosDecodeFixedI64(buf, &(pInfo->fsize)); - - return buf; -} - -int64_t tdWriteTFile(STFile *pTFile, void *buf, int64_t nbyte) { - ASSERT(TD_TFILE_OPENED(pTFile)); - - int64_t nwrite = taosWriteFile(pTFile->pFile, buf, nbyte); - if (nwrite < nbyte) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return nwrite; -} - -int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence) { - ASSERT(TD_TFILE_OPENED(pTFile)); - - int64_t loffset = taosLSeekFile(TD_TFILE_PFILE(pTFile), offset, whence); - if (loffset < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return loffset; -} - -int64_t tdGetTFileSize(STFile *pTFile, int64_t *size) { - ASSERT(TD_TFILE_OPENED(pTFile)); - return taosFStatFile(pTFile->pFile, size, NULL); -} - -int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte) { - ASSERT(TD_TFILE_OPENED(pTFile)); - - int64_t nread = taosReadFile(pTFile->pFile, buf, nbyte); - if (nread < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return nread; -} - -int32_t tdUpdateTFileHeader(STFile *pTFile) { - char buf[TD_FILE_HEAD_SIZE] = "\0"; - - if (tdSeekTFile(pTFile, 0, SEEK_SET) < 0) { - return -1; - } - - void *ptr = buf; - tdEncodeTFInfo(&ptr, &(pTFile->info)); - - taosCalcChecksumAppend(0, (uint8_t *)buf, TD_FILE_HEAD_SIZE); - if (tdWriteTFile(pTFile, buf, TD_FILE_HEAD_SIZE) < 0) { - return -1; - } - - return 0; -} - -int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo) { - char buf[TD_FILE_HEAD_SIZE] = "\0"; - uint32_t _version; - - ASSERT(TD_TFILE_OPENED(pTFile)); - - if (tdSeekTFile(pTFile, 0, SEEK_SET) < 0) { - return -1; - } - - if (tdReadTFile(pTFile, buf, TD_FILE_HEAD_SIZE) < 0) { - return -1; - } - - if (!taosCheckChecksumWhole((uint8_t *)buf, TD_FILE_HEAD_SIZE)) { - terrno = TSDB_CODE_FILE_CORRUPTED; - return -1; - } - - void *pBuf = buf; - pBuf = tdDecodeTFInfo(pBuf, pInfo); - return 0; -} - -void tdUpdateTFileMagic(STFile *pTFile, void *pCksm) { - pTFile->info.magic = taosCalcChecksum(pTFile->info.magic, (uint8_t *)(pCksm), sizeof(TSCKSUM)); +void tdRSmaQTaskInfoGetFileName(int32_t vgId, int64_t suid, int8_t level, int64_t version, char *outputName) { + tdRSmaGetFileName(vgId, NULL, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, suid, level, version, outputName); } -int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset) { - ASSERT(TD_TFILE_OPENED(pTFile)); - - int64_t toffset; - - if ((toffset = tdSeekTFile(pTFile, 0, SEEK_END)) < 0) { - return -1; - } - -#if 0 - smaDebug("append to file %s, offset:%" PRIi64 " nbyte:%" PRIi64 " fsize:%" PRIi64, TD_TFILE_FULL_NAME(pTFile), - toffset, nbyte, toffset + nbyte); -#endif - - ASSERT(pTFile->info.fsize == toffset); - - if (offset) { - *offset = toffset; - } - - if (tdWriteTFile(pTFile, buf, nbyte) < 0) { - return -1; - } - - pTFile->info.fsize += nbyte; - - return nbyte; +void tdRSmaQTaskInfoGetFullName(int32_t vgId, int64_t suid, int8_t level, int64_t version, const char *path, + char *outputName) { + tdRSmaGetFileName(vgId, path, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, suid, level, version, outputName); } -int32_t tdOpenTFile(STFile *pTFile, int flags) { - ASSERT(!TD_TFILE_OPENED(pTFile)); - - pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), flags); - if (pTFile->pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return 0; +void tdRSmaQTaskInfoGetFullPath(int32_t vgId, int8_t level, const char *path, char *outputName) { + tdRSmaGetDirName(vgId, path, VNODE_RSMA_DIR, true, outputName); + int32_t rsmaLen = strlen(outputName); + snprintf(outputName + rsmaLen, TSDB_FILENAME_LEN - rsmaLen, "%" PRIi8, level); } -void tdCloseTFile(STFile *pTFile) { - if (TD_TFILE_OPENED(pTFile)) { - taosCloseFile(&pTFile->pFile); - TD_TFILE_SET_CLOSED(pTFile); - } +void tdRSmaQTaskInfoGetFullPathEx(int32_t vgId, tb_uid_t suid, int8_t level, const char *path, char *outputName) { + tdRSmaGetDirName(vgId, path, VNODE_RSMA_DIR, true, outputName); + int32_t rsmaLen = strlen(outputName); + snprintf(outputName + rsmaLen, TSDB_FILENAME_LEN - rsmaLen, "%" PRIi8 "%s%" PRIi64, level, TD_DIRSEP, suid); } -void tdDestroyTFile(STFile *pTFile) { taosMemoryFreeClear(TD_TFILE_FULL_NAME(pTFile)); } - -#endif - -void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version, - char *outputName) { - if (version < 0) { - if (pdname) { - snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%sv%d%s", pdname, TD_DIRSEP, TD_DIRSEP, vgId, - TD_DIRSEP, dname, TD_DIRSEP, vgId, fname); +void tdRSmaGetFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t suid, + int8_t level, int64_t version, char *outputName) { + if (level >= 0 && suid > 0) { + if (version >= 0) { + if (pdname) { + snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%s%" PRIi8 "%s%" PRIi64 "%s%s.%" PRIi64, pdname, + TD_DIRSEP, TD_DIRSEP, vgId, TD_DIRSEP, dname, TD_DIRSEP, level, TD_DIRSEP, suid, TD_DIRSEP, fname, + version); + } else { + snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%s%" PRIi8 "%s%" PRIi64 "%s%s.%" PRIi64, TD_DIRSEP, + vgId, TD_DIRSEP, dname, TD_DIRSEP, level, TD_DIRSEP, suid, TD_DIRSEP, fname, version); + } } else { - snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%sv%d%s", TD_DIRSEP, vgId, TD_DIRSEP, dname, TD_DIRSEP, - vgId, fname); + if (pdname) { + snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%s%" PRIi8 "%s%" PRIi64 "%s%s", pdname, + TD_DIRSEP, TD_DIRSEP, vgId, TD_DIRSEP, dname, TD_DIRSEP, level, TD_DIRSEP, suid, TD_DIRSEP, fname); + } else { + snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%s%" PRIi8 "%s%" PRIi64 "%s%s", TD_DIRSEP, vgId, + TD_DIRSEP, dname, TD_DIRSEP, level, TD_DIRSEP, suid, TD_DIRSEP, fname); + } } } else { - if (pdname) { - snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%sv%d%s%" PRIi64, pdname, TD_DIRSEP, TD_DIRSEP, - vgId, TD_DIRSEP, dname, TD_DIRSEP, vgId, fname, version); + if (version >= 0) { + if (pdname) { + snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%sv%d%s%" PRIi64, pdname, TD_DIRSEP, TD_DIRSEP, + vgId, TD_DIRSEP, dname, TD_DIRSEP, vgId, fname, version); + } else { + snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%sv%d%s%" PRIi64, TD_DIRSEP, vgId, TD_DIRSEP, dname, + TD_DIRSEP, vgId, fname, version); + } } else { - snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%sv%d%s%" PRIi64, TD_DIRSEP, vgId, TD_DIRSEP, dname, - TD_DIRSEP, vgId, fname, version); + if (pdname) { + snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%sv%d%s", pdname, TD_DIRSEP, TD_DIRSEP, vgId, + TD_DIRSEP, dname, TD_DIRSEP, vgId, fname); + } else { + snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%sv%d%s", TD_DIRSEP, vgId, TD_DIRSEP, dname, + TD_DIRSEP, vgId, fname); + } } } } -void tdGetVndDirName(int32_t vgId, const char *pdname, const char *dname, bool endWithSep, char *outputName) { +void tdRSmaGetDirName(int32_t vgId, const char *pdname, const char *dname, bool endWithSep, char *outputName) { if (pdname) { if (endWithSep) { snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%s", pdname, TD_DIRSEP, TD_DIRSEP, vgId, TD_DIRSEP, @@ -223,81 +98,13 @@ void tdGetVndDirName(int32_t vgId, const char *pdname, const char *dname, bool e } } -#if 0 -int32_t tdInitTFile(STFile *pTFile, const char *dname, const char *fname) { - TD_TFILE_SET_STATE(pTFile, TD_FILE_STATE_OK); - TD_TFILE_SET_CLOSED(pTFile); - - memset(&(pTFile->info), 0, sizeof(pTFile->info)); - pTFile->info.magic = TD_FILE_INIT_MAGIC; - - char tmpName[TSDB_FILENAME_LEN * 2 + 32] = {0}; - snprintf(tmpName, TSDB_FILENAME_LEN * 2 + 32, "%s%s%s", dname, TD_DIRSEP, fname); - int32_t tmpNameLen = strlen(tmpName) + 1; - pTFile->fname = taosMemoryMalloc(tmpNameLen); - if (!pTFile->fname) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - tstrncpy(pTFile->fname, tmpName, tmpNameLen); - - return 0; -} - -int32_t tdCreateTFile(STFile *pTFile, bool updateHeader, int8_t fType) { - ASSERT(pTFile->info.fsize == 0 && pTFile->info.magic == TD_FILE_INIT_MAGIC); - pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pTFile->pFile == NULL) { - if (errno == ENOENT) { - // Try to create directory recursively - char *s = strdup(TD_TFILE_FULL_NAME(pTFile)); - if (taosMulMkDir(taosDirName(s)) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - taosMemoryFree(s); - return -1; - } - taosMemoryFree(s); - pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pTFile->pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - } - } - - if (!updateHeader) { - return 0; - } - - pTFile->info.fsize += TD_FILE_HEAD_SIZE; - pTFile->info.fver = 0; - - if (tdUpdateTFileHeader(pTFile) < 0) { - tdCloseTFile(pTFile); - tdRemoveTFile(pTFile); - return -1; - } - - return 0; -} - -int32_t tdRemoveTFile(STFile *pTFile) { - if (taosRemoveFile(TD_TFILE_FULL_NAME(pTFile)) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - }; - return 0; -} - -#endif - // smaXXXUtil ================ void *tdAcquireSmaRef(int32_t rsetId, int64_t refId) { void *pResult = taosAcquireRef(rsetId, refId); if (!pResult) { smaWarn("rsma acquire ref for rsetId:%d refId:%" PRIi64 " failed since %s", rsetId, refId, terrstr()); } else { - smaDebug("rsma acquire ref for rsetId:%d refId:%" PRIi64 " success", rsetId, refId); + smaTrace("rsma acquire ref for rsetId:%d refId:%" PRIi64 " success", rsetId, refId); } return pResult; } @@ -307,7 +114,7 @@ int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId) { smaWarn("rsma release ref for rsetId:%d refId:%" PRIi64 " failed since %s", rsetId, refId, terrstr()); return TSDB_CODE_FAILED; } - smaDebug("rsma release ref for rsetId:%d refId:%" PRIi64 " success", rsetId, refId); + smaTrace("rsma release ref for rsetId:%d refId:%" PRIi64 " success", rsetId, refId); return TSDB_CODE_SUCCESS; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 1d5fae33eb8049633c8c4b568de98da68ead3d68..a27b6988c58d9173ca4250440a7abbac98f71114 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -93,21 +93,21 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { taosHashSetFreeFp(pTq->pCheckInfo, (FDelete)tDeleteSTqCheckInfo); if (tqMetaOpen(pTq) < 0) { - ASSERT(0); + return NULL; } pTq->pOffsetStore = tqOffsetOpen(pTq); if (pTq->pOffsetStore == NULL) { - ASSERT(0); + return NULL; } pTq->pStreamMeta = streamMetaOpen(path, pTq, (FTaskExpand*)tqExpandTask, pTq->pVnode->config.vgId); if (pTq->pStreamMeta == NULL) { - ASSERT(0); + return NULL; } if (streamLoadTasks(pTq->pStreamMeta) < 0) { - ASSERT(0); + return NULL; } return pTq; @@ -167,19 +167,17 @@ int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { SMqDataRsp* pRsp = &pPushEntry->dataRsp; - ASSERT(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); - ASSERT(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); +#if 0 + A(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); + A(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); - ASSERT(!pRsp->withSchema); - ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0); + A(!pRsp->withSchema); + A(taosArrayGetSize(pRsp->blockSchema) == 0); if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) { - /*if (pRsp->blockNum > 0) {*/ - /*ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version);*/ - /*} else {*/ - ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version); - /*}*/ + A(pRsp->rspOffset.version > pRsp->reqOffset.version); } +#endif int32_t len = 0; int32_t code = 0; @@ -224,19 +222,21 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { } int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp) { - ASSERT(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); - ASSERT(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); +#if 0 + A(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); + A(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); - ASSERT(!pRsp->withSchema); - ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0); + A(!pRsp->withSchema); + A(taosArrayGetSize(pRsp->blockSchema) == 0); if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) { if (pRsp->blockNum > 0) { - ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version); + A(pRsp->rspOffset.version > pRsp->reqOffset.version); } else { - ASSERT(pRsp->rspOffset.version >= pRsp->reqOffset.version); + A(pRsp->rspOffset.version >= pRsp->reqOffset.version); } } +#endif int32_t len = 0; int32_t code = 0; @@ -280,22 +280,24 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con } int32_t tqSendTaosxRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const STaosxRsp* pRsp) { - ASSERT(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); - ASSERT(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); +#if 0 + A(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); + A(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); if (pRsp->withSchema) { - ASSERT(taosArrayGetSize(pRsp->blockSchema) == pRsp->blockNum); + A(taosArrayGetSize(pRsp->blockSchema) == pRsp->blockNum); } else { - ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0); + A(taosArrayGetSize(pRsp->blockSchema) == 0); } if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) { if (pRsp->blockNum > 0) { - ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version); + A(pRsp->rspOffset.version > pRsp->reqOffset.version); } else { - ASSERT(pRsp->rspOffset.version >= pRsp->reqOffset.version); + A(pRsp->rspOffset.version >= pRsp->reqOffset.version); } } +#endif int32_t len = 0; int32_t code = 0; @@ -349,7 +351,6 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t version, char* msg, int32_t m SDecoder decoder; tDecoderInit(&decoder, msg, msgLen); if (tDecodeSTqOffset(&decoder, &offset) < 0) { - ASSERT(0); return -1; } tDecoderClear(&decoder); @@ -363,8 +364,8 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t version, char* msg, int32_t m if (offset.val.version + 1 == version) { offset.val.version += 1; } - } else { - ASSERT(0); + /*} else {*/ + /*A(0);*/ } STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, offset.subKey); if (pOffset != NULL && tqOffsetLessOrEqual(&offset, pOffset)) { @@ -372,7 +373,6 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t version, char* msg, int32_t m } if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) { - ASSERT(0); return -1; } @@ -435,7 +435,7 @@ static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t su } #endif - ASSERT(subType == TOPIC_SUB_TYPE__COLUMN); + /*A(subType == TOPIC_SUB_TYPE__COLUMN);*/ pRsp->withSchema = false; return 0; @@ -474,7 +474,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { // 1.find handle STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); - /*ASSERT(pHandle);*/ if (pHandle == NULL) { tqError("tmq poll: no consumer handle for consumer:%" PRId64 ", in vgId:%d, subkey %s", consumerId, TD_VID(pTq->pVnode), req.subKey); @@ -561,7 +560,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { tqInitDataRsp(&dataRsp, &req, pHandle->execHandle.subType); // lock taosWLockLatch(&pTq->pushLock); - tqScanData(pTq, pHandle, &dataRsp, &fetchOffsetNew); + if (tqScanData(pTq, pHandle, &dataRsp, &fetchOffsetNew) < 0) { + return -1; + } #if 1 if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && @@ -600,7 +601,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } // for taosx - ASSERT(pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN); + /*A(pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN);*/ SMqMetaRsp metaRsp = {0}; @@ -608,7 +609,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { tqInitTaosxRsp(&taosxRsp, &req); if (fetchOffsetNew.type != TMQ_OFFSET__LOG) { - tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, &fetchOffsetNew); + if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, &fetchOffsetNew) < 0) { + return -1; + } if (metaRsp.metaRspLen > 0) { if (tqSendMetaPollRsp(pTq, pMsg, &req, &metaRsp) < 0) { @@ -674,9 +677,12 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { req.epoch, TD_VID(pTq->pVnode), fetchVer, pHead->msgType); if (pHead->msgType == TDMT_VND_SUBMIT) { - SSubmitReq* pCont = (SSubmitReq*)&pHead->body; - - if (tqTaosxScanLog(pTq, pHandle, pCont, &taosxRsp) < 0) { + SPackedData submit = { + .msgStr = POINTER_SHIFT(pHead->body, sizeof(SMsgHead)), + .msgLen = pHead->bodyLen - sizeof(SMsgHead), + .ver = pHead->version, + }; + if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp) < 0) { } if (taosxRsp.blockNum > 0 /* threshold */) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); @@ -691,8 +697,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } } else { - ASSERT(pHandle->fetchMeta); - ASSERT(IS_META_MSG(pHead->msgType)); + /*A(pHandle->fetchMeta);*/ + /*A(IS_META_MSG(pHead->msgType));*/ tqDebug("fetch meta msg, ver:%" PRId64 ", type:%d", pHead->version, pHead->msgType); tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer); metaRsp.resMsgType = pHead->msgType; @@ -719,6 +725,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) { SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg; + tqDebug("vgId:%d, tq process delete sub req %s", pTq->pVnode->config.vgId, pReq->subKey); + taosWLockLatch(&pTq->pushLock); int32_t code = taosHashRemove(pTq->pPushMgr, pReq->subKey, strlen(pReq->subKey)); if (code != 0) { @@ -728,6 +736,7 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t version, char* msg, int32_t msgL STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey)); if (pHandle) { + // walCloseRef(pHandle->pWalReader->pWal, pHandle->pRef->refId); if (pHandle->pRef) { walCloseRef(pTq->pVnode->pWal, pHandle->pRef->refId); } @@ -784,6 +793,9 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t version, char* msg, int32_t msgL SMqRebVgReq req = {0}; tDecodeSMqRebVgReq(msg, &req); // todo lock + + tqDebug("vgId:%d, tq process sub req %s", pTq->pVnode->config.vgId, req.subKey); + STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); if (pHandle == NULL) { if (req.oldConsumerId != -1) { @@ -809,7 +821,6 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t version, char* msg, int32_t msgL // TODO version should be assigned and refed during preprocess SWalRef* pRef = walRefCommittedVer(pTq->pVnode->pWal); if (pRef == NULL) { - ASSERT(0); return -1; } int64_t ver = pRef->refVer; @@ -830,12 +841,12 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t version, char* msg, int32_t msgL pHandle->execHandle.task = qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle, &pHandle->execHandle.numOfCols, NULL); - ASSERT(pHandle->execHandle.task); + /*A(pHandle->execHandle.task);*/ void* scanner = NULL; qExtractStreamScanner(pHandle->execHandle.task, &scanner); - ASSERT(scanner); + /*A(scanner);*/ pHandle->execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner); - ASSERT(pHandle->execHandle.pExecReader); + /*A(pHandle->execHandle.pExecReader);*/ } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) { pHandle->pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); pHandle->execHandle.pExecReader = tqOpenReader(pTq->pVnode); @@ -868,19 +879,17 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t version, char* msg, int32_t msgL taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle)); tqDebug("try to persist handle %s consumer %" PRId64, req.subKey, pHandle->consumerId); if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) { - // TODO - ASSERT(0); } } else { - /*ASSERT(pExec->consumerId == req.oldConsumerId);*/ // TODO handle qmsg and exec modification atomic_store_32(&pHandle->epoch, -1); atomic_store_64(&pHandle->consumerId, req.newConsumerId); atomic_add_fetch_32(&pHandle->epoch, 1); taosMemoryFree(req.qmsg); + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + qStreamCloseTsdbReader(pHandle->execHandle.task); + } if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) { - // TODO - ASSERT(0); } // close handle } @@ -889,9 +898,11 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t version, char* msg, int32_t msgL } int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { +#if 0 if (pTask->taskLevel == TASK_LEVEL__AGG) { - ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0); + A(taosArrayGetSize(pTask->childEpInfo) != 0); } +#endif pTask->refCnt = 1; pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE; @@ -928,7 +939,9 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { .pStateBackend = pTask->pState, }; pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); - ASSERT(pTask->exec.executor); + if (pTask->exec.executor == NULL) { + return -1; + } } else if (pTask->taskLevel == TASK_LEVEL__AGG) { pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1); @@ -941,7 +954,9 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { .pStateBackend = pTask->pState, }; pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle); - ASSERT(pTask->exec.executor); + if (pTask->exec.executor == NULL) { + return -1; + } } // sink @@ -951,13 +966,13 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { pTask->smaSink.smaSink = smaHandleRes; } else if (pTask->outputType == TASK_OUTPUT__TABLE) { pTask->tbSink.vnode = pTq->pVnode; - pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline; + pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline2; - ASSERT(pTask->tbSink.pSchemaWrapper); - ASSERT(pTask->tbSink.pSchemaWrapper->pSchema); + /*A(pTask->tbSink.pSchemaWrapper);*/ + /*A(pTask->tbSink.pSchemaWrapper->pSchema);*/ pTask->tbSink.pTSchema = - tdGetSTSChemaFromSSChema(pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols, 1); + tBuildTSchema(pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols, 1); ASSERT(pTask->tbSink.pTSchema); } @@ -1004,7 +1019,7 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { int32_t len; tEncodeSize(tEncodeSStreamTaskCheckRsp, &rsp, len, code); if (code < 0) { - tqDebug("tq encode stream check rsp error"); + tqError("unable to encode rsp %d", __LINE__); return -1; } @@ -1099,12 +1114,10 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { if (pTask == NULL) { return -1; } - ASSERT(pReq->taskId == pTask->taskId); // check param int64_t fillVer1 = pTask->startVer; if (fillVer1 <= 0) { - ASSERT(0); streamMetaReleaseTask(pTq->pStreamMeta, pTask); return -1; } @@ -1313,7 +1326,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { } int32_t ref = atomic_sub_fetch_32(pRef, 1); - ASSERT(ref >= 0); + /*A(ref >= 0);*/ if (ref == 0) { blockDataDestroy(pDelBlock); taosMemoryFree(pRef); @@ -1348,12 +1361,12 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { return 0; } -int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* pReq, int64_t ver) { - void* pIter = NULL; - bool failed = false; - SStreamDataSubmit* pSubmit = NULL; +int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { + void* pIter = NULL; + bool failed = false; + SStreamDataSubmit2* pSubmit = NULL; - pSubmit = streamDataSubmitNew(pReq); + pSubmit = streamDataSubmitNew(submit); if (pSubmit == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to create data submit for stream since out of memory"); @@ -1370,7 +1383,7 @@ int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* pReq, int64_t ver) { continue; } - tqDebug("data submit enqueue stream task: %d, ver: %" PRId64, pTask->taskId, ver); + tqDebug("data submit enqueue stream task: %d, ver: %" PRId64, pTask->taskId, submit.ver); if (!failed) { if (streamTaskInput(pTask, (SStreamQueueItem*)pSubmit) < 0) { diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 093186ebbb188841bf0fa012be4dd59f8d0b5a1b..40a82cc8e82edbab0a776f0f5f69d6013b5e2de3 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -29,7 +29,6 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); actualLen += sizeof(SRetrieveTableRsp); - ASSERT(actualLen <= dataStrLen); taosArrayPush(pRsp->blockDataLen, &actualLen); taosArrayPush(pRsp->blockData, &buf); return 0; @@ -62,7 +61,6 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, i int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset) { const STqExecHandle* pExec = &pHandle->execHandle; - ASSERT(pExec->subType == TOPIC_SUB_TYPE__COLUMN); qTaskInfo_t task = pExec->task; @@ -87,7 +85,8 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs uint64_t ts = 0; tqDebug("vgId:%d, tmq task start to execute", pTq->pVnode->config.vgId); if (qExecTask(task, &pDataBlock, &ts) < 0) { - ASSERT(0); + tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, terrstr()); + return -1; } tqDebug("vgId:%d, tmq task executed, get %p", pTq->pVnode->config.vgId, pDataBlock); @@ -105,19 +104,16 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs } if (qStreamExtractOffset(task, &pRsp->rspOffset) < 0) { - ASSERT(0); return -1; } - ASSERT(pRsp->rspOffset.type != 0); - if (pRsp->withTbName) { - if (pRsp->rspOffset.type == TMQ_OFFSET__LOG) { - int64_t uid = pExec->pExecReader->msgIter.uid; - tqAddTbNameToRsp(pTq, uid, pRsp, 1); - } else { - pRsp->withTbName = false; - } + if (pRsp->rspOffset.type == 0) { + tqError("expected rsp offset: type %d %" PRId64 " %" PRId64 " %" PRId64, pRsp->rspOffset.type, pRsp->rspOffset.ts, + pRsp->rspOffset.uid, pRsp->rspOffset.version); + return -1; } + + ASSERT(pRsp->withTbName == false); ASSERT(pRsp->withSchema == false); return 0; @@ -148,15 +144,15 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta uint64_t ts = 0; tqDebug("tmqsnap task start to execute"); if (qExecTask(task, &pDataBlock, &ts) < 0) { - ASSERT(0); + tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, terrstr()); + return -1; } tqDebug("tmqsnap task execute end, get %p", pDataBlock); if (pDataBlock != NULL) { if (pRsp->withTbName) { - int64_t uid = 0; if (pOffset->type == TMQ_OFFSET__LOG) { - uid = pExec->pExecReader->msgIter.uid; + int64_t uid = pExec->pExecReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp, 1) < 0) { continue; } @@ -215,25 +211,29 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta break; } - if (qStreamExtractOffset(task, &pRsp->rspOffset) < 0) { - ASSERT(0); + qStreamExtractOffset(task, &pRsp->rspOffset); + + if (pRsp->rspOffset.type == 0) { + tqError("expected rsp offset: type %d %" PRId64 " %" PRId64 " %" PRId64, pRsp->rspOffset.type, pRsp->rspOffset.ts, + pRsp->rspOffset.uid, pRsp->rspOffset.version); + return -1; } - ASSERT(pRsp->rspOffset.type != 0); return 0; } -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp) { +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp) { STqExecHandle* pExec = &pHandle->execHandle; - ASSERT(pExec->subType != TOPIC_SUB_TYPE__COLUMN); + /*A(pExec->subType != TOPIC_SUB_TYPE__COLUMN);*/ SArray* pBlocks = taosArrayInit(0, sizeof(SSDataBlock)); SArray* pSchemas = taosArrayInit(0, sizeof(void*)); if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { STqReader* pReader = pExec->pExecReader; - tqReaderSetDataMsg(pReader, pReq, 0); - while (tqNextDataBlock(pReader)) { + /*tqReaderSetDataMsg(pReader, pReq, 0);*/ + tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver); + while (tqNextDataBlock2(pReader)) { /*SSDataBlock block = {0};*/ /*if (tqRetrieveDataBlock(&block, pReader) < 0) {*/ /*if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;*/ @@ -241,11 +241,12 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp taosArrayClear(pBlocks); taosArrayClear(pSchemas); - if (tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas) < 0) { + if (tqRetrieveTaosxBlock2(pReader, pBlocks, pSchemas) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; } if (pRsp->withTbName) { - int64_t uid = pExec->pExecReader->msgIter.uid; + /*int64_t uid = pExec->pExecReader->msgIter.uid;*/ + int64_t uid = pExec->pExecReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlocks)) < 0) { taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper); @@ -255,7 +256,9 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp } } if (pHandle->fetchMeta) { +#if 0 SSubmitBlk* pBlk = pReader->pBlock; + int64_t uid = pExec->pExecReader->lastBlkUid; int32_t schemaLen = htonl(pBlk->schemaLen); if (schemaLen > 0) { if (pRsp->createTableNum == 0) { @@ -268,6 +271,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp taosArrayPush(pRsp->createTableReq, &createReq); pRsp->createTableNum++; } +#endif } for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) { SSDataBlock* pBlock = taosArrayGet(pBlocks, i); @@ -281,19 +285,20 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp } } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { STqReader* pReader = pExec->pExecReader; - tqReaderSetDataMsg(pReader, pReq, 0); - while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) { + /*tqReaderSetDataMsg(pReader, pReq, 0);*/ + tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver); + while (tqNextDataBlockFilterOut2(pReader, pExec->execDb.pFilterOutTbUid)) { /*SSDataBlock block = {0};*/ /*if (tqRetrieveDataBlock(&block, pReader) < 0) {*/ /*if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;*/ /*}*/ taosArrayClear(pBlocks); taosArrayClear(pSchemas); - if (tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas) < 0) { + if (tqRetrieveTaosxBlock2(pReader, pBlocks, pSchemas) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; } if (pRsp->withTbName) { - int64_t uid = pExec->pExecReader->msgIter.uid; + int64_t uid = pExec->pExecReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlocks)) < 0) { taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper); @@ -303,6 +308,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp } } if (pHandle->fetchMeta) { +#if 0 SSubmitBlk* pBlk = pReader->pBlock; int32_t schemaLen = htonl(pBlk->schemaLen); if (schemaLen > 0) { @@ -316,6 +322,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp taosArrayPush(pRsp->createTableReq, &createReq); pRsp->createTableNum++; } +#endif } /*tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock),*/ /*pTq->pVnode->config.tsdbCfg.precision);*/ diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 3ad01e2370bae65d474d06c0f4dde37c1df96f48..05ed8d734854415fefdd8aa6b830edbdf007793b 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -71,17 +71,14 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) { int32_t tqMetaOpen(STQ* pTq) { if (tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaDB, 0) < 0) { - ASSERT(0); return -1; } if (tdbTbOpen("tq.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pExecStore, 0) < 0) { - ASSERT(0); return -1; } if (tdbTbOpen("tq.check.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pCheckStore, 0) < 0) { - ASSERT(0); return -1; } @@ -135,19 +132,19 @@ int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key) { if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { - ASSERT(0); + return -1; } if (tdbTbDelete(pTq->pCheckStore, key, (int)strlen(key), txn) < 0) { - /*ASSERT(0);*/ + tqWarn("vgId:%d, tq try delete checkinfo failed %s", pTq->pVnode->config.vgId, key); } if (tdbCommit(pTq->pMetaDB, txn) < 0) { - ASSERT(0); + return -1; } if (tdbPostCommit(pTq->pMetaDB, txn) < 0) { - ASSERT(0); + return -1; } return 0; @@ -156,7 +153,6 @@ int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key) { int32_t tqMetaRestoreCheckInfo(STQ* pTq) { TBC* pCur = NULL; if (tdbTbcOpen(pTq->pCheckStore, &pCur, NULL) < 0) { - ASSERT(0); return -1; } @@ -197,40 +193,42 @@ int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) { int32_t code; int32_t vlen; tEncodeSize(tEncodeSTqHandle, pHandle, vlen, code); - ASSERT(code == 0); + if (code < 0) { + return -1; + } tqDebug("tq save %s(%d) consumer %" PRId64 " vgId:%d", pHandle->subKey, (int32_t)strlen(pHandle->subKey), pHandle->consumerId, TD_VID(pTq->pVnode)); void* buf = taosMemoryCalloc(1, vlen); if (buf == NULL) { - ASSERT(0); + return -1; } SEncoder encoder; tEncoderInit(&encoder, buf, vlen); if (tEncodeSTqHandle(&encoder, pHandle) < 0) { - ASSERT(0); + return -1; } TXN* txn; if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { - ASSERT(0); + return -1; } if (tdbTbUpsert(pTq->pExecStore, key, (int)strlen(key), buf, vlen, txn) < 0) { - ASSERT(0); + return -1; } if (tdbCommit(pTq->pMetaDB, txn) < 0) { - ASSERT(0); + return -1; } if (tdbPostCommit(pTq->pMetaDB, txn) < 0) { - ASSERT(0); + return -1; } tEncoderClear(&encoder); @@ -243,19 +241,18 @@ int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) { if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { - ASSERT(0); + return -1; } if (tdbTbDelete(pTq->pExecStore, key, (int)strlen(key), txn) < 0) { - /*ASSERT(0);*/ } if (tdbCommit(pTq->pMetaDB, txn) < 0) { - ASSERT(0); + return -1; } if (tdbPostCommit(pTq->pMetaDB, txn) < 0) { - ASSERT(0); + return -1; } return 0; @@ -264,7 +261,6 @@ int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) { int32_t tqMetaRestoreHandle(STQ* pTq) { TBC* pCur = NULL; if (tdbTbcOpen(pTq->pExecStore, &pCur, NULL) < 0) { - ASSERT(0); return -1; } @@ -284,7 +280,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { handle.pRef = walOpenRef(pTq->pVnode->pWal); if (handle.pRef == NULL) { - continue; + return -1; } walRefVer(handle.pRef, handle.snapshotVer); @@ -299,12 +295,19 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { handle.execHandle.task = qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, &handle.execHandle.numOfCols, NULL); - ASSERT(handle.execHandle.task); + if (handle.execHandle.task == NULL) { + tqError("cannot create exec task for %s", handle.subKey); + return -1; + } void* scanner = NULL; qExtractStreamScanner(handle.execHandle.task, &scanner); - ASSERT(scanner); + if (scanner == NULL) { + tqError("cannot extract stream scanner for %s", handle.subKey); + } handle.execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner); - ASSERT(handle.execHandle.pExecReader); + if (handle.execHandle.pExecReader == NULL) { + tqError("cannot extract exec reader for %s", handle.subKey); + } } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__DB) { handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); handle.execHandle.pExecReader = tqOpenReader(pTq->pVnode); diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 9e04b70f41d7041a9c8d72a5c63be5a4b59105bb..66d1ac2c7e530e4b8c732deaaa83afc6191b5e44 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -40,8 +40,7 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { if (code == 0) { break; } else { - ASSERT(0); - // TODO handle error + return -1; } } int32_t size = htonl(head.size); @@ -101,7 +100,9 @@ STqOffsetStore* tqOffsetOpen(STQ* pTq) { } char* fname = tqOffsetBuildFName(pStore->pTq->path, 0); if (tqOffsetRestoreFromFile(pStore, fname) < 0) { - ASSERT(0); + taosMemoryFree(fname); + taosMemoryFree(pStore); + return NULL; } taosMemoryFree(fname); return pStore; @@ -152,9 +153,7 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) { int32_t bodyLen; int32_t code; tEncodeSize(tEncodeSTqOffset, pOffset, bodyLen, code); - ASSERT(code == 0); if (code < 0) { - ASSERT(0); taosHashCancelIterate(pStore->pHash, pIter); return -1; } @@ -170,7 +169,6 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) { // write file int64_t writeLen; if ((writeLen = taosWriteFile(pFile, buf, totLen)) != totLen) { - ASSERT(0); tqError("write offset incomplete, len %d, write len %" PRId64, bodyLen, writeLen); taosHashCancelIterate(pStore->pHash, pIter); taosMemoryFree(buf); diff --git a/source/dnode/vnode/src/tq/tqOffsetSnapshot.c b/source/dnode/vnode/src/tq/tqOffsetSnapshot.c index b63ff8af1d623aa2017ce5768a80e2dbf783d5a4..2413a792c60c36a059e5411f2ff51d3792c12aa0 100644 --- a/source/dnode/vnode/src/tq/tqOffsetSnapshot.c +++ b/source/dnode/vnode/src/tq/tqOffsetSnapshot.c @@ -56,24 +56,28 @@ int32_t tqOffsetSnapRead(STqOffsetReader* pReader, uint8_t** ppData) { TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); if (pFile == NULL) { taosMemoryFree(fname); - return 0; + return -1; } int64_t sz = 0; if (taosStatFile(fname, &sz, NULL) < 0) { - ASSERT(0); + taosCloseFile(&pFile); + taosMemoryFree(fname); + return -1; } taosMemoryFree(fname); SSnapDataHdr* buf = taosMemoryCalloc(1, sz + sizeof(SSnapDataHdr)); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + taosCloseFile(&pFile); return terrno; } void* abuf = POINTER_SHIFT(buf, sizeof(SSnapDataHdr)); int64_t contLen = taosReadFile(pFile, abuf, sz); if (contLen != sz) { - ASSERT(0); + taosCloseFile(&pFile); + taosMemoryFree(buf); return -1; } buf->size = sz; @@ -122,14 +126,17 @@ int32_t tqOffsetWriterClose(STqOffsetWriter** ppWriter, int8_t rollback) { if (rollback) { if (taosRemoveFile(pWriter->fname) < 0) { - ASSERT(0); + taosMemoryFree(fname); + return -1; } } else { if (taosRenameFile(pWriter->fname, fname) < 0) { - ASSERT(0); + taosMemoryFree(fname); + return -1; } if (tqOffsetRestoreFromFile(pTq->pOffsetStore, fname) < 0) { - ASSERT(0); + taosMemoryFree(fname); + return -1; } } taosMemoryFree(fname); @@ -146,14 +153,13 @@ int32_t tqOffsetSnapWrite(STqOffsetWriter* pWriter, uint8_t* pData, uint32_t nDa TdFilePtr pFile = taosOpenFile(pWriter->fname, TD_FILE_CREATE | TD_FILE_WRITE); SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; int64_t size = pHdr->size; - ASSERT(size == nData - sizeof(SSnapDataHdr)); if (pFile) { int64_t contLen = taosWriteFile(pFile, pHdr->data, size); if (contLen != size) { - ASSERT(0); + taosCloseFile(&pFile); + return -1; } } else { - ASSERT(0); return -1; } return 0; diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index f89bc2036252961ff10d81774a9c6f756fdebf43..559a3b76fec3a532d2e40fe62a203e515ce38252 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -25,9 +25,7 @@ void tqTmrRspFunc(void* param, void* tmrId) { static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubmit** ppSubmit, SMqDataRsp* pRsp) { SStreamDataSubmit* pSubmit = *ppSubmit; while (pSubmit != NULL) { - ASSERT(pSubmit->ver == pHandle->pushHandle.processedVer + 1); if (tqLogScanExec(pTq, &pHandle->execHandle, pSubmit->data, pRsp, 0) < 0) { - /*ASSERT(0);*/ } // update processed atomic_store_64(&pHandle->pushHandle.processedVer, pSubmit->ver); @@ -160,8 +158,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ if (msgType == TDMT_VND_SUBMIT) { tqLogScanExec(pTq, &pHandle->execHandle, pReq, &rsp, workerId); } else { - // TODO - ASSERT(0); + tqError("tq push unexpected msg type %d", msgType); } if (rsp.blockNum == 0) { @@ -169,9 +166,6 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ continue; } - ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum); - ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum); - rsp.rspOffset = fetchOffset; int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, &rsp); @@ -213,7 +207,11 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ #endif int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { - tqDebug("vgId:%d, tq push msg ver %" PRId64 ", type: %s", pTq->pVnode->config.vgId, ver, TMSG_INFO(msgType)); + void* pReq = POINTER_SHIFT(msg, sizeof(SMsgHead)); + int32_t len = msgLen - sizeof(SMsgHead); + + tqDebug("vgId:%d, tq push msg ver %" PRId64 ", type: %s, p head %p, p body %p, len %d", pTq->pVnode->config.vgId, ver, + TMSG_INFO(msgType), msg, pReq, len); if (msgType == TDMT_VND_SUBMIT) { // lock push mgr to avoid potential msg lost @@ -222,7 +220,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) if (taosHashGetSize(pTq->pPushMgr) != 0) { SArray* cachedKeys = taosArrayInit(0, sizeof(void*)); SArray* cachedKeyLens = taosArrayInit(0, sizeof(size_t)); - void* data = taosMemoryMalloc(msgLen); + void* data = taosMemoryMalloc(len); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to copy data for stream since out of memory"); @@ -230,9 +228,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) taosArrayDestroy(cachedKeyLens); return -1; } - memcpy(data, msg, msgLen); - SSubmitReq* pReq = (SSubmitReq*)data; - pReq->version = ver; + memcpy(data, pReq, len); void* pIter = NULL; while (1) { @@ -256,14 +252,19 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) SMqDataRsp* pRsp = &pPushEntry->dataRsp; // prepare scan mem data - qStreamScanMemData(task, pReq); + SPackedData submit = { + .msgStr = data, + .msgLen = len, + .ver = ver, + }; + qStreamSetScanMemData(task, submit); // exec while (1) { SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; if (qExecTask(task, &pDataBlock, &ts) < 0) { - ASSERT(0); + tqDebug("vgId:%d, tq exec error since %s", pTq->pVnode->config.vgId, terrstr()); } if (pDataBlock == NULL) { @@ -282,7 +283,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) // remove from hash size_t kLen; void* key = taosHashGetKey(pIter, &kLen); - void* keyCopy = taosMemoryMalloc(kLen); + void* keyCopy = taosMemoryCalloc(1, kLen + 1); memcpy(keyCopy, key, kLen); taosArrayPush(cachedKeys, &keyCopy); @@ -296,7 +297,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) void* key = taosArrayGetP(cachedKeys, i); size_t kLen = *(size_t*)taosArrayGet(cachedKeyLens, i); if (taosHashRemove(pTq->pPushMgr, key, kLen) != 0) { - ASSERT(0); + tqError("vgId:%d, tq push hash remove key error, key: %s", pTq->pVnode->config.vgId, (char*)key); } } taosArrayDestroyP(cachedKeys, (FDelete)taosMemoryFree); @@ -310,17 +311,22 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) if (vnodeIsRoleLeader(pTq->pVnode)) { if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) return 0; if (msgType == TDMT_VND_SUBMIT) { - void* data = taosMemoryMalloc(msgLen); + void* data = taosMemoryMalloc(len); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to copy data for stream since out of memory"); return -1; } - memcpy(data, msg, msgLen); - SSubmitReq* pReq = (SSubmitReq*)data; - pReq->version = ver; + memcpy(data, pReq, len); + SPackedData submit = { + .msgStr = data, + .msgLen = len, + .ver = ver, + }; + + tqDebug("tq copy write msg %p %d %" PRId64 " from %p", data, len, ver, pReq); - tqProcessSubmitReq(pTq, data, ver); + tqProcessSubmitReq(pTq, submit); } if (msgType == TDMT_VND_DELETE) { tqProcessDelReq(pTq, POINTER_SHIFT(msg, sizeof(SMsgHead)), msgLen - sizeof(SMsgHead), ver); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 46b31bc5b0fed0f42e3f570772ec46a190f70ea1..eb9c0c3eeb252b53caf2b04a0674e879242cca42 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -176,8 +176,6 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) { goto end; } realTbSuid = req.suid; - } else { - ASSERT(0); } end: @@ -206,7 +204,6 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea code = walFetchBody(pHandle->pWalReader, ppCkHead); if (code < 0) { - ASSERT(0); *fetchOffset = offset; code = -1; goto END; @@ -220,7 +217,6 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea if (IS_META_MSG(pHead->msgType)) { code = walFetchBody(pHandle->pWalReader, ppCkHead); if (code < 0) { - ASSERT(0); *fetchOffset = offset; code = -1; goto END; @@ -238,7 +234,6 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea } code = walSkipFetchBody(pHandle->pWalReader, *ppCkHead); if (code < 0) { - ASSERT(0); *fetchOffset = offset; code = -1; goto END; @@ -252,7 +247,7 @@ END: } STqReader* tqOpenReader(SVnode* pVnode) { - STqReader* pReader = taosMemoryMalloc(sizeof(STqReader)); + STqReader* pReader = taosMemoryCalloc(1, sizeof(STqReader)); if (pReader == NULL) { return NULL; } @@ -264,7 +259,7 @@ STqReader* tqOpenReader(SVnode* pVnode) { } pReader->pVnodeMeta = pVnode->pMeta; - pReader->pMsg = NULL; + /*pReader->pMsg = NULL;*/ pReader->ver = -1; pReader->pColIdList = NULL; pReader->cachedSchemaVer = 0; @@ -297,16 +292,13 @@ void tqCloseReader(STqReader* pReader) { int32_t tqSeekVer(STqReader* pReader, int64_t ver) { if (walReadSeekVer(pReader->pWalReader, ver) < 0) { - ASSERT(pReader->pWalReader->curInvalid); - ASSERT(pReader->pWalReader->curVersion == ver); return -1; } - ASSERT(pReader->pWalReader->curVersion == ver); return 0; } int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { - bool fromProcessedMsg = pReader->pMsg != NULL; + bool fromProcessedMsg = pReader->msg2.msgStr != NULL; while (1) { if (!fromProcessedMsg) { @@ -317,10 +309,11 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { ret->offset.version = pReader->ver; ret->fetchType = FETCH_TYPE__NONE; tqDebug("return offset %" PRId64 ", no more valid", ret->offset.version); - ASSERT(ret->offset.version >= 0); return -1; } - void* body = pReader->pWalReader->pHead->head.body; + void* body = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SMsgHead)); + int32_t bodyLen = pReader->pWalReader->pHead->head.bodyLen - sizeof(SMsgHead); + int64_t ver = pReader->pWalReader->pHead->head.version; #if 0 if (pReader->pWalReader->pHead->head.msgType != TDMT_VND_SUBMIT) { // TODO do filter @@ -329,18 +322,18 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { return 0; } else { #endif - tqReaderSetDataMsg(pReader, body, pReader->pWalReader->pHead->head.version); + tqReaderSetSubmitReq2(pReader, body, bodyLen, ver); + /*tqReaderSetDataMsg(pReader, body, pReader->pWalReader->pHead->head.version);*/ #if 0 } #endif } - while (tqNextDataBlock(pReader)) { + while (tqNextDataBlock2(pReader)) { // TODO mem free memset(&ret->data, 0, sizeof(SSDataBlock)); - int32_t code = tqRetrieveDataBlock(&ret->data, pReader); + int32_t code = tqRetrieveDataBlock2(&ret->data, pReader); if (code != 0 || ret->data.info.rows == 0) { - ASSERT(0); continue; } ret->fetchType = FETCH_TYPE__DATA; @@ -351,7 +344,6 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { if (fromProcessedMsg) { ret->offset.type = TMQ_OFFSET__LOG; ret->offset.version = pReader->ver; - ASSERT(pReader->ver >= 0); ret->fetchType = FETCH_TYPE__SEP; tqDebug("return offset %" PRId64 ", processed finish", ret->offset.version); return 0; @@ -359,6 +351,7 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { } } +#if 0 int32_t tqReaderSetDataMsg(STqReader* pReader, const SSubmitReq* pMsg, int64_t ver) { pReader->pMsg = pMsg; @@ -373,7 +366,33 @@ int32_t tqReaderSetDataMsg(STqReader* pReader, const SSubmitReq* pMsg, int64_t v memset(&pReader->blkIter, 0, sizeof(SSubmitBlkIter)); return 0; } +#endif + +int32_t tqReaderSetSubmitReq2(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) { + ASSERT(pReader->msg2.msgStr == NULL); + ASSERT(msgStr); + ASSERT(msgLen); + ASSERT(ver >= 0); + pReader->msg2.msgStr = msgStr; + pReader->msg2.msgLen = msgLen; + pReader->msg2.ver = ver; + pReader->ver = ver; + + tqDebug("tq reader set msg %p %d", msgStr, msgLen); + + if (pReader->setMsg == 0) { + SDecoder decoder; + tDecoderInit(&decoder, pReader->msg2.msgStr, pReader->msg2.msgLen); + if (tDecodeSSubmitReq2(&decoder, &pReader->submit) < 0) { + ASSERT(0); + } + tDecoderClear(&decoder); + pReader->setMsg = 1; + } + return 0; +} +#if 0 bool tqNextDataBlock(STqReader* pReader) { if (pReader->pMsg == NULL) return false; while (1) { @@ -397,6 +416,59 @@ bool tqNextDataBlock(STqReader* pReader) { } return false; } +#endif + +bool tqNextDataBlock2(STqReader* pReader) { + if (pReader->msg2.msgStr == NULL) return false; + ASSERT(pReader->setMsg == 1); + + tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg2.msgStr, pReader->msg2.msgLen, + pReader->msg2.ver, pReader->nextBlk); + + int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); + while (pReader->nextBlk < blockSz) { + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); + ASSERT(pSubmitTbData->uid); + + if (pReader->tbIdHash == NULL) return true; + + void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); + if (ret != NULL) { + return true; + } + pReader->nextBlk++; + } + + tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); + pReader->setMsg = 0; + pReader->nextBlk = 0; + pReader->msg2.msgStr = NULL; + + return false; +} + +bool tqNextDataBlockFilterOut2(STqReader* pReader, SHashObj* filterOutUids) { + if (pReader->msg2.msgStr == NULL) return false; + ASSERT(pReader->setMsg == 1); + + int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); + while (pReader->nextBlk < blockSz) { + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); + if (pReader->tbIdHash == NULL) return true; + + void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); + if (ret == NULL) { + return true; + } + } + + tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); + pReader->setMsg = 0; + pReader->nextBlk = 0; + pReader->msg2.msgStr = NULL; + + return false; +} int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrapper* pSrc, char* mask) { int32_t code; @@ -427,6 +499,7 @@ int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrap return 0; } +#if 0 bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) { while (1) { if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { @@ -434,7 +507,6 @@ bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) { } if (pHandle->pBlock == NULL) return false; - ASSERT(pHandle->tbIdHash == NULL); void* ret = taosHashGet(filterOutUids, &pHandle->msgIter.uid, sizeof(int64_t)); if (ret == NULL) { return true; @@ -443,6 +515,253 @@ bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) { return false; } +int32_t tqScanSubmitSplit(SArray* pBlocks, SArray* schemas, STqReader* pReader) { + // + int32_t sversion = htonl(pReader->pBlock->sversion); + if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || + pReader->cachedSchemaSuid != pReader->msgIter.suid) { + taosMemoryFree(pReader->pSchema); + pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); + if (pReader->pSchema == NULL) { + tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 + "), version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, pReader->msgIter.uid, pReader->msgIter.suid, sversion); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + tDeleteSSchemaWrapper(pReader->pSchemaWrapper); + pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); + if (pReader->pSchemaWrapper == NULL) { + tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, pReader->msgIter.uid, pReader->cachedSchemaVer); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + STSchema* pTschema = pReader->pSchema; + SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; + + int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList); + } + return 0; +} +#endif + +int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader) { + int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); + ASSERT(pReader->nextBlk < blockSz); + + tqDebug("tq reader retrieve data block %p, %d", pReader->msg2.msgStr, pReader->nextBlk); + + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); + pReader->nextBlk++; + + int32_t sversion = pSubmitTbData->sver; + int64_t suid = pSubmitTbData->suid; + int64_t uid = pSubmitTbData->uid; + pReader->lastBlkUid = uid; + + pBlock->info.id.uid = uid; + pBlock->info.version = pReader->msg2.ver; + + if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || pReader->cachedSchemaSuid != suid) { + taosMemoryFree(pReader->pSchema); + pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, uid, sversion, 1); + if (pReader->pSchema == NULL) { + tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 + "), version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, uid, suid, sversion); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + tDeleteSSchemaWrapper(pReader->pSchemaWrapper); + pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1); + if (pReader->pSchemaWrapper == NULL) { + tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, uid, pReader->cachedSchemaVer); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + STSchema* pTschema = pReader->pSchema; + SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; + + int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList); + + if (colNumNeed == 0) { + int32_t colMeta = 0; + while (colMeta < pSchemaWrapper->nCols) { + SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { + goto FAIL; + } + colMeta++; + } + } else { + if (colNumNeed > pSchemaWrapper->nCols) { + colNumNeed = pSchemaWrapper->nCols; + } + + int32_t colMeta = 0; + int32_t colNeed = 0; + while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { + SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; + col_id_t colIdSchema = pColSchema->colId; + col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pReader->pColIdList, colNeed); + if (colIdSchema < colIdNeed) { + colMeta++; + } else if (colIdSchema > colIdNeed) { + colNeed++; + } else { + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { + goto FAIL; + } + colMeta++; + colNeed++; + } + } + } + + int32_t numOfRows = 0; + + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + SArray* pCols = pSubmitTbData->aCol; + SColData* pCol = taosArrayGet(pCols, 0); + numOfRows = pCol->nVal; + } else { + SArray* pRows = pSubmitTbData->aRowP; + numOfRows = taosArrayGetSize(pRows); + } + + if (blockDataEnsureCapacity(pBlock, numOfRows) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto FAIL; + } + pBlock->info.rows = numOfRows; + + int32_t colActual = blockDataGetNumOfCols(pBlock); + + // convert and scan one block + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + SArray* pCols = pSubmitTbData->aCol; + int32_t numOfCols = taosArrayGetSize(pCols); + int32_t targetIdx = 0; + int32_t sourceIdx = 0; + while (targetIdx < colActual) { + ASSERT(sourceIdx < numOfCols); + + SColData* pCol = taosArrayGet(pCols, sourceIdx); + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx); + SColVal colVal; + + ASSERT(pCol->nVal == numOfRows); + + if (pCol->cid < pColData->info.colId) { + sourceIdx++; + } else if (pCol->cid == pColData->info.colId) { + for (int32_t i = 0; i < pCol->nVal; i++) { + tColDataGetValue(pCol, sourceIdx, &colVal); +#if 0 + void* val = NULL; + if (IS_STR_DATA_TYPE(colVal.type)) { + val = colVal.value.pData; + } else { + val = &colVal.value.val; + } + if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } +#endif + if (IS_STR_DATA_TYPE(colVal.type)) { + if (colVal.value.pData != NULL) { + char val[65535 + 2]; + memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); + varDataSetLen(val, colVal.value.nData); + if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } else { + colDataAppendNULL(pColData, i); + } + } else { + if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } + } + sourceIdx++; + targetIdx++; + } else { + ASSERT(0); + } + } + } else { + SArray* pRows = pSubmitTbData->aRowP; + + for (int32_t i = 0; i < numOfRows; i++) { + SRow* pRow = taosArrayGetP(pRows, i); + int32_t targetIdx = 0; + int32_t sourceIdx = 0; + + for (int32_t j = 0; j < colActual; j++) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, j); + while (1) { + ASSERT(sourceIdx < pTschema->numOfCols); + + SColVal colVal; + tRowGet(pRow, pTschema, sourceIdx, &colVal); + if (colVal.cid < pColData->info.colId) { + sourceIdx++; + continue; + } else if (colVal.cid == pColData->info.colId) { + if (IS_STR_DATA_TYPE(colVal.type)) { + if (colVal.value.pData != NULL) { + char val[65535 + 2]; + memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); + varDataSetLen(val, colVal.value.nData); + if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } else { + colDataAppendNULL(pColData, i); + } + /*val = colVal.value.pData;*/ + } else { + if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } + + sourceIdx++; + targetIdx++; + break; + } else { + ASSERT(0); + } + } + } + } + } + } + + return 0; + +FAIL: + blockDataFreeRes(pBlock); + return -1; +} + +#if 0 int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { // TODO: cache multiple schema int32_t sversion = htonl(pReader->pBlock->sversion); @@ -453,7 +772,6 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { if (pReader->pSchema == NULL) { tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table", pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer); - /*ASSERT(0);*/ pReader->cachedSchemaSuid = 0; terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; return -1; @@ -464,7 +782,6 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { if (pReader->pSchemaWrapper == NULL) { tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", pReader->msgIter.uid, pReader->cachedSchemaVer); - /*ASSERT(0);*/ pReader->cachedSchemaSuid = 0; terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; return -1; @@ -567,7 +884,6 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas if (pReader->pSchema == NULL) { tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table", pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer); - /*ASSERT(0);*/ pReader->cachedSchemaSuid = 0; terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; return -1; @@ -578,7 +894,6 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas if (pReader->pSchemaWrapper == NULL) { tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", pReader->msgIter.uid, pReader->cachedSchemaVer); - /*ASSERT(0);*/ pReader->cachedSchemaSuid = 0; terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; return -1; @@ -671,8 +986,6 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas break; } - ASSERT(sVal.valType != TD_VTYPE_NONE); - if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) { goto FAIL; } @@ -691,6 +1004,18 @@ FAIL: taosMemoryFree(assigned); return -1; } +#endif + +int32_t tqRetrieveTaosxBlock2(STqReader* pReader, SArray* blocks, SArray* schemas) { + SSDataBlock block = {0}; + if (tqRetrieveDataBlock2(&block, pReader) == 0) { + taosArrayPush(blocks, &block); + SSchemaWrapper* pSW = tCloneSSchemaWrapper(pReader->pSchemaWrapper); + taosArrayPush(schemas, &pSW); + return 0; + } + return -1; +} void tqReaderSetColIdList(STqReader* pReader, SArray* pColIdList) { pReader->pColIdList = pColIdList; } @@ -732,8 +1057,6 @@ int tqReaderAddTbUidList(STqReader* pReader, const SArray* tbUidList) { } int tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) { - ASSERT(pReader->tbIdHash != NULL); - for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i); taosHashRemove(pReader->tbIdHash, pKey, sizeof(int64_t)); @@ -750,7 +1073,10 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { STqHandle* pExec = (STqHandle*)pIter; if (pExec->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { int32_t code = qUpdateQualifiedTableId(pExec->execHandle.task, tbUidList, isAdd); - ASSERT(code == 0); + if (code != 0) { + tqError("update qualified table error for %s", pExec->subKey); + continue; + } } else if (pExec->execHandle.subType == TOPIC_SUB_TYPE__DB) { if (!isAdd) { int32_t sz = taosArrayGetSize(tbUidList); @@ -769,7 +1095,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { int32_t code = metaGetTableEntryByUidCache(&mr, *id); if (code != TSDB_CODE_SUCCESS) { - qError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno)); + tqError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno)); continue; } @@ -790,8 +1116,6 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } else { // TODO handle delete table from stb } - } else { - ASSERT(0); } } while (1) { @@ -800,7 +1124,10 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->taskLevel == TASK_LEVEL__SOURCE) { int32_t code = qUpdateQualifiedTableId(pTask->exec.executor, tbUidList, isAdd); - ASSERT(code == 0); + if (code != 0) { + tqError("update qualified table error for stream task %d", pTask->taskId); + continue; + } } } return 0; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 6d474d5aa1360e9d7425b8b6926439ee1aafd160..f1103ad48a1be4101da041e40ddf1e7b4a6b181b 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -19,7 +19,6 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq) { - ASSERT(pDataBlock->info.type == STREAM_DELETE_RESULT); int32_t totRow = pDataBlock->info.rows; SColumnInfoData* pStartTsCol = taosArrayGet(pDataBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pEndTsCol = taosArrayGet(pDataBlock->pDataBlock, END_TS_COLUMN_INDEX); @@ -72,233 +71,6 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBl return 0; } -SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pTSchema, - SSchemaWrapper* pTagSchemaWrapper, bool createTb, int64_t suid, const char* stbFullName, - SBatchDeleteReq* pDeleteReq) { - SSubmitReq* ret = NULL; - SArray* schemaReqs = NULL; - SArray* schemaReqSz = NULL; - SArray* tagArray = taosArrayInit(1, sizeof(STagVal)); - if (!tagArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - int32_t sz = taosArrayGetSize(pBlocks); - - if (createTb) { - schemaReqs = taosArrayInit(sz, sizeof(void*)); - schemaReqSz = taosArrayInit(sz, sizeof(int32_t)); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - if (pDataBlock->info.type == STREAM_DELETE_RESULT) { - int32_t padding1 = 0; - void* padding2 = NULL; - taosArrayPush(schemaReqSz, &padding1); - taosArrayPush(schemaReqs, &padding2); - continue; - } - - // STag* pTag = NULL; - // taosArrayClear(tagArray); - // SArray *tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); - // for(int j = 0; j < pTagSchemaWrapper->nCols; j++){ - // STagVal tagVal = { - // .cid = pTagSchemaWrapper->pSchema[j].colId, - // .type = pTagSchemaWrapper->pSchema[j].type, - // .i64 = (int64_t)pDataBlock->info.id.groupId, - // }; - // taosArrayPush(tagArray, &tagVal); - // taosArrayPush(tagName, pTagSchemaWrapper->pSchema[j].name); - // } - // - // tTagNew(tagArray, 1, false, &pTag); - // if (pTag == NULL) { - // terrno = TSDB_CODE_OUT_OF_MEMORY; - // taosArrayDestroy(tagArray); - // taosArrayDestroy(tagName); - // return NULL; - // } - - SVCreateTbReq createTbReq = {0}; - - // set const - createTbReq.flags = 0; - createTbReq.type = TSDB_CHILD_TABLE; - createTbReq.ctb.suid = suid; - - // set super table name - SName name = {0}; - tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - createTbReq.ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName); - - // set tag content - taosArrayClear(tagArray); - STagVal tagVal = { - .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, - .type = TSDB_DATA_TYPE_UBIGINT, - .i64 = (int64_t)pDataBlock->info.id.groupId, - }; - taosArrayPush(tagArray, &tagVal); - createTbReq.ctb.tagNum = taosArrayGetSize(tagArray); - - STag* pTag = NULL; - tTagNew(tagArray, 1, false, &pTag); - if (pTag == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosArrayDestroy(tagArray); - taosArrayDestroyP(schemaReqs, taosMemoryFree); - taosArrayDestroy(schemaReqSz); - return NULL; - } - createTbReq.ctb.pTag = (uint8_t*)pTag; - - // set tag name - SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); - char tagNameStr[TSDB_COL_NAME_LEN] = {0}; - strcpy(tagNameStr, "group_id"); - taosArrayPush(tagName, tagNameStr); - createTbReq.ctb.tagName = tagName; - - // set table name - if (pDataBlock->info.parTbName[0]) { - createTbReq.name = strdup(pDataBlock->info.parTbName); - } else { - createTbReq.name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId); - } - - // save schema len - int32_t code; - int32_t schemaLen; - tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); - if (code < 0) { - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); - taosArrayDestroyP(schemaReqs, taosMemoryFree); - taosArrayDestroy(schemaReqSz); - return NULL; - } - taosArrayPush(schemaReqSz, &schemaLen); - - // save schema str - void* schemaStr = taosMemoryMalloc(schemaLen); - if (schemaStr == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); - taosArrayDestroyP(schemaReqs, taosMemoryFree); - taosArrayDestroy(schemaReqSz); - return NULL; - } - taosArrayPush(schemaReqs, &schemaStr); - - SEncoder encoder = {0}; - tEncoderInit(&encoder, schemaStr, schemaLen); - code = tEncodeSVCreateTbReq(&encoder, &createTbReq); - if (code < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); - taosArrayDestroyP(schemaReqs, taosMemoryFree); - taosArrayDestroy(schemaReqSz); - tEncoderClear(&encoder); - return NULL; - } - tEncoderClear(&encoder); - tdDestroySVCreateTbReq(&createTbReq); - } - } - taosArrayDestroy(tagArray); - - // cal size - int32_t cap = sizeof(SSubmitReq); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - if (pDataBlock->info.type == STREAM_DELETE_RESULT) { - continue; - } - int32_t rows = pDataBlock->info.rows; - /*int32_t rowSize = pDataBlock->info.rowSize;*/ - int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); - - int32_t schemaLen = 0; - if (createTb) { - schemaLen = *(int32_t*)taosArrayGet(schemaReqSz, i); - } - cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen; - } - - // assign data - ret = rpcMallocCont(cap); - ret->header.vgId = pVnode->config.vgId; - ret->length = sizeof(SSubmitReq); - ret->numOfBlocks = htonl(sz); - - SSubmitBlk* blkHead = POINTER_SHIFT(ret, sizeof(SSubmitReq)); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - if (pDataBlock->info.type == STREAM_DELETE_RESULT) { - pDeleteReq->suid = suid; - pDeleteReq->deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); - tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, pDeleteReq); - continue; - } - - blkHead->numOfRows = htonl(pDataBlock->info.rows); - blkHead->sversion = htonl(pTSchema->version); - blkHead->suid = htobe64(suid); - // uid is assigned by vnode - blkHead->uid = 0; - - int32_t rows = pDataBlock->info.rows; - - tqDebug("tq sink, convert block1 %d, rows: %d", i, rows); - - int32_t dataLen = 0; - int32_t schemaLen = 0; - void* blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); - if (createTb) { - schemaLen = *(int32_t*)taosArrayGet(schemaReqSz, i); - void* schemaStr = taosArrayGetP(schemaReqs, i); - memcpy(blkSchema, schemaStr, schemaLen); - } - blkHead->schemaLen = htonl(schemaLen); - - STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - for (int32_t j = 0; j < rows; j++) { - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTSchema->version); - tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); - tdSRowResetBuf(&rb, rowData); - - for (int32_t k = 0; k < pTSchema->numOfCols; k++) { - const STColumn* pColumn = &pTSchema->columns[k]; - SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, k); - if (colDataIsNull_s(pColData, j)) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, pColumn->offset, k); - } else { - void* data = colDataGetData(pColData, j); - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); - } - } - tdSRowEnd(&rb); - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - dataLen += rowLen; - } - blkHead->dataLen = htonl(dataLen); - - ret->length += sizeof(SSubmitBlk) + schemaLen + dataLen; - blkHead = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk) + schemaLen + dataLen); - } - - ret->length = htonl(ret->length); - - taosArrayDestroyP(schemaReqs, taosMemoryFree); - taosArrayDestroy(schemaReqSz); - - return ret; -} void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { const SArray* pBlocks = (const SArray*)data; @@ -334,8 +106,8 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d int32_t code; tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); if (code < 0) { - // - ASSERT(0); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return; } SEncoder encoder; void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); @@ -493,7 +265,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d blkHead->uid = 0; blkHead->schemaLen = 0; - tqDebug("tq sink, convert block2 %d, rows: %d", i, rows); + tqDebug("tq sink pipe1, convert block2 %d, rows: %d", i, rows); int32_t dataLen = 0; void* blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); @@ -522,7 +294,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d } else { void* colData = colDataGetData(pColData, j); if (k == 0) { - tqDebug("tq sink, row %d ts %" PRId64, j, *(int64_t*)colData); + tqDebug("tq sink pipe1, row %d ts %" PRId64, j, *(int64_t*)colData); } tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, pColumn->offset, k); } @@ -551,60 +323,409 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d taosArrayDestroy(tagArray); } -#if 0 -void tqSinkToTableMerge(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { - const SArray* pRes = (const SArray*)data; - SVnode* pVnode = (SVnode*)vnode; - SBatchDeleteReq deleteReq = {0}; - - tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, (int32_t)pRes->size); +static int32_t encodeCreateChildTableForRPC(SVCreateTbBatchReq* pReqs, int32_t vgId, void** pBuf, int32_t* contLen) { + int32_t ret = 0; - ASSERT(pTask->tbSink.pTSchema); - deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); - SSubmitReq* submitReq = tqBlockToSubmit(pVnode, pRes, pTask->tbSink.pTSchema, pTask->tbSink.pSchemaWrapper, true, - pTask->tbSink.stbUid, pTask->tbSink.stbFullName, &deleteReq); + tEncodeSize(tEncodeSVCreateTbBatchReq, pReqs, *contLen, ret); + if (ret < 0) { + ret = -1; + goto end; + } + *contLen += sizeof(SMsgHead); + *pBuf = rpcMallocCont(*contLen); + if (NULL == *pBuf) { + ret = -1; + goto end; + } + ((SMsgHead*)(*pBuf))->vgId = vgId; + ((SMsgHead*)(*pBuf))->contLen = htonl(*contLen); + SEncoder coder = {0}; + tEncoderInit(&coder, POINTER_SHIFT(*pBuf, sizeof(SMsgHead)), (*contLen) - sizeof(SMsgHead) ); + if (tEncodeSVCreateTbBatchReq(&coder, pReqs) < 0) { + rpcFreeCont(*pBuf); + *pBuf = NULL; + *contLen = 0; + tEncoderClear(&coder); + ret = -1; + goto end; + } + tEncoderClear(&coder); - tqDebug("vgId:%d, task %d convert blocks over, put into write-queue", TD_VID(pVnode), pTask->taskId); +end: + return ret; +} - if (taosArrayGetSize(deleteReq.deleteReqs) != 0) { - int32_t code; - int32_t len; - tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); - if (code < 0) { - // - ASSERT(0); - } - SEncoder encoder; - void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); - void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); - tEncoderInit(&encoder, abuf, len); - tEncodeSBatchDeleteReq(&encoder, &deleteReq); - tEncoderClear(&encoder); - - ((SMsgHead*)serializedDeleteReq)->vgId = pVnode->config.vgId; - - SRpcMsg msg = { - .msgType = TDMT_VND_BATCH_DEL, - .pCont = serializedDeleteReq, - .contLen = len + sizeof(SMsgHead), - }; - if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { - rpcFreeCont(serializedDeleteReq); - tqDebug("failed to put into write-queue since %s", terrstr()); - } - } - taosArrayDestroy(deleteReq.deleteReqs); +int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) { + void* buf = NULL; + int32_t tlen = 0; + encodeCreateChildTableForRPC(pReqs, TD_VID(pVnode), &buf, &tlen); - /*tPrintFixedSchemaSubmitReq(pReq, pTask->tbSink.pTSchema);*/ - // build write msg SRpcMsg msg = { - .msgType = TDMT_VND_SUBMIT, - .pCont = submitReq, - .contLen = ntohl(submitReq->length), + .msgType = TDMT_VND_CREATE_TABLE, + .pCont = buf, + .contLen = tlen, }; if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { - tqDebug("failed to put into write-queue since %s", terrstr()); + tqError("failed to put into write-queue since %s", terrstr()); } + + return TSDB_CODE_SUCCESS; + +_error: + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("failed to encode submit req since %s", terrstr()); + return TSDB_CODE_OUT_OF_MEMORY; +} + +void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { + const SArray* pBlocks = (const SArray*)data; + SVnode* pVnode = (SVnode*)vnode; + int64_t suid = pTask->tbSink.stbUid; + char* stbFullName = pTask->tbSink.stbFullName; + STSchema* pTSchema = pTask->tbSink.pTSchema; + /*SSchemaWrapper* pSchemaWrapper = pTask->tbSink.pSchemaWrapper;*/ + + int32_t blockSz = taosArrayGetSize(pBlocks); + + tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz); + + void* pBuf = NULL; + SArray* tagArray = NULL; + SArray* pVals = NULL; + SArray* crTblArray = NULL; + + for (int32_t i = 0; i < blockSz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + int32_t rows = pDataBlock->info.rows; + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + SBatchDeleteReq deleteReq = {0}; + deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); + deleteReq.suid = suid; + tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, &deleteReq); + if (taosArrayGetSize(deleteReq.deleteReqs) == 0) { + taosArrayDestroy(deleteReq.deleteReqs); + continue; + } + + int32_t len; + int32_t code; + tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); + if (code < 0) { + // + ASSERT(0); + } + SEncoder encoder; + void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); + void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); + tEncoderInit(&encoder, abuf, len); + tEncodeSBatchDeleteReq(&encoder, &deleteReq); + tEncoderClear(&encoder); + taosArrayDestroy(deleteReq.deleteReqs); + + ((SMsgHead*)serializedDeleteReq)->vgId = pVnode->config.vgId; + + SRpcMsg msg = { + .msgType = TDMT_VND_BATCH_DEL, + .pCont = serializedDeleteReq, + .contLen = len + sizeof(SMsgHead), + }; + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + tqDebug("failed to put delete req into write-queue since %s", terrstr()); + } + } else if (pDataBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + SVCreateTbBatchReq reqs = {0}; + crTblArray = reqs.pArray = taosArrayInit(1, sizeof(struct SVCreateTbReq)); + if (NULL == reqs.pArray) { + goto _end; + } + for (int32_t rowId = 0; rowId < rows; rowId++) { + SVCreateTbReq createTbReq = {0}; + SVCreateTbReq* pCreateTbReq = &createTbReq; + if (!pCreateTbReq) { + goto _end; + } + + // set const + pCreateTbReq->flags = 0; + pCreateTbReq->type = TSDB_CHILD_TABLE; + pCreateTbReq->ctb.suid = suid; + + // set super table name + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + pCreateTbReq->ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName); + + // set tag content + int32_t size = taosArrayGetSize(pDataBlock->pDataBlock); + if (size == 2) { + tagArray = taosArrayInit(1, sizeof(STagVal)); + if (!tagArray) { + goto _end; + } + STagVal tagVal = { + .cid = pTSchema->numOfCols + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.id.groupId, + }; + taosArrayPush(tagArray, &tagVal); + + // set tag name + SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = "group_id"; + taosArrayPush(tagName, tagNameStr); + pCreateTbReq->ctb.tagName = tagName; + } else { + tagArray = taosArrayInit(size - 1, sizeof(STagVal)); + if (!tagArray) { + goto _end; + } + for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) { + SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId); + STagVal tagVal = { + .cid = pTSchema->numOfCols + step, + .type = pTagData->info.type, + }; + void* pData = colDataGetData(pTagData, rowId); + if (colDataIsNull_s(pTagData, rowId)) { + tagVal.type = TSDB_DATA_TYPE_NULL; + tagVal.pData = NULL; + tagVal.nData = 0; + } else if (IS_VAR_DATA_TYPE(pTagData->info.type)) { + tagVal.nData = varDataLen(pData); + tagVal.pData = varDataVal(pData); + } else { + memcpy(&tagVal.i64, pData, pTagData->info.bytes); + } + taosArrayPush(tagArray, &tagVal); + } + } + pCreateTbReq->ctb.tagNum = taosArrayGetSize(tagArray); + + STag* pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + tagArray = taosArrayDestroy(tagArray); + if (pTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + pCreateTbReq->ctb.pTag = (uint8_t*)pTag; + + // set table name + SColumnInfoData* pTbColInfo = taosArrayGet(pDataBlock->pDataBlock, UD_TABLE_NAME_COLUMN_INDEX); + if (colDataIsNull_s(pTbColInfo, rowId)) { + SColumnInfoData* pGpIdColInfo = taosArrayGet(pDataBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX); + void* pGpIdData = colDataGetData(pGpIdColInfo, rowId); + pCreateTbReq->name = buildCtbNameByGroupId(stbFullName, *(uint64_t*)pGpIdData); + } else { + void* pTbData = colDataGetData(pTbColInfo, rowId); + pCreateTbReq->name = taosMemoryCalloc(1, varDataLen(pTbData) + 1); + memcpy(pCreateTbReq->name, varDataVal(pTbData), varDataLen(pTbData)); + } + taosArrayPush(reqs.pArray, pCreateTbReq); + } + reqs.nReqs = taosArrayGetSize(reqs.pArray); + if (tqPutReqToQueue(pVnode, &reqs) != TSDB_CODE_SUCCESS) { + goto _end; + } + tagArray = taosArrayDestroy(tagArray); + taosArrayDestroyEx(crTblArray, (FDelete)tdDestroySVCreateTbReq); + crTblArray = NULL; + } else { + SSubmitTbData tbData = {0}; + tqDebug("tq sink pipe2, convert block1 %d, rows: %d", i, rows); + + if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow*)))) { + goto _end; + } + + tbData.suid = suid; + tbData.uid = 0; // uid is assigned by vnode + tbData.sver = pTSchema->version; + + char* ctbName = NULL; + tqDebug("vgId:%d, stream write into %s, table auto created", TD_VID(pVnode), pDataBlock->info.parTbName); + if (pDataBlock->info.parTbName[0]) { + ctbName = strdup(pDataBlock->info.parTbName); + } else { + ctbName = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId); + } + + SMetaReader mr = {0}; + metaReaderInit(&mr, pVnode->pMeta, 0); + if (metaGetTableEntryByName(&mr, ctbName) < 0) { + metaReaderClear(&mr); + tqDebug("vgId:%d, stream write into %s, table auto created", TD_VID(pVnode), ctbName); + + SVCreateTbReq* pCreateTbReq = NULL; + + if (!(pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq)))) { + goto _end; + }; + + // set const + pCreateTbReq->flags = 0; + pCreateTbReq->type = TSDB_CHILD_TABLE; + pCreateTbReq->ctb.suid = suid; + + // set super table name + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + pCreateTbReq->ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName); + + // set tag content + tagArray = taosArrayInit(1, sizeof(STagVal)); + if (!tagArray) { + goto _end; + } + STagVal tagVal = { + .cid = pTSchema->numOfCols + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.id.groupId, + }; + taosArrayPush(tagArray, &tagVal); + pCreateTbReq->ctb.tagNum = taosArrayGetSize(tagArray); + + STag* pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + tagArray = taosArrayDestroy(tagArray); + if (pTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + pCreateTbReq->ctb.pTag = (uint8_t*)pTag; + + // set tag name + SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = {0}; + strcpy(tagNameStr, "group_id"); + taosArrayPush(tagName, tagNameStr); + pCreateTbReq->ctb.tagName = tagName; + + // set table name + pCreateTbReq->name = ctbName; + ctbName = NULL; + + tbData.pCreateTbReq = pCreateTbReq; + tbData.flags = SUBMIT_REQ_AUTO_CREATE_TABLE; + } else { + if (mr.me.type != TSDB_CHILD_TABLE) { + tqError("vgId:%d, failed to write into %s, since table type incorrect, type %d", TD_VID(pVnode), ctbName, + mr.me.type); + metaReaderClear(&mr); + taosMemoryFree(ctbName); + continue; + } + + if (mr.me.ctbEntry.suid != suid) { + tqError("vgId:%d, failed to write into %s, since suid mismatch, expect suid: %" PRId64 + ", actual suid %" PRId64 "", + TD_VID(pVnode), ctbName, suid, mr.me.ctbEntry.suid); + metaReaderClear(&mr); + taosMemoryFree(ctbName); + } + + tbData.uid = mr.me.uid; + metaReaderClear(&mr); + taosMemoryFreeClear(ctbName); + } + + // rows + if (!pVals && !(pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)))) { + taosArrayDestroy(tbData.aRowP); + goto _end; + } + + for (int32_t j = 0; j < rows; j++) { + taosArrayClear(pVals); + int32_t dataIndex = 0; + for (int32_t k = 0; k < pTSchema->numOfCols; k++) { + const STColumn* pCol = &pTSchema->columns[k]; + if (k == 0) { + SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, dataIndex); + void* colData = colDataGetData(pColData, j); + tqDebug("tq sink pipe2, row %d, col %d ts %" PRId64, j, k, *(int64_t*)colData); + } + if (IS_SET_NULL(pCol)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else{ + SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, dataIndex); + if (colDataIsNull_s(pColData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + dataIndex++; + } else { + void* colData = colDataGetData(pColData, j); + if (IS_STR_DATA_TYPE(pCol->type)) { + SValue sv = + (SValue){.nData = varDataLen(colData), .pData = varDataVal(colData)}; // address copy, no value + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } else { + SValue sv; + memcpy(&sv.val, colData, tDataTypes[pCol->type].bytes); + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } + dataIndex++; + } + } + } + SRow* pRow = NULL; + if ((terrno = tRowBuild(pVals, (STSchema*)pTSchema, &pRow)) < 0) { + tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + goto _end; + } + ASSERT(pRow); + taosArrayPush(tbData.aRowP, &pRow); + } + + SSubmitReq2 submitReq = {0}; + if (!(submitReq.aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { + goto _end; + } + + taosArrayPush(submitReq.aSubmitTbData, &tbData); + + // encode + int32_t len; + int32_t code; + tEncodeSize(tEncodeSSubmitReq2, &submitReq, len, code); + SEncoder encoder; + len += sizeof(SMsgHead); + pBuf = rpcMallocCont(len); + if (NULL == pBuf) { + goto _end; + } + ((SMsgHead*)pBuf)->vgId = TD_VID(pVnode); + ((SMsgHead*)pBuf)->contLen = htonl(len); + tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SMsgHead)), len - sizeof(SMsgHead)); + if (tEncodeSSubmitReq2(&encoder, &submitReq) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("failed to encode submit req since %s", terrstr()); + tEncoderClear(&encoder); + rpcFreeCont(pBuf); + continue; + } + tEncoderClear(&encoder); + tDestroySSubmitReq2(&submitReq, TSDB_MSG_FLG_ENCODE); + + SRpcMsg msg = { + .msgType = TDMT_VND_SUBMIT, + .pCont = pBuf, + .contLen = len, + }; + + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + tqDebug("failed to put into write-queue since %s", terrstr()); + } + } + } +_end: + taosArrayDestroy(tagArray); + taosArrayDestroy(pVals); + taosArrayDestroyEx(crTblArray, (FDelete)tdDestroySVCreateTbReq); + // TODO: change } -#endif diff --git a/source/dnode/vnode/src/tq/tqSnapshot.c b/source/dnode/vnode/src/tq/tqSnapshot.c index 129a2dd8b309a78a1ef31bde0ab5367167999021..5c0649c10985b05dc1f5677fa9bfe1812dd07808 100644 --- a/source/dnode/vnode/src/tq/tqSnapshot.c +++ b/source/dnode/vnode/src/tq/tqSnapshot.c @@ -100,8 +100,6 @@ int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) { } } - ASSERT(pVal && vLen); - *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen); if (*ppData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c index b1f00bdf7446789fa3c572ff366bd20319db10ec..ab7093a701e0a5207b5f64072350ba5bc2bae703 100644 --- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c @@ -100,8 +100,6 @@ int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) { } } - ASSERT(pVal && vLen); - *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen); if (*ppData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -168,7 +166,6 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { if (rollback) { tdbAbort(pWriter->pTq->pMetaDB, pWriter->txn); - ASSERT(0); } else { code = tdbCommit(pWriter->pTq->pMetaDB, pWriter->txn); if (code) goto _err; diff --git a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c index 305378bc932f0484c71a9d1c91935a580189488e..ab7093a701e0a5207b5f64072350ba5bc2bae703 100644 --- a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c @@ -100,8 +100,6 @@ int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) { } } - ASSERT(pVal && vLen); - *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen); if (*ppData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -146,7 +144,7 @@ int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** p pWriter->sver = sver; pWriter->ever = ever; - if (tdbBegin(pTq->pMetaStore, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { + if (tdbBegin(pTq->pMetaDB, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { code = -1; taosMemoryFree(pWriter); goto _err; @@ -167,12 +165,11 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { STQ* pTq = pWriter->pTq; if (rollback) { - tdbAbort(pWriter->pTq->pMetaStore, pWriter->txn); - ASSERT(0); + tdbAbort(pWriter->pTq->pMetaDB, pWriter->txn); } else { - code = tdbCommit(pWriter->pTq->pMetaStore, pWriter->txn); + code = tdbCommit(pWriter->pTq->pMetaDB, pWriter->txn); if (code) goto _err; - code = tdbPostCommit(pWriter->pTq->pMetaStore, pWriter->txn); + code = tdbPostCommit(pWriter->pTq->pMetaDB, pWriter->txn); if (code) goto _err; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 0fc5b617bbb83e514ff8e00295d68295f70d1bb1..d18014c37b096f044a6cf4fbb7a933193e297d02 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -91,12 +91,10 @@ int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { } } + taosLRUCacheRelease(pCache, h, invalidate); if (invalidate) { - taosLRUCacheRelease(pCache, h, true); - } else { - taosLRUCacheRelease(pCache, h, false); + taosLRUCacheErase(pCache, key, keyLen); } - // void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen); } return code; @@ -124,12 +122,10 @@ int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { } } + taosLRUCacheRelease(pCache, h, invalidate); if (invalidate) { - taosLRUCacheRelease(pCache, h, true); - } else { - taosLRUCacheRelease(pCache, h, false); + taosLRUCacheErase(pCache, key, keyLen); } - // void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen); } return code; @@ -190,7 +186,7 @@ int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { return code; } -int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, STSRow *row, bool dup) { +int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, TSDBROW *row, bool dup) { int32_t code = 0; STSRow *cacheRow = NULL; char key[32] = {0}; @@ -201,13 +197,51 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); if (h) { STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); - TSKEY keyTs = row->ts; + TSKEY keyTs = TSDBROW_TS(row); bool invalidate = false; SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h); int16_t nCol = taosArrayGetSize(pLast); int16_t iCol = 0; + if (nCol <= 0) { + nCol = pTSchema->numOfCols; + + STColumn *pTColumn = &pTSchema->columns[0]; + SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = keyTs}); + if (taosArrayPush(pLast, &(SLastCol){.ts = keyTs, .colVal = tColVal}) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + + for (iCol = 1; iCol < nCol; ++iCol) { + SColVal colVal = {0}; + tsdbRowGetColVal(row, pTSchema, iCol, &colVal); + + SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; + if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { + SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); + taosMemoryFree(pLastCol->colVal.value.pData); + + lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); + if (lastCol.colVal.value.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); + } + + if (taosArrayPush(pLast, &lastCol) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + } + + goto _invalidate; + } + SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); if (keyTs > tTsVal->ts) { STColumn *pTColumn = &pTSchema->columns[0]; @@ -222,7 +256,7 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST SColVal *tColVal = &tTsVal1->colVal; SColVal colVal = {0}; - tTSRowGetVal(row, pTSchema, iCol, &colVal); + tsdbRowGetColVal(row, pTSchema, iCol, &colVal); if (!COL_VAL_IS_NONE(&colVal)) { if (keyTs == tTsVal1->ts && !COL_VAL_IS_NONE(tColVal)) { invalidate = true; @@ -261,7 +295,7 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST return code; } -int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb) { +int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, TSDBROW *row, STsdb *pTsdb) { int32_t code = 0; STSRow *cacheRow = NULL; char key[32] = {0}; @@ -272,13 +306,51 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); if (h) { STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); - TSKEY keyTs = row->ts; + TSKEY keyTs = TSDBROW_TS(row); bool invalidate = false; SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h); int16_t nCol = taosArrayGetSize(pLast); int16_t iCol = 0; + if (nCol <= 0) { + nCol = pTSchema->numOfCols; + + STColumn *pTColumn = &pTSchema->columns[0]; + SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = keyTs}); + if (taosArrayPush(pLast, &(SLastCol){.ts = keyTs, .colVal = tColVal}) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + + for (iCol = 1; iCol < nCol; ++iCol) { + SColVal colVal = {0}; + tsdbRowGetColVal(row, pTSchema, iCol, &colVal); + + SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; + if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { + SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); + taosMemoryFree(pLastCol->colVal.value.pData); + + lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); + if (lastCol.colVal.value.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); + } + + if (taosArrayPush(pLast, &lastCol) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + } + + goto _invalidate; + } + SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); if (keyTs > tTsVal->ts) { STColumn *pTColumn = &pTSchema->columns[0]; @@ -293,7 +365,7 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb SColVal *tColVal = &tTsVal1->colVal; SColVal colVal = {0}; - tTSRowGetVal(row, pTSchema, iCol, &colVal); + tsdbRowGetColVal(row, pTSchema, iCol, &colVal); if (COL_VAL_IS_VALUE(&colVal)) { if (keyTs == tTsVal1->ts && COL_VAL_IS_VALUE(tColVal)) { invalidate = true; @@ -617,7 +689,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { } else { // tBlockDataDestroy(&state->blockData, 1); if (state->pBlockData) { - tBlockDataDestroy(state->pBlockData, 1); + tBlockDataDestroy(state->pBlockData); state->pBlockData = NULL; } @@ -685,7 +757,8 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { tBlockDataReset(state->pBlockData); tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetDataBlk); - /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); */ + /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); + */ tBlockDataReset(state->pBlockData); TABLEID tid = {.suid = state->suid, .uid = state->uid}; code = tBlockDataInit(state->pBlockData, &tid, state->pTSchema, NULL, 0); @@ -739,7 +812,7 @@ _err: state->aBlockIdx = NULL; } if (state->pBlockData) { - tBlockDataDestroy(state->pBlockData, 1); + tBlockDataDestroy(state->pBlockData); state->pBlockData = NULL; } @@ -766,7 +839,7 @@ int32_t clearNextRowFromFS(void *iter) { } if (state->pBlockData) { // tBlockDataDestroy(&state->blockData, 1); - tBlockDataDestroy(state->pBlockData, 1); + tBlockDataDestroy(state->pBlockData); state->pBlockData = NULL; } @@ -1221,12 +1294,12 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo // build the result ts row here *dup = false; - if (taosArrayGetSize(pColArray) != nCol) { - *ppColArray = NULL; - taosArrayDestroy(pColArray); - } else { - *ppColArray = pColArray; - } + // if (taosArrayGetSize(pColArray) != nCol) { + //*ppColArray = NULL; + // taosArrayDestroy(pColArray); + //} else { + *ppColArray = pColArray; + //} nextRowIterClose(&iter); // taosMemoryFreeClear(pTSchema); @@ -1339,12 +1412,12 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach } } while (setNoneCol); - if (taosArrayGetSize(pColArray) <= 0) { - *ppLastArray = NULL; - taosArrayDestroy(pColArray); - } else { - *ppLastArray = pColArray; - } + // if (taosArrayGetSize(pColArray) <= 0) { + //*ppLastArray = NULL; + // taosArrayDestroy(pColArray); + //} else { + *ppLastArray = pColArray; + //} nextRowIterClose(&iter); // taosMemoryFreeClear(pTSchema); @@ -1375,8 +1448,8 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader * SArray *pArray = NULL; bool dup = false; // which is always false for now code = mergeLastRow(uid, pTsdb, &dup, &pArray, pr); - // if table's empty or error, return code of -1 - if (code < 0 || pArray == NULL) { + // if table's empty or error, set handle NULL and return + if (code < 0 /* || pArray == NULL*/) { if (!dup && pArray) { taosArrayDestroy(pArray); } @@ -1394,13 +1467,8 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader * if (status != TAOS_LRU_STATUS_OK) { code = -1; } - - // taosThreadMutexUnlock(&pTsdb->lruMutex); - - // h = taosLRUCacheLookup(pCache, key, keyLen); - } // else { + } taosThreadMutexUnlock(&pTsdb->lruMutex); - //} } *handle = h; @@ -1424,8 +1492,8 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, if (!h) { SArray *pLastArray = NULL; code = mergeLast(uid, pTsdb, &pLastArray, pr); - // if table's empty or error, return code of -1 - if (code < 0 || pLastArray == NULL) { + // if table's empty or error, set handle NULL and return + if (code < 0 /* || pLastArray == NULL*/) { taosThreadMutexUnlock(&pTsdb->lruMutex); *handle = NULL; @@ -1439,13 +1507,8 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, if (status != TAOS_LRU_STATUS_OK) { code = -1; } - - // taosThreadMutexUnlock(&pTsdb->lruMutex); - - // h = taosLRUCacheLookup(pCache, key, keyLen); - } // else { + } taosThreadMutexUnlock(&pTsdb->lruMutex); - //} } *handle = h; diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index f05f5d5c88f7ef19c8598410019326edc9b1c5c6..6a5acecfc3226753108863e3537f281028c8a34f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -109,6 +109,8 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, p->type = type; p->pVnode = pVnode; + p->pTsdb = p->pVnode->pTsdb; + p->verRange = (SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}; p->numOfCols = numOfCols; p->suid = suid; @@ -145,6 +147,7 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, } p->idstr = taosMemoryStrDup(idstr); + taosThreadMutexInit(&p->readerMutex, NULL); *pReader = p; return TSDB_CODE_SUCCESS; @@ -164,7 +167,9 @@ void* tsdbCacherowsReaderClose(void* pReader) { destroyLastBlockLoadInfo(p->pLoadInfo); - taosMemoryFree((void*) p->idstr); + taosMemoryFree((void*)p->idstr); + taosThreadMutexDestroy(&p->readerMutex); + taosMemoryFree(pReader); return NULL; } @@ -199,6 +204,27 @@ static void freeItem(void* pItem) { } } +static int32_t tsdbCacheQueryReseek(void* pQHandle) { + int32_t code = 0; + SCacheRowsReader* pReader = pQHandle; + + code = taosThreadMutexTryLock(&pReader->readerMutex); + if (code == 0) { + // pause current reader's state if not paused, save ts & version for resuming + // just wait for the big all tables' snapshot untaking for now + + code = TSDB_CODE_VND_QUERY_BUSY; + + taosThreadMutexUnlock(&pReader->readerMutex); + + return code; + } else if (code == EBUSY) { + return TSDB_CODE_VND_QUERY_BUSY; + } else { + return -1; + } +} + int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32_t* slotIds, SArray* pTableUidList) { if (pReader == NULL || pResBlock == NULL) { return TSDB_CODE_INVALID_PARA; @@ -241,7 +267,8 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 taosArrayPush(pLastCols, &p); } - tsdbTakeReadSnap(pr->pVnode->pTsdb, &pr->pReadSnap, "cache-l"); + taosThreadMutexLock(&pr->readerMutex); + tsdbTakeReadSnap((STsdbReader*)pr, tsdbCacheQueryReseek, &pr->pReadSnap); pr->pDataFReader = NULL; pr->pDataFReaderLast = NULL; @@ -258,6 +285,10 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 if (h == NULL) { continue; } + if (taosArrayGetSize(pRow) <= 0) { + tsdbCacheRelease(lruCache, h); + continue; + } { for (int32_t k = 0; k < pr->numOfCols; ++k) { @@ -327,6 +358,10 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 if (h == NULL) { continue; } + if (taosArrayGetSize(pRow) <= 0) { + tsdbCacheRelease(lruCache, h); + continue; + } saveOneRow(pRow, pResBlock, pr, slotIds, pRes, pr->idstr); // TODO reset the pRes @@ -347,8 +382,9 @@ _end: tsdbDataFReaderClose(&pr->pDataFReaderLast); tsdbDataFReaderClose(&pr->pDataFReader); - tsdbUntakeReadSnap(pr->pVnode->pTsdb, pr->pReadSnap, "cache-l"); resetLastBlockLoadInfo(pr->pLoadInfo); + tsdbUntakeReadSnap((STsdbReader*)pr, pr->pReadSnap, true); + taosThreadMutexUnlock(&pr->readerMutex); for (int32_t j = 0; j < pr->numOfCols; ++j) { taosMemoryFree(pRes[j]); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 8ec59ea95949a4bccda38c7cc2fda25e2f769fb4..d15f848cfdcde11c946a943d96b0397933324b14 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -175,7 +175,7 @@ int32_t tsdbCommit(STsdb *pTsdb, SCommitInfo *pInfo) { pTsdb->imem = NULL; taosThreadRwlockUnlock(&pTsdb->rwLock); - tsdbUnrefMemTable(pMemTable); + tsdbUnrefMemTable(pMemTable, NULL, true); goto _exit; } @@ -913,24 +913,24 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { // reader taosArrayDestroy(pCommitter->dReader.aBlockIdx); tMapDataClear(&pCommitter->dReader.mBlock); - tBlockDataDestroy(&pCommitter->dReader.bData, 1); + tBlockDataDestroy(&pCommitter->dReader.bData); // merger for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) { SDataIter *pIter = &pCommitter->aDataIter[iStt]; taosArrayDestroy(pIter->aSttBlk); - tBlockDataDestroy(&pIter->bData, 1); + tBlockDataDestroy(&pIter->bData); } // writer taosArrayDestroy(pCommitter->dWriter.aBlockIdx); taosArrayDestroy(pCommitter->dWriter.aSttBlk); tMapDataClear(&pCommitter->dWriter.mBlock); - tBlockDataDestroy(&pCommitter->dWriter.bData, 1); + tBlockDataDestroy(&pCommitter->dWriter.bData); #if USE_STREAM_COMPRESSION tDiskDataBuilderDestroy(pCommitter->dWriter.pBuilder); #else - tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1); + tBlockDataDestroy(&pCommitter->dWriter.bDatal); #endif tDestroyTSchema(pCommitter->skmTable.pTSchema); tDestroyTSchema(pCommitter->skmRow.pTSchema); @@ -1183,7 +1183,6 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) tBlockDataClear(pBlockData); while (pRowInfo) { - ASSERT(pRowInfo->row.type == 0); code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); TSDB_CHECK_CODE(code, lino, _exit); @@ -1251,7 +1250,6 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) pRow = NULL; } } else if (c > 0) { - ASSERT(pRowInfo->row.type == 0); code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); TSDB_CHECK_CODE(code, lino, _exit); @@ -1493,7 +1491,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { while (pRowInfo) { STSchema *pTSchema = NULL; - if (pRowInfo->row.type == 0) { + if (pRowInfo->row.type == TSDBROW_ROW_FMT) { code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); TSDB_CHECK_CODE(code, lino, _exit); pTSchema = pCommitter->skmRow.pTSchema; @@ -1536,7 +1534,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { while (pRowInfo) { STSchema *pTSchema = NULL; - if (pRowInfo->row.type == 0) { + if (pRowInfo->row.type == TSDBROW_ROW_FMT) { code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); TSDB_CHECK_CODE(code, lino, _exit); pTSchema = pCommitter->skmRow.pTSchema; @@ -1666,7 +1664,7 @@ int32_t tsdbFinishCommit(STsdb *pTsdb) { // unlock taosThreadRwlockUnlock(&pTsdb->rwLock); if (pMemTable) { - tsdbUnrefMemTable(pMemTable); + tsdbUnrefMemTable(pMemTable, NULL, true); } _exit: diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index b46a00363817666cdff7bc7756ff204483787c36..ae9af11f5ae8226c1fceefb57b4d83167800772a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -596,7 +596,7 @@ int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTS if (pBuilder->bi.maxKey < kRow.ts) pBuilder->bi.maxKey = kRow.ts; STSDBRowIter iter = {0}; - tsdbRowIterInit(&iter, pRow, pTSchema); + tsdbRowIterOpen(&iter, pRow, pTSchema); SColVal *pColVal = tsdbRowIterNext(&iter); for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) { diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index ac27a0309610338f5da22afbc3941add7bf57c71..d68afbed1f0e4a446e64b1aabe36f7b6bdb0a4fc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -32,8 +32,10 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags); static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData); -static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, int64_t version, - SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubmitBlkRsp *pRsp); +static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version, + SSubmitTbData *pSubmitTbData, int32_t *affectedRows); +static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version, + SSubmitTbData *pSubmitTbData, int32_t *affectedRows); int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { int32_t code = 0; @@ -48,6 +50,8 @@ int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { pMemTable->pTsdb = pTsdb; pMemTable->pPool = pTsdb->pVnode->inUse; pMemTable->nRef = 1; + pMemTable->minVer = VERSION_MAX; + pMemTable->maxVer = VERSION_MIN; pMemTable->minKey = TSKEY_MAX; pMemTable->maxKey = TSKEY_MIN; pMemTable->nRow = 0; @@ -60,6 +64,8 @@ int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { taosMemoryFree(pMemTable); goto _err; } + // pMemTable->qList.pNext = &pMemTable->qList; + // pMemTable->qList.ppNext = &pMemTable->qList.pNext; vnodeBufPoolRef(pMemTable->pPool); *ppMemTable = pMemTable; @@ -70,9 +76,9 @@ _err: return code; } -void tsdbMemTableDestroy(SMemTable *pMemTable) { +void tsdbMemTableDestroy(SMemTable *pMemTable, bool proactive) { if (pMemTable) { - vnodeBufPoolUnRef(pMemTable->pPool); + vnodeBufPoolUnRef(pMemTable->pPool, proactive); taosMemoryFree(pMemTable->aBucket); taosMemoryFree(pMemTable); } @@ -99,14 +105,14 @@ STbData *tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t return pTbData; } -int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, - SSubmitBlkRsp *pRsp) { +int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitTbData *pSubmitTbData, int32_t *affectedRows) { int32_t code = 0; SMemTable *pMemTable = pTsdb->mem; STbData *pTbData = NULL; - tb_uid_t suid = pMsgIter->suid; - tb_uid_t uid = pMsgIter->uid; + tb_uid_t suid = pSubmitTbData->suid; + tb_uid_t uid = pSubmitTbData->uid; +#if 0 SMetaInfo info; code = metaGetInfo(pTsdb->pVnode->pMeta, uid, &info, NULL); if (code) { @@ -120,14 +126,15 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI if (info.suid) { metaGetInfo(pTsdb->pVnode->pMeta, info.suid, &info, NULL); } - if (pMsgIter->sversion != info.skmVer) { + if (pSubmitTbData->sver != info.skmVer) { tsdbError("vgId:%d, req sver:%d, skmVer:%d suid:%" PRId64 " uid:%" PRId64, TD_VID(pTsdb->pVnode), - pMsgIter->sversion, info.skmVer, suid, uid); + pSubmitTbData->sver, info.skmVer, suid, uid); code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; goto _err; } - pRsp->sver = info.skmVer; + if (pRsp) pRsp->sver = info.skmVer; +#endif // create/get STbData to op code = tsdbGetOrCreateTbData(pMemTable, suid, uid, &pTbData); @@ -136,10 +143,16 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI } // do insert impl - code = tsdbInsertTableDataImpl(pMemTable, pTbData, version, pMsgIter, pBlock, pRsp); - if (code) { - goto _err; + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + code = tsdbInsertColDataToTable(pMemTable, pTbData, version, pSubmitTbData, affectedRows); + } else { + code = tsdbInsertRowDataToTable(pMemTable, pTbData, version, pSubmitTbData, affectedRows); } + if (code) goto _err; + + // update + pMemTable->minVer = TMIN(pMemTable->minVer, version); + pMemTable->maxVer = TMAX(pMemTable->maxVer, version); return code; @@ -192,6 +205,8 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid } pMemTable->nDel++; + pMemTable->minVer = TMIN(pMemTable->minVer, version); + pMemTable->maxVer = TMIN(pMemTable->maxVer, version); if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config) && tsdbKeyCmprFn(&lastKey, &pTbData->maxKey) >= 0) { tsdbCacheDeleteLastrow(pTsdb->lruCache, pTbData->uid, eKey); @@ -232,7 +247,6 @@ void *tsdbTbDataIterDestroy(STbDataIter *pIter) { if (pIter) { taosMemoryFree(pIter); } - return NULL; } @@ -246,7 +260,6 @@ void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDa pIter->pTbData = pTbData; pIter->backward = backward; pIter->pRow = NULL; - pIter->row.type = 0; if (pFrom == NULL) { // create from head or tail if (backward) { @@ -414,8 +427,13 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p for (int8_t iLevel = pTbData->sl.level - 1; iLevel >= 0; iLevel--) { pn = SL_GET_NODE_BACKWARD(px, iLevel); while (pn != pTbData->sl.pHead) { - tKey.version = pn->version; - tKey.ts = pn->pTSRow->ts; + if (pn->flag == TSDBROW_ROW_FMT) { + tKey.version = pn->version; + tKey.ts = ((SRow *)pn->pData)->ts; + } else if (pn->flag == TSDBROW_COL_FMT) { + tKey.version = ((SBlockData *)pn->pData)->aVersion[pn->iRow]; + tKey.ts = ((SBlockData *)pn->pData)->aTSKEY[pn->iRow]; + } int32_t c = tsdbKeyCmprFn(&tKey, pKey); if (c <= 0) { @@ -444,8 +462,13 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p for (int8_t iLevel = pTbData->sl.level - 1; iLevel >= 0; iLevel--) { pn = SL_GET_NODE_FORWARD(px, iLevel); while (pn != pTbData->sl.pTail) { - tKey.version = pn->version; - tKey.ts = pn->pTSRow->ts; + if (pn->flag == TSDBROW_ROW_FMT) { + tKey.version = pn->version; + tKey.ts = ((SRow *)pn->pData)->ts; + } else if (pn->flag == TSDBROW_COL_FMT) { + tKey.version = ((SBlockData *)pn->pData)->aVersion[pn->iRow]; + tKey.ts = ((SBlockData *)pn->pData)->aTSKEY[pn->iRow]; + } int32_t c = tsdbKeyCmprFn(&tKey, pKey); if (c >= 0) { @@ -472,8 +495,8 @@ static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) { return level; } -static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListNode **pos, int64_t version, - STSRow *pRow, int8_t forward) { +static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListNode **pos, TSDBROW *pRow, + int8_t forward) { int32_t code = 0; int8_t level; SMemSkipListNode *pNode; @@ -483,15 +506,30 @@ static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListN // create node level = tsdbMemSkipListRandLevel(&pTbData->sl); nSize = SL_NODE_SIZE(level); - pNode = (SMemSkipListNode *)vnodeBufPoolMallocAligned(pPool, nSize + pRow->len); + if (pRow->type == TSDBROW_ROW_FMT) { + pNode = (SMemSkipListNode *)vnodeBufPoolMallocAligned(pPool, nSize + pRow->pTSRow->len); + } else if (pRow->type == TSDBROW_COL_FMT) { + pNode = (SMemSkipListNode *)vnodeBufPoolMallocAligned(pPool, nSize); + } else { + ASSERT(0); + } if (pNode == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } + pNode->level = level; - pNode->version = version; - pNode->pTSRow = (STSRow *)((char *)pNode + nSize); - memcpy(pNode->pTSRow, pRow, pRow->len); + pNode->flag = pRow->type; + if (pRow->type == TSDBROW_ROW_FMT) { + pNode->version = pRow->version; + pNode->pData = (char *)pNode + nSize; + memcpy(pNode->pData, pRow->pTSRow, pRow->pTSRow->len); + } else if (pRow->type == TSDBROW_COL_FMT) { + pNode->iRow = pRow->iRow; + pNode->pData = pRow->pBlockData; + } else { + ASSERT(0); + } // set node if (forward) { @@ -536,69 +574,166 @@ _exit: return code; } -static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, int64_t version, - SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubmitBlkRsp *pRsp) { - int32_t code = 0; - SSubmitBlkIter blkIter = {0}; - TSDBKEY key = {.version = version}; - SMemSkipListNode *pos[SL_MAX_LEVEL]; - TSDBROW row = tsdbRowFromTSRow(version, NULL); - int32_t nRow = 0; - STSRow *pLastRow = NULL; +static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version, + SSubmitTbData *pSubmitTbData, int32_t *affectedRows) { + int32_t code = 0; - tInitSubmitBlkIter(pMsgIter, pBlock, &blkIter); + SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; + int32_t nColData = TARRAY_SIZE(pSubmitTbData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pSubmitTbData->aCol); - // backward put first data - row.pTSRow = tGetSubmitBlkNext(&blkIter); - if (row.pTSRow == NULL) return code; + ASSERT(aColData[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); + ASSERT(aColData[0].type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(aColData[0].flag == HAS_VALUE); - key.ts = row.pTSRow->ts; - nRow++; - tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD); - code = tbDataDoPut(pMemTable, pTbData, pos, version, row.pTSRow, 0); - if (code) { - goto _err; + // copy and construct block data + SBlockData *pBlockData = vnodeBufPoolMalloc(pPool, sizeof(*pBlockData)); + if (pBlockData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; } + pBlockData->suid = pTbData->suid; + pBlockData->uid = pTbData->uid; + pBlockData->nRow = aColData[0].nVal; + pBlockData->aUid = NULL; + pBlockData->aVersion = vnodeBufPoolMalloc(pPool, aColData[0].nData); + if (pBlockData->aVersion == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + for (int32_t i = 0; i < pBlockData->nRow; i++) { // todo: here can be optimized + pBlockData->aVersion[i] = version; + } + + pBlockData->aTSKEY = vnodeBufPoolMalloc(pPool, aColData[0].nData); + if (pBlockData->aTSKEY == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(pBlockData->aTSKEY, aColData[0].pData, aColData[0].nData); + + pBlockData->nColData = nColData - 1; + pBlockData->aColData = vnodeBufPoolMalloc(pPool, sizeof(SColData) * pBlockData->nColData); + if (pBlockData->aColData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + for (int32_t iColData = 0; iColData < pBlockData->nColData; ++iColData) { + code = tColDataCopy(&aColData[iColData + 1], &pBlockData->aColData[iColData], (xMallocFn)vnodeBufPoolMalloc, pPool); + if (code) goto _exit; + } + + // loop to add each row to the skiplist + SMemSkipListNode *pos[SL_MAX_LEVEL]; + TSDBROW tRow = tsdbRowFromBlockData(pBlockData, 0); + TSDBKEY key = {.version = version, .ts = pBlockData->aTSKEY[0]}; + TSDBROW lRow; // last row + + // first row + tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD); + if ((code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 0))) goto _exit; pTbData->minKey = TMIN(pTbData->minKey, key.ts); + lRow = tRow; - pLastRow = row.pTSRow; + // remain row + ++tRow.iRow; + if (tRow.iRow < pBlockData->nRow) { + for (int8_t iLevel = pos[0]->level; iLevel < pTbData->sl.maxLevel; iLevel++) { + pos[iLevel] = SL_NODE_BACKWARD(pos[iLevel], iLevel); + } + + while (tRow.iRow < pBlockData->nRow) { + key.ts = pBlockData->aTSKEY[tRow.iRow]; + + if (SL_NODE_FORWARD(pos[0], 0) != pTbData->sl.pTail) { + tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_FROM_POS); + } + + if ((code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 1))) goto _exit; + lRow = tRow; + + ++tRow.iRow; + } + } + + if (key.ts >= pTbData->maxKey) { + pTbData->maxKey = key.ts; + + if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config)) { + tsdbCacheInsertLastrow(pMemTable->pTsdb->lruCache, pMemTable->pTsdb, pTbData->uid, &lRow, true); + } + } + + if (TSDB_CACHE_LAST(pMemTable->pTsdb->pVnode->config)) { + tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, &lRow, pMemTable->pTsdb); + } + + // SMemTable + pMemTable->minKey = TMIN(pMemTable->minKey, pTbData->minKey); + pMemTable->maxKey = TMAX(pMemTable->maxKey, pTbData->maxKey); + pMemTable->nRow += pBlockData->nRow; + + if (affectedRows) *affectedRows = pBlockData->nRow; + +_exit: + return code; +} + +static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version, + SSubmitTbData *pSubmitTbData, int32_t *affectedRows) { + int32_t code = 0; + + int32_t nRow = TARRAY_SIZE(pSubmitTbData->aRowP); + SRow **aRow = (SRow **)TARRAY_DATA(pSubmitTbData->aRowP); + TSDBKEY key = {.version = version}; + SMemSkipListNode *pos[SL_MAX_LEVEL]; + TSDBROW tRow = {.type = TSDBROW_ROW_FMT, .version = version}; + int32_t iRow = 0; + TSDBROW lRow; + + // backward put first data + tRow.pTSRow = aRow[iRow++]; + key.ts = tRow.pTSRow->ts; + tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD); + code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 0); + if (code) goto _exit; + lRow = tRow; + + pTbData->minKey = TMIN(pTbData->minKey, key.ts); // forward put rest data - row.pTSRow = tGetSubmitBlkNext(&blkIter); - if (row.pTSRow) { + if (iRow < nRow) { for (int8_t iLevel = pos[0]->level; iLevel < pTbData->sl.maxLevel; iLevel++) { pos[iLevel] = SL_NODE_BACKWARD(pos[iLevel], iLevel); } - do { - key.ts = row.pTSRow->ts; - nRow++; + + while (iRow < nRow) { + tRow.pTSRow = aRow[iRow]; + key.ts = tRow.pTSRow->ts; + if (SL_NODE_FORWARD(pos[0], 0) != pTbData->sl.pTail) { tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_FROM_POS); } - code = tbDataDoPut(pMemTable, pTbData, pos, version, row.pTSRow, 1); - if (code) { - goto _err; - } - pLastRow = row.pTSRow; + code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 1); + if (code) goto _exit; + + lRow = tRow; - row.pTSRow = tGetSubmitBlkNext(&blkIter); - } while (row.pTSRow); + iRow++; + } } if (key.ts >= pTbData->maxKey) { - if (key.ts > pTbData->maxKey) { - pTbData->maxKey = key.ts; - } + pTbData->maxKey = key.ts; - if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config) && pLastRow != NULL) { - tsdbCacheInsertLastrow(pMemTable->pTsdb->lruCache, pMemTable->pTsdb, pTbData->uid, pLastRow, true); + if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config)) { + tsdbCacheInsertLastrow(pMemTable->pTsdb->lruCache, pMemTable->pTsdb, pTbData->uid, &lRow, true); } } if (TSDB_CACHE_LAST(pMemTable->pTsdb->pVnode->config)) { - tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, pLastRow, pMemTable->pTsdb); + tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, &lRow, pMemTable->pTsdb); } // SMemTable @@ -606,27 +741,38 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i pMemTable->maxKey = TMAX(pMemTable->maxKey, pTbData->maxKey); pMemTable->nRow += nRow; - pRsp->numOfRows = nRow; - pRsp->affectedRows = nRow; - - return code; + if (affectedRows) *affectedRows = nRow; -_err: +_exit: return code; } int32_t tsdbGetNRowsInTbData(STbData *pTbData) { return pTbData->sl.size; } -void tsdbRefMemTable(SMemTable *pMemTable) { +int32_t tsdbRefMemTable(SMemTable *pMemTable, SQueryNode *pQNode) { + int32_t code = 0; + int32_t nRef = atomic_fetch_add_32(&pMemTable->nRef, 1); ASSERT(nRef > 0); + + vnodeBufPoolRegisterQuery(pMemTable->pPool, pQNode); + +_exit: + return code; } -void tsdbUnrefMemTable(SMemTable *pMemTable) { - int32_t nRef = atomic_sub_fetch_32(&pMemTable->nRef, 1); - if (nRef == 0) { - tsdbMemTableDestroy(pMemTable); +int32_t tsdbUnrefMemTable(SMemTable *pMemTable, SQueryNode *pNode, bool proactive) { + int32_t code = 0; + + if (pNode) { + vnodeBufPoolDeregisterQuery(pMemTable->pPool, pNode, proactive); + } + + if (atomic_sub_fetch_32(&pMemTable->nRef, 1) == 0) { + tsdbMemTableDestroy(pMemTable, proactive); } + + return code; } static FORCE_INLINE int32_t tbDataPCmprFn(const void *p1, const void *p2) { @@ -666,3 +812,29 @@ SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable) { _exit: return aTbDataP; } + +// int32_t tsdbRecycleMemTable(SMemTable *pMemTable) { +// int32_t code = 0; + +// SQueryNode *pNode = pMemTable->qList.pNext; +// while (1) { +// ASSERT(pNode != &pMemTable->qList); +// SQueryNode *pNextNode = pNode->pNext; + +// if (pNextNode == &pMemTable->qList) { +// code = (*pNode->reseek)(pNode->pQHandle); +// if (code) goto _exit; +// break; +// } else { +// code = (*pNode->reseek)(pNode->pQHandle); +// if (code) goto _exit; +// pNode = pMemTable->qList.pNext; +// ASSERT(pNode == pNextNode); +// } +// } + +// // NOTE: Take care here, pMemTable is destroyed + +// _exit: +// return code; +// } diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index af1a42d018919b81dac90a8a27124f5eb9a15b65..1f99444cf3cc655be656ce86a70ea22920c26fe8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -89,8 +89,8 @@ void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { pLoadInfo[i].blockIndex[0] = -1; pLoadInfo[i].blockIndex[1] = -1; - tBlockDataDestroy(&pLoadInfo[i].blockData[0], true); - tBlockDataDestroy(&pLoadInfo[i].blockData[1], true); + tBlockDataDestroy(&pLoadInfo[i].blockData[0]); + tBlockDataDestroy(&pLoadInfo[i].blockData[1]); taosArrayDestroy(pLoadInfo[i].aSttBlk); } diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index f71b5b6706ccb152f4dc221d01f4a1ea6e207c7a..8901f644598ec4ca5343f4a35a7b063bf39096fd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -88,7 +88,7 @@ _err: int tsdbClose(STsdb **pTsdb) { if (*pTsdb) { taosThreadRwlockWrlock(&(*pTsdb)->rwLock); - tsdbMemTableDestroy((*pTsdb)->mem); + tsdbMemTableDestroy((*pTsdb)->mem, true); (*pTsdb)->mem = NULL; taosThreadRwlockUnlock(&(*pTsdb)->rwLock); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b21050b2ae7d31b3a44d6fb13f5d5d84d5508a3a..d8c379f476943d64de5c93d9643b04ebfa832322 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -82,13 +82,13 @@ typedef struct SIOCostSummary { } SIOCostSummary; typedef struct SBlockLoadSuppInfo { - SArray* pColAgg; - SColumnDataAgg tsColAgg; - int16_t* colId; - int16_t* slotId; - int32_t numOfCols; - char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. - bool smaValid; // the sma on all queried columns are activated + SArray* pColAgg; + SColumnDataAgg tsColAgg; + int16_t* colId; + int16_t* slotId; + int32_t numOfCols; + char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. + bool smaValid; // the sma on all queried columns are activated } SBlockLoadSuppInfo; typedef struct SLastBlockReader { @@ -156,6 +156,9 @@ typedef struct SBlockInfoBuf { struct STsdbReader { STsdb* pTsdb; + SVersionRange verRange; + TdThreadMutex readerMutex; + bool suspended; uint64_t suid; int16_t order; bool freeBlock; @@ -168,15 +171,15 @@ struct STsdbReader { SBlockLoadSuppInfo suppInfo; STsdbReadSnap* pReadSnap; SIOCostSummary cost; - STSchema* pSchema; // the newest version schema - STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times - SDataFReader* pFileReader; // the file reader - SDelFReader* pDelFReader; // the del file reader - SArray* pDelIdx; // del file block index; - SVersionRange verRange; - SBlockInfoBuf blockInfoBuf; - int32_t step; - STsdbReader* innerReader[2]; + STSchema* pSchema; // the newest version schema + STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times + SDataFReader* pFileReader; // the file reader + SDelFReader* pDelFReader; // the del file reader + SArray* pDelIdx; // del file block index; + // SVersionRange verRange; + SBlockInfoBuf blockInfoBuf; + int32_t step; + STsdbReader* innerReader[2]; }; static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter); @@ -189,8 +192,8 @@ static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STabl SRowMerger* pMerger, SVersionRange* pVerRange); static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, STsdbReader* pReader); -static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, - STableBlockScanInfo* pScanInfo); +static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, + STableBlockScanInfo* pInfo); static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex); static void setComposedBlockFlag(STsdbReader* pReader, bool composed); @@ -198,9 +201,9 @@ static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* SVersionRange* pVerRange); static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, - STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow); + TSDBROW* pTSRow, STsdbReader* pReader, bool* freeTSRow); static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, - STsdbReader* pReader, STSRow** pTSRow); + STsdbReader* pReader, SRow** pTSRow); static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader); @@ -218,17 +221,18 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBl static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } -static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, int32_t numOfCols) { +static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, + int32_t numOfCols) { pSupInfo->smaValid = true; pSupInfo->numOfCols = numOfCols; - pSupInfo->colId = taosMemoryMalloc(numOfCols * (sizeof(int16_t)*2 + POINTER_BYTES)); + pSupInfo->colId = taosMemoryMalloc(numOfCols * (sizeof(int16_t) * 2 + POINTER_BYTES)); if (pSupInfo->colId == NULL) { taosMemoryFree(pSupInfo->colId); return TSDB_CODE_OUT_OF_MEMORY; } pSupInfo->slotId = (int16_t*)((char*)pSupInfo->colId + (sizeof(int16_t) * numOfCols)); - pSupInfo->buildBuf = (char**) ((char*)pSupInfo->slotId + (sizeof(int16_t) * numOfCols)); + pSupInfo->buildBuf = (char**)((char*)pSupInfo->slotId + (sizeof(int16_t) * numOfCols)); for (int32_t i = 0; i < numOfCols; ++i) { pSupInfo->colId[i] = pCols[i].colId; pSupInfo->slotId[i] = pSlotIdList[i]; @@ -246,7 +250,7 @@ static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pC static int32_t updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) { int32_t i = 0, j = 0; - while(i < pSchema->numOfCols && j < pSupInfo->numOfCols) { + while (i < pSchema->numOfCols && j < pSupInfo->numOfCols) { STColumn* pTCol = &pSchema->columns[i]; if (pTCol->colId == pSupInfo->colId[j]) { if (!IS_BSMA_ON(pTCol)) { @@ -311,7 +315,8 @@ static void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index) { } // NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model -static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, int32_t numOfTables) { +static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, + int32_t numOfTables) { // allocate buffer in order to load data blocks from file // todo use simple hash instead, optimize the memory consumption SHashObj* pTableMap = @@ -397,9 +402,7 @@ static void destroyAllBlockScanInfo(SHashObj* pTableMap) { taosHashCleanup(pTableMap); } -static bool isEmptyQueryTimeWindow(STimeWindow* pWindow) { - return pWindow->skey > pWindow->ekey; -} +static bool isEmptyQueryTimeWindow(STimeWindow* pWindow) { return pWindow->skey > pWindow->ekey; } // Update the query time window according to the data time to live(TTL) information, in order to avoid to return // the expired data to client, even it is queried already. @@ -633,6 +636,8 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd setColumnIdSlotList(&pReader->suppInfo, pCond->colList, pCond->pSlotList, pCond->numOfCols); + taosThreadMutexInit(&pReader->readerMutex, NULL); + *ppReader = pReader; return code; @@ -727,12 +732,25 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN sizeInDisk += pScanInfo->mapData.nData; + int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; + STimeWindow w = pReader->window; + if (ASCENDING_TRAVERSE(pReader->order)) { + w.skey = pScanInfo->lastKey + step; + } else { + w.ekey = pScanInfo->lastKey + step; + } + + if (isEmptyQueryTimeWindow(&w)) { + continue; + } + SDataBlk block = {0}; for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) { tGetDataBlk(pScanInfo->mapData.pData + pScanInfo->mapData.aOffset[j], &block); // 1. time range check - if (block.minKey.ts > pReader->window.ekey || block.maxKey.ts < pReader->window.skey) { + // if (block.minKey.ts > pReader->window.ekey || block.maxKey.ts < pReader->window.skey) { + if (block.minKey.ts > w.ekey || block.maxKey.ts < w.skey) { continue; } @@ -768,7 +786,6 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastFiles, sizeInDisk / 1000.0, el, pReader->idStr); - pReader->cost.numOfBlocks += total; pReader->cost.headFileLoadTime += el; @@ -902,7 +919,7 @@ static void copyPrimaryTsCol(const SBlockData* pBlockData, SFileBlockDumpInfo* p // a faster version of copy procedure. static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData, - int32_t dumpedRows, bool asc) { + int32_t dumpedRows, bool asc) { uint8_t* p = NULL; if (asc) { p = pData->pData + tDataTypes[pData->type].bytes * pDumpInfo->rowIndex; @@ -911,22 +928,21 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo p = pData->pData + tDataTypes[pData->type].bytes * startIndex; } - int32_t step = asc? 1:-1; + int32_t step = asc ? 1 : -1; // make sure it is aligned to 8bit, the allocated memory address is aligned to 256bit -// ASSERT((((uint64_t)pColData->pData) & (0x8 - 1)) == 0); + // ASSERT((((uint64_t)pColData->pData) & (0x8 - 1)) == 0); // 1. copy data in a batch model memcpy(pColData->pData, p, dumpedRows * tDataTypes[pData->type].bytes); // 2. reverse the array list in case of descending order scan data block if (!asc) { - switch(pColData->info.type) { + switch (pColData->info.type) { case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - { + case TSDB_DATA_TYPE_UBIGINT: { int32_t mid = dumpedRows >> 1u; int64_t* pts = (int64_t*)pColData->pData; for (int32_t j = 0; j < mid; ++j) { @@ -940,7 +956,7 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_UTINYINT: { - int32_t mid = dumpedRows >> 1u; + int32_t mid = dumpedRows >> 1u; int8_t* pts = (int8_t*)pColData->pData; for (int32_t j = 0; j < mid; ++j) { int8_t t = pts[j]; @@ -1107,6 +1123,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn setBlockAllDumped(pDumpInfo, ts, pReader->order); } + pBlockScanInfo->lastKey = pDumpInfo->lastKey; + double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; pReader->cost.blockLoadTime += elapsedTime; @@ -1688,7 +1706,7 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, SIterInfo* pIter, int64_t key, SLastBlockReader* pLastBlockReader) { SRowMerger merge = {0}; - STSRow* pTSRow = NULL; + SRow* pTSRow = NULL; SBlockData* pBlockData = &pReader->status.fileBlockData; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -1736,7 +1754,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (pReader->order == TSDB_ORDER_ASC) { if (minKey == key) { init = true; - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1746,10 +1764,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); } else { init = true; - int32_t code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1759,11 +1777,11 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { if (init) { - tRowMerge(&merge, pRow); + tsdbRowMerge(&merge, pRow); } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1777,7 +1795,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1791,10 +1809,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); } else { init = true; - int32_t code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1804,10 +1822,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { if (init) { - tRowMerge(&merge, &fRow); + tsdbRowMerge(&merge, &fRow); } else { init = true; - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1816,7 +1834,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } } - int32_t code = tRowMergerGetRow(&merge, &pTSRow); + int32_t code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1824,7 +1842,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); return TSDB_CODE_SUCCESS; } @@ -1834,7 +1852,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); - STSRow* pTSRow = NULL; + SRow* pTSRow = NULL; SRowMerger merge = {0}; TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", fRow.pBlockData, fRow.iRow, pLastBlockReader->uid, pReader->idStr); @@ -1845,16 +1863,16 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; } else { - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge, &pReader->verRange); - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1862,10 +1880,10 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); } } else { // not merge block data - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1877,7 +1895,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1885,7 +1903,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); } return TSDB_CODE_SUCCESS; @@ -1910,10 +1928,10 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader if (key < ts) { // imem, mem are all empty, file blocks (data blocks and last block) exist return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); } else if (key == ts) { - STSRow* pTSRow = NULL; + SRow* pTSRow = NULL; SRowMerger merge = {0}; - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1921,11 +1939,11 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge, &pReader->verRange); - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1933,7 +1951,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); return code; } else { return TSDB_CODE_SUCCESS; @@ -1949,7 +1967,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { SRowMerger merge = {0}; - STSRow* pTSRow = NULL; + SRow* pTSRow = NULL; int32_t code = TSDB_CODE_SUCCESS; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SArray* pDelList = pBlockScanInfo->delSkyline; @@ -2012,7 +2030,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { init = true; TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2023,10 +2041,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); } else { init = true; - code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2037,7 +2055,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == ik.ts) { if (init) { - tRowMerge(&merge, piRow); + tsdbRowMerge(&merge, piRow); } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); @@ -2045,7 +2063,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - code = tRowMergerInit(&merge, piRow, pSchema); + code = tsdbRowMergerInit(&merge, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2064,10 +2082,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - tRowMerge(&merge, pRow); + tsdbRowMerge(&merge, pRow); } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tRowMergerInit(&merge, pRow, pSchema); + code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2082,7 +2100,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tRowMergerInit(&merge, pRow, pSchema); + code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2096,11 +2114,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == ik.ts) { if (init) { - tRowMerge(&merge, piRow); + tsdbRowMerge(&merge, piRow); } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - code = tRowMergerInit(&merge, piRow, pSchema); + code = tsdbRowMergerInit(&merge, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2115,10 +2133,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); } else { init = true; - code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2129,7 +2147,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); if (!init) { - code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2137,7 +2155,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (merge.pTSchema == NULL) { return code; } - tRowMerge(&merge, &fRow); + tsdbRowMerge(&merge, &fRow); } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } @@ -2147,7 +2165,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2155,7 +2173,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); return code; } @@ -2168,9 +2186,11 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea TSDBKEY startKey = {0}; if (ASCENDING_TRAVERSE(pReader->order)) { - startKey = (TSDBKEY){.ts = pReader->window.skey, .version = pReader->verRange.minVer}; + // startKey = (TSDBKEY){.ts = pReader->window.skey, .version = pReader->verRange.minVer}; + startKey = (TSDBKEY){.ts = pBlockScanInfo->lastKey + 1, .version = pReader->verRange.minVer}; } else { - startKey = (TSDBKEY){.ts = pReader->window.ekey, .version = pReader->verRange.maxVer}; + // startKey = (TSDBKEY){.ts = pReader->window.ekey, .version = pReader->verRange.maxVer}; + startKey = (TSDBKEY){.ts = pBlockScanInfo->lastKey - 1, .version = pReader->verRange.maxVer}; } int32_t backward = (!ASCENDING_TRAVERSE(pReader->order)); @@ -2293,7 +2313,7 @@ static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLas bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo) { if ((pBlockData->nRow > 0) && (pBlockData->nRow != pDumpInfo->totalRows)) { - return false; // this is an invalid result. + return false; // this is an invalid result. } return pBlockData->nRow > 0 && (!pDumpInfo->allDumped); } @@ -2307,16 +2327,16 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc } else { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - STSRow* pTSRow = NULL; + SRow* pTSRow = NULL; SRowMerger merge = {0}; - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2324,7 +2344,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); return TSDB_CODE_SUCCESS; } } @@ -2444,7 +2464,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); // it is a clean block, load it directly - if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader) && + if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader) && pBlock->nRow <= pReader->capacity) { if (asc || ((!asc) && (!hasDataInLastBlock(pLastBlockReader)))) { copyBlockDataToSDataBlock(pReader, pBlockScanInfo); @@ -2464,7 +2484,8 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { while (1) { bool hasBlockData = false; { - while (pBlockData->nRow > 0) { // find the first qualified row in data block + while (pBlockData->nRow > 0 && + pBlockData->uid == pBlockScanInfo->uid) { // find the first qualified row in data block if (isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) { hasBlockData = true; break; @@ -2638,7 +2659,6 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { taosArrayDestroy(pIndexList); if (pReader->pReadSnap != NULL) { - SDelFile* pDelFile = pReader->pReadSnap->fs.pDelFile; if (pReader->pDelFReader == NULL && pDelFile != NULL) { int32_t code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pReader->pTsdb); @@ -2965,9 +2985,16 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { } // set the correct start position in case of the first/last file block, according to the query time window -void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { - SDataBlk* pBlock = getCurrentBlock(pBlockIter); - +static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { + int64_t lastKey = ASCENDING_TRAVERSE(pReader->order) ? INT64_MIN : INT64_MAX; + SDataBlk* pBlock = getCurrentBlock(pBlockIter); + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + if (pBlockInfo) { + STableBlockScanInfo* pScanInfo = taosHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + if (pScanInfo) { + lastKey = pScanInfo->lastKey; + } + } SReaderStatus* pStatus = &pReader->status; SFileBlockDumpInfo* pDumpInfo = &pStatus->fBlockDumpInfo; @@ -2975,6 +3002,7 @@ void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { pDumpInfo->totalRows = pBlock->nRow; pDumpInfo->allDumped = false; pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->order) ? 0 : pBlock->nRow - 1; + pDumpInfo->lastKey = lastKey; } static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBlockIter) { @@ -3066,7 +3094,12 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { } else { if (pReader->status.pCurrentFileset->nSttF > 0) { // data blocks in current file are exhausted, let's try the next file now - tBlockDataReset(&pReader->status.fileBlockData); + SBlockData* pBlockData = &pReader->status.fileBlockData; + if (pBlockData->uid != 0) { + tBlockDataClear(pBlockData); + } + + tBlockDataReset(pBlockData); resetDataBlockIterator(pBlockIter, pReader->order); goto _begin; } else { @@ -3174,7 +3207,8 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32 return false; } else if (pKey->ts == last->ts) { TSDBKEY* prev = taosArrayGet(pDelList, num - 2); - return (prev->version >= pKey->version && prev->version <= pVerRange->maxVer && prev->version >= pVerRange->minVer); + return (prev->version >= pKey->version && prev->version <= pVerRange->maxVer && + prev->version >= pVerRange->minVer); } } else { TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); @@ -3264,7 +3298,8 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p } TSDBROW* pRow = tsdbTbDataIterGet(pIter->iter); - TSDBKEY key = {.ts = pRow->pTSRow->ts, .version = pRow->version}; + TSDBKEY key = TSDBROW_KEY(pRow); + if (outOfTimeWindow(key.ts, &pReader->window)) { pIter->hasVal = false; return NULL; @@ -3317,12 +3352,16 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDe break; } - STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid); - if (pTSchema == NULL) { - return terrno; - } + if (pRow->type == TSDBROW_ROW_FMT) { + STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid); + if (pTSchema == NULL) { + return terrno; + } - tRowMergerAdd(pMerger, pRow, pTSchema); + tsdbRowMergerAdd(pMerger, pRow, pTSchema); + } else { // column format + tsdbRowMerge(pMerger, pRow); + } } return TSDB_CODE_SUCCESS; @@ -3337,7 +3376,7 @@ static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t rowInd } TSDBROW fRow = tsdbRowFromBlockData(pBlockData, rowIndex); - tRowMerge(pMerger, &fRow); + tsdbRowMerge(pMerger, &fRow); rowIndex += step; } @@ -3358,7 +3397,7 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn *state = CHECK_FILEBLOCK_QUIT; int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; - bool loadNeighbor = true; + bool loadNeighbor = true; int32_t code = loadNeighborIfOverlap(pFBlock, pScanInfo, pReader, &loadNeighbor); if (loadNeighbor && (code == TSDB_CODE_SUCCESS)) { @@ -3393,6 +3432,11 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc SFileDataBlockInfo* pFileBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); SDataBlk* pCurrentBlock = getCurrentBlock(&pReader->status.blockIter); + if (pFileBlockInfo == NULL) { + st = CHECK_FILEBLOCK_QUIT; + break; + } + checkForNeighborFileBlock(pReader, pScanInfo, pCurrentBlock, pFileBlockInfo, pMerger, key, &st); if (st == CHECK_FILEBLOCK_QUIT) { break; @@ -3409,7 +3453,7 @@ int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockSc int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 == ts) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tRowMerge(pMerger, &fRow1); + tsdbRowMerge(pMerger, &fRow1); } else { break; } @@ -3418,7 +3462,7 @@ int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockSc return TSDB_CODE_SUCCESS; } -int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, +int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, TSDBROW* pResRow, STsdbReader* pReader, bool* freeTSRow) { TSDBROW* pNextRow = NULL; TSDBROW current = *pRow; @@ -3427,19 +3471,19 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, pIter->hasVal = tsdbTbDataIterNext(pIter->iter); if (!pIter->hasVal) { - *pTSRow = current.pTSRow; + *pResRow = *pRow; *freeTSRow = false; return TSDB_CODE_SUCCESS; } else { // has next point in mem/imem pNextRow = getValidMemRow(pIter, pDelList, pReader); if (pNextRow == NULL) { - *pTSRow = current.pTSRow; + *pResRow = current; *freeTSRow = false; return TSDB_CODE_SUCCESS; } - if (current.pTSRow->ts != pNextRow->pTSRow->ts) { - *pTSRow = current.pTSRow; + if (TSDBROW_TS(¤t) != TSDBROW_TS(pNextRow)) { + *pResRow = current; *freeTSRow = false; return TSDB_CODE_SUCCESS; } @@ -3447,47 +3491,60 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, } SRowMerger merge = {0}; - - // get the correct schema for data in memory terrno = 0; - STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(¤t), pReader, uid); - if (pTSchema == NULL) { - return terrno; - } + int32_t code = 0; - if (pReader->pSchema == NULL) { - pReader->pSchema = pTSchema; - } + // start to merge duplicated rows + if (current.type == TSDBROW_ROW_FMT) { + // get the correct schema for data in memory + STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(¤t), pReader, uid); + if (pTSchema == NULL) { + return terrno; + } - int32_t code = tRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + if (pReader->pSchema == NULL) { + pReader->pSchema = pTSchema; + } - STSchema* pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid); - if (pTSchema1 == NULL) { - return terrno; - } + code = tsdbRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - tRowMergerAdd(&merge, pNextRow, pTSchema1); + STSchema* pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid); + if (pTSchema1 == NULL) { + return terrno; + } - code = doMergeRowsInBuf(pIter, uid, current.pTSRow->ts, pDelList, &merge, pReader); + tsdbRowMergerAdd(&merge, pNextRow, pTSchema1); + } else { // let's merge rows in file block + code = tsdbRowMergerInit(&merge, ¤t, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + tsdbRowMerge(&merge, pNextRow); + } + + code = doMergeRowsInBuf(pIter, uid, TSDBROW_TS(¤t), pDelList, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } - code = tRowMergerGetRow(&merge, pTSRow); + code = tsdbRowMergerGetRow(&merge, &pResRow->pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } - tRowMergerClear(&merge); + pResRow->type = TSDBROW_ROW_FMT; + tsdbRowMergerClear(&merge); *freeTSRow = true; + return TSDB_CODE_SUCCESS; } int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, - STSRow** pTSRow) { + SRow** pTSRow) { SRowMerger merge = {0}; TSDBKEY k = TSDBROW_KEY(pRow); @@ -3496,7 +3553,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, piRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3507,7 +3564,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p return code; } - tRowMerge(&merge, pRow); + tsdbRowMerge(&merge, pRow); code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { @@ -3517,7 +3574,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS || merge.pTSchema == NULL) { return code; } @@ -3528,7 +3585,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p return code; } - tRowMerge(&merge, piRow); + tsdbRowMerge(&merge, piRow); code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { @@ -3536,13 +3593,12 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p } } - int32_t code = tRowMergerGetRow(&merge, pTSRow); - tRowMergerClear(&merge); - + int32_t code = tsdbRowMergerGetRow(&merge, pTSRow); + tsdbRowMergerClear(&merge); return code; } -int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STSRow** pTSRow, int64_t endKey, +int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, TSDBROW* pResRow, int64_t endKey, bool* freeTSRow) { TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); @@ -3572,13 +3628,14 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR int32_t code = TSDB_CODE_SUCCESS; if (ik.ts != k.ts) { if (((ik.ts < k.ts) && asc) || ((ik.ts > k.ts) && (!asc))) { // ik.ts < k.ts - code = doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader, freeTSRow); + code = doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pResRow, pReader, freeTSRow); } else if (((k.ts < ik.ts) && asc) || ((k.ts > ik.ts) && (!asc))) { - code = doMergeMemTableMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, freeTSRow); + code = doMergeMemTableMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, pResRow, pReader, freeTSRow); } } else { // ik.ts == k.ts *freeTSRow = true; - code = doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, pTSRow); + pResRow->type = TSDBROW_ROW_FMT; + code = doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, &pResRow->pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3588,29 +3645,30 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR } if (pBlockScanInfo->iter.hasVal && pRow != NULL) { - return doMergeMemTableMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, + return doMergeMemTableMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pResRow, pReader, freeTSRow); } if (pBlockScanInfo->iiter.hasVal && piRow != NULL) { - return doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader, freeTSRow); + return doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pResRow, pReader, freeTSRow); } return TSDB_CODE_SUCCESS; } -int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, - STableBlockScanInfo* pScanInfo) { +int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, STableBlockScanInfo* pScanInfo) { int32_t outputRowIndex = pBlock->info.rows; int64_t uid = pScanInfo->uid; + int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); + SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; STSchema* pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid); SColVal colVal = {0}; int32_t i = 0, j = 0; - if (pSupInfo->colId[i]== PRIMARYKEY_TIMESTAMP_COL_ID) { + if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts; i += 1; @@ -3622,7 +3680,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* if (colId == pSchema->columns[j].colId) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); - tTSRowGetVal(pTSRow, pSchema, j, &colVal); + tRowGet(pTSRow, pSchema, j, &colVal); doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo); i += 1; j += 1; @@ -3655,7 +3713,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S int32_t outputRowIndex = pResBlock->info.rows; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - if (pReader->suppInfo.colId[i]== PRIMARYKEY_TIMESTAMP_COL_ID) { + if (pReader->suppInfo.colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; i += 1; @@ -3701,17 +3759,22 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e SSDataBlock* pBlock = pReader->pResBlock; do { - STSRow* pTSRow = NULL; + // SRow* pTSRow = NULL; + TSDBROW row = {.type = -1}; bool freeTSRow = false; - tsdbGetNextRowInMem(pBlockScanInfo, pReader, &pTSRow, endKey, &freeTSRow); - if (pTSRow == NULL) { + tsdbGetNextRowInMem(pBlockScanInfo, pReader, &row, endKey, &freeTSRow); + if (row.type == -1) { break; } - doAppendRowFromTSRow(pBlock, pReader, pTSRow, pBlockScanInfo); + if (row.type == TSDBROW_ROW_FMT) { + doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo); - if (freeTSRow) { - taosMemoryFree(pTSRow); + if (freeTSRow) { + taosMemoryFree(row.pTSRow); + } + } else { + doAppendRowFromFileBlock(pBlock, pReader, row.pBlockData, row.iRow); } // no data in buffer, return immediately @@ -3875,40 +3938,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL goto _err; } - if (numOfTables > 0) { - code = tsdbTakeReadSnap(pReader->pTsdb, &pReader->pReadSnap, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } - - if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) { - code = doOpenReaderImpl(pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } else { - STsdbReader* pPrevReader = pReader->innerReader[0]; - STsdbReader* pNextReader = pReader->innerReader[1]; - - // we need only one row - pPrevReader->capacity = 1; - pPrevReader->status.pTableMap = pReader->status.pTableMap; - pPrevReader->pSchema = pReader->pSchema; - pPrevReader->pMemSchema = pReader->pMemSchema; - pPrevReader->pReadSnap = pReader->pReadSnap; - - pNextReader->capacity = 1; - pNextReader->status.pTableMap = pReader->status.pTableMap; - pNextReader->pSchema = pReader->pSchema; - pNextReader->pMemSchema = pReader->pMemSchema; - pNextReader->pReadSnap = pReader->pReadSnap; - - code = doOpenReaderImpl(pPrevReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - } + pReader->suspended = true; tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr); return code; @@ -3959,7 +3989,7 @@ void tsdbReaderClose(STsdbReader* pReader) { } taosMemoryFree(pSupInfo->colId); - tBlockDataDestroy(&pReader->status.fileBlockData, true); + tBlockDataDestroy(&pReader->status.fileBlockData); cleanupDataBlockIterator(&pReader->status.blockIter); size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); @@ -3981,7 +4011,10 @@ void tsdbReaderClose(STsdbReader* pReader) { pReader->pDelIdx = NULL; } - tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap, pReader->idStr); + qTrace("tsdb/reader: %p, untake snapshot", pReader); + tsdbUntakeReadSnap(pReader, pReader->pReadSnap, true); + + taosThreadMutexDestroy(&pReader->readerMutex); taosMemoryFree(pReader->status.uidCheckInfo.tableUidList); SIOCostSummary* pCost = &pReader->cost; @@ -4013,16 +4046,180 @@ void tsdbReaderClose(STsdbReader* pReader) { if (pReader->pMemSchema != pReader->pSchema) { taosMemoryFree(pReader->pMemSchema); } + taosMemoryFreeClear(pReader); } +int32_t tsdbReaderSuspend(STsdbReader* pReader) { + int32_t code = 0; + + // save reader's base state & reset top state to be reconstructed from base state + SReaderStatus* pStatus = &pReader->status; + STableBlockScanInfo* pBlockScanInfo = NULL; + + if (pStatus->loadFromFile) { + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + if (pBlockInfo != NULL) { + pBlockScanInfo = + *(STableBlockScanInfo**)taosHashGet(pStatus->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + if (pBlockScanInfo == NULL) { + code = TSDB_CODE_INVALID_PARA; + tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", pBlockInfo->uid, + taosHashGetSize(pReader->status.pTableMap), pReader->idStr); + goto _err; + } + } else { + pBlockScanInfo = *pStatus->pTableIter; + } + + tsdbDataFReaderClose(&pReader->pFileReader); + + // resetDataBlockScanInfo excluding lastKey + STableBlockScanInfo** p = NULL; + + while ((p = taosHashIterate(pStatus->pTableMap, p)) != NULL) { + STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; + + pInfo->iterInit = false; + pInfo->iter.hasVal = false; + pInfo->iiter.hasVal = false; + + if (pInfo->iter.iter != NULL) { + pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter); + } + + if (pInfo->iiter.iter != NULL) { + pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter); + } + + pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); + // pInfo->lastKey = ts; + } + } else { + pBlockScanInfo = pStatus->pTableIter == NULL ? NULL : *pStatus->pTableIter; + if (pBlockScanInfo) { + // save lastKey to restore memory iterator + STimeWindow w = pReader->pResBlock->info.window; + pBlockScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->order) ? w.ekey : w.skey; + + // reset current current table's data block scan info, + pBlockScanInfo->iterInit = false; + // pBlockScanInfo->iiter.hasVal = false; + if (pBlockScanInfo->iter.iter != NULL) { + pBlockScanInfo->iter.iter = tsdbTbDataIterDestroy(pBlockScanInfo->iter.iter); + } + + if (pBlockScanInfo->iiter.iter != NULL) { + pBlockScanInfo->iiter.iter = tsdbTbDataIterDestroy(pBlockScanInfo->iiter.iter); + } + + pBlockScanInfo->pBlockList = taosArrayDestroy(pBlockScanInfo->pBlockList); + tMapDataClear(&pBlockScanInfo->mapData); + // TODO: keep skyline for reuse + pBlockScanInfo->delSkyline = taosArrayDestroy(pBlockScanInfo->delSkyline); + } + } + + tsdbUntakeReadSnap(pReader, pReader->pReadSnap, false); + pReader->pReadSnap = NULL; + + pReader->suspended = true; + + tsdbDebug("reader: %p suspended uid %" PRIu64 " in this query %s", pReader, pBlockScanInfo ? pBlockScanInfo->uid : 0, + pReader->idStr); + return code; + +_err: + tsdbError("failed to suspend data reader, code:%s %s", tstrerror(code), pReader->idStr); + return code; +} + +static int32_t tsdbSetQueryReseek(void* pQHandle) { + int32_t code = 0; + STsdbReader* pReader = pQHandle; + + code = taosThreadMutexTryLock(&pReader->readerMutex); + if (code == 0) { + if (pReader->suspended) { + taosThreadMutexUnlock(&pReader->readerMutex); + return code; + } + + tsdbReaderSuspend(pReader); + + taosThreadMutexUnlock(&pReader->readerMutex); + + return code; + } else if (code == EBUSY) { + return TSDB_CODE_VND_QUERY_BUSY; + } else { + terrno = TAOS_SYSTEM_ERROR(code); + return TSDB_CODE_FAILED; + } +} + +int32_t tsdbReaderResume(STsdbReader* pReader) { + int32_t code = 0; + + STableBlockScanInfo** pBlockScanInfo = pReader->status.pTableIter; + + // restore reader's state + // task snapshot + int32_t numOfTables = taosHashGetSize(pReader->status.pTableMap); + if (numOfTables > 0) { + qTrace("tsdb/reader: %p, take snapshot", pReader); + code = tsdbTakeReadSnap(pReader, tsdbSetQueryReseek, &pReader->pReadSnap); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + + if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) { + code = doOpenReaderImpl(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else { + STsdbReader* pPrevReader = pReader->innerReader[0]; + STsdbReader* pNextReader = pReader->innerReader[1]; + + // we need only one row + pPrevReader->capacity = 1; + pPrevReader->status.pTableMap = pReader->status.pTableMap; + pPrevReader->pSchema = pReader->pSchema; + pPrevReader->pMemSchema = pReader->pMemSchema; + pPrevReader->pReadSnap = pReader->pReadSnap; + + pNextReader->capacity = 1; + pNextReader->status.pTableMap = pReader->status.pTableMap; + pNextReader->pSchema = pReader->pSchema; + pNextReader->pMemSchema = pReader->pMemSchema; + pNextReader->pReadSnap = pReader->pReadSnap; + + code = doOpenReaderImpl(pPrevReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + } + + pReader->suspended = false; + + tsdbDebug("reader: %p resumed uid %" PRIu64 ", numOfTable:%" PRId32 ", in this query %s", pReader, + pBlockScanInfo ? (*pBlockScanInfo)->uid : 0, numOfTables, pReader->idStr); + return code; + +_err: + tsdbError("failed to resume data reader, code:%s %s", tstrerror(code), pReader->idStr); + return code; +} + static bool doTsdbNextDataBlock(STsdbReader* pReader) { // cleanup the data that belongs to the previous data block SSDataBlock* pBlock = pReader->pResBlock; blockDataCleanup(pBlock); SReaderStatus* pStatus = &pReader->status; - if (taosHashGetSize(pStatus->pTableMap) == 0){ + if (taosHashGetSize(pStatus->pTableMap) == 0) { return false; } @@ -4049,10 +4246,24 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { return false; } + SReaderStatus* pStatus = &pReader->status; + + qTrace("tsdb/read: %p, take read mutex", pReader); + taosThreadMutexLock(&pReader->readerMutex); + if (pReader->suspended) { + tsdbReaderResume(pReader); + } + if (pReader->innerReader[0] != NULL && pReader->step == 0) { bool ret = doTsdbNextDataBlock(pReader->innerReader[0]); pReader->step = EXTERNAL_ROWS_PREV; if (ret) { + pStatus = &pReader->innerReader[0]->status; + if (pStatus->composedDataBlock) { + qTrace("tsdb/read: %p, unlock read mutex", pReader); + taosThreadMutexUnlock(&pReader->readerMutex); + } + return ret; } } @@ -4071,6 +4282,11 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { bool ret = doTsdbNextDataBlock(pReader); if (ret) { + if (pStatus->composedDataBlock) { + qTrace("tsdb/read: %p, unlock read mutex", pReader); + taosThreadMutexUnlock(&pReader->readerMutex); + } + return ret; } @@ -4085,10 +4301,19 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { bool ret1 = doTsdbNextDataBlock(pReader->innerReader[1]); pReader->step = EXTERNAL_ROWS_NEXT; if (ret1) { + pStatus = &pReader->innerReader[1]->status; + if (pStatus->composedDataBlock) { + qTrace("tsdb/read: %p, unlock read mutex", pReader); + taosThreadMutexUnlock(&pReader->readerMutex); + } + return ret1; } } + qTrace("tsdb/read: %p, unlock read mutex", pReader); + taosThreadMutexUnlock(&pReader->readerMutex); + return false; } @@ -4112,12 +4337,10 @@ void tsdbRetrieveDataBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64 } } - -static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, - SColumnDataAgg* pTsAgg) { +static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) { // do fill all null column value SMA info int32_t i = 0, j = 0; - int32_t size = (int32_t) taosArrayGetSize(pSup->pColAgg); + int32_t size = (int32_t)taosArrayGetSize(pSup->pColAgg); taosArrayInsert(pSup->pColAgg, 0, pTsAgg); while (j < numOfCols && i < size) { @@ -4130,7 +4353,7 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ } else if (pSup->colId[j] < pAgg->colId) { if (pSup->colId[j] != PRIMARYKEY_TIMESTAMP_COL_ID) { SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows}; - taosArrayInsert(pSup->pColAgg, i ,&nullColAgg); + taosArrayInsert(pSup->pColAgg, i, &nullColAgg); } j += 1; } @@ -4226,12 +4449,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, } static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { - SReaderStatus* pStatus = &pReader->status; - - if (pStatus->composedDataBlock) { - return pReader->pResBlock; - } - + SReaderStatus* pStatus = &pReader->status; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter); STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)taosHashGet(pStatus->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); @@ -4244,7 +4462,7 @@ static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { - tBlockDataDestroy(&pStatus->fileBlockData, 1); + tBlockDataDestroy(&pStatus->fileBlockData); terrno = code; return NULL; } @@ -4253,20 +4471,51 @@ static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { return pReader->pResBlock; } +void tsdbReleaseDataBlock(STsdbReader* pReader) { + // SReaderStatus* pStatus = &pReader->status; + // if (!pStatus->composedDataBlock) { + taosThreadMutexUnlock(&pReader->readerMutex); + //} +} + SSDataBlock* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { + STsdbReader* pTReader = pReader; if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { if (pReader->step == EXTERNAL_ROWS_PREV) { - return doRetrieveDataBlock(pReader->innerReader[0]); + pTReader = pReader->innerReader[0]; } else if (pReader->step == EXTERNAL_ROWS_NEXT) { - return doRetrieveDataBlock(pReader->innerReader[1]); + pTReader = pReader->innerReader[1]; } } - return doRetrieveDataBlock(pReader); + SReaderStatus* pStatus = &pTReader->status; + if (pStatus->composedDataBlock) { + return pTReader->pResBlock; + } + + SSDataBlock* ret = doRetrieveDataBlock(pTReader); + + qTrace("tsdb/read-retrieve: %p, unlock read mutex", pReader); + taosThreadMutexUnlock(&pReader->readerMutex); + + return ret; } int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { + SReaderStatus* pStatus = &pReader->status; + + qTrace("tsdb/reader-reset: %p, take read mutex", pReader); + taosThreadMutexLock(&pReader->readerMutex); + + if (pReader->suspended) { + tsdbReaderResume(pReader); + } + if (isEmptyQueryTimeWindow(&pReader->window) || pReader->pReadSnap == NULL) { + tsdbDebug("tsdb reader reset return %p", pReader->pReadSnap); + + taosThreadMutexUnlock(&pReader->readerMutex); + return TSDB_CODE_SUCCESS; } @@ -4302,6 +4551,9 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { if (code != TSDB_CODE_SUCCESS) { tsdbError("%p reset reader failed, numOfTables:%d, query range:%" PRId64 " - %" PRId64 " in query %s", pReader, numOfTables, pReader->window.skey, pReader->window.ekey, pReader->idStr); + + taosThreadMutexUnlock(&pReader->readerMutex); + return code; } } @@ -4311,6 +4563,8 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { pReader, pReader->suid, numOfTables, pCond->twindows.skey, pReader->window.skey, pReader->window.ekey, pReader->idStr); + taosThreadMutexUnlock(&pReader->readerMutex); + return code; } @@ -4322,6 +4576,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa int32_t code = TSDB_CODE_SUCCESS; pTableBlockInfo->totalSize = 0; pTableBlockInfo->totalRows = 0; + pTableBlockInfo->numOfVgroups = 1; // find the start data block in file SReaderStatus* pStatus = &pReader->status; @@ -4393,13 +4648,18 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { int64_t rows = 0; SReaderStatus* pStatus = &pReader->status; + taosThreadMutexLock(&pReader->readerMutex); + if (pReader->suspended) { + tsdbReaderResume(pReader); + } + pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, NULL); while (pStatus->pTableIter != NULL) { STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; STbData* d = NULL; - if (pReader->pTsdb->mem != NULL) { + if (pReader->pReadSnap->pMem != NULL) { d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid); if (d != NULL) { rows += tsdbGetNRowsInTbData(d); @@ -4407,7 +4667,7 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { } STbData* di = NULL; - if (pReader->pTsdb->imem != NULL) { + if (pReader->pReadSnap->pIMem != NULL) { di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->suid, pBlockScanInfo->uid); if (di != NULL) { rows += tsdbGetNRowsInTbData(di); @@ -4418,6 +4678,8 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); } + taosThreadMutexUnlock(&pReader->readerMutex); + return rows; } @@ -4459,66 +4721,92 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 return TSDB_CODE_SUCCESS; } -int32_t tsdbTakeReadSnap(STsdb* pTsdb, STsdbReadSnap** ppSnap, const char* idStr) { - int32_t code = 0; +int32_t tsdbTakeReadSnap(STsdbReader* pReader, _query_reseek_func_t reseek, STsdbReadSnap** ppSnap) { + int32_t code = 0; + STsdb* pTsdb = pReader->pTsdb; + SVersionRange* pRange = &pReader->verRange; // alloc - *ppSnap = (STsdbReadSnap*)taosMemoryCalloc(1, sizeof(STsdbReadSnap)); - if (*ppSnap == NULL) { + STsdbReadSnap* pSnap = (STsdbReadSnap*)taosMemoryCalloc(1, sizeof(*pSnap)); + if (pSnap == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } // lock - code = taosThreadRwlockRdlock(&pTsdb->rwLock); - if (code) { - code = TAOS_SYSTEM_ERROR(code); - goto _exit; - } + taosThreadRwlockRdlock(&pTsdb->rwLock); // take snapshot - (*ppSnap)->pMem = pTsdb->mem; - (*ppSnap)->pIMem = pTsdb->imem; + if (pTsdb->mem && (pRange->minVer <= pTsdb->mem->maxVer && pRange->maxVer >= pTsdb->mem->minVer)) { + pSnap->pMem = pTsdb->mem; + pSnap->pNode = taosMemoryMalloc(sizeof(*pSnap->pNode)); + if (pSnap->pNode == NULL) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + pSnap->pNode->pQHandle = pReader; + pSnap->pNode->reseek = reseek; - if ((*ppSnap)->pMem) { - tsdbRefMemTable((*ppSnap)->pMem); + tsdbRefMemTable(pTsdb->mem, pSnap->pNode); } - if ((*ppSnap)->pIMem) { - tsdbRefMemTable((*ppSnap)->pIMem); + if (pTsdb->imem && (pRange->minVer <= pTsdb->imem->maxVer && pRange->maxVer >= pTsdb->imem->minVer)) { + pSnap->pIMem = pTsdb->imem; + pSnap->pINode = taosMemoryMalloc(sizeof(*pSnap->pINode)); + if (pSnap->pINode == NULL) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + pSnap->pINode->pQHandle = pReader; + pSnap->pINode->reseek = reseek; + + tsdbRefMemTable(pTsdb->imem, pSnap->pINode); } // fs - code = tsdbFSRef(pTsdb, &(*ppSnap)->fs); + code = tsdbFSRef(pTsdb, &pSnap->fs); if (code) { taosThreadRwlockUnlock(&pTsdb->rwLock); goto _exit; } // unlock - code = taosThreadRwlockUnlock(&pTsdb->rwLock); - if (code) { - code = TAOS_SYSTEM_ERROR(code); - goto _exit; - } + taosThreadRwlockUnlock(&pTsdb->rwLock); + + tsdbTrace("vgId:%d, take read snapshot", TD_VID(pTsdb->pVnode)); - tsdbTrace("vgId:%d, take read snapshot, %s", TD_VID(pTsdb->pVnode), idStr); _exit: + if (code) { + *ppSnap = NULL; + if (pSnap) { + if (pSnap->pNode) taosMemoryFree(pSnap->pNode); + if (pSnap->pINode) taosMemoryFree(pSnap->pINode); + taosMemoryFree(pSnap); + } + } else { + *ppSnap = pSnap; + } return code; } -void tsdbUntakeReadSnap(STsdb* pTsdb, STsdbReadSnap* pSnap, const char* idStr) { +void tsdbUntakeReadSnap(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proactive) { + STsdb* pTsdb = pReader->pTsdb; + if (pSnap) { if (pSnap->pMem) { - tsdbUnrefMemTable(pSnap->pMem); + tsdbUnrefMemTable(pSnap->pMem, pSnap->pNode, proactive); } if (pSnap->pIMem) { - tsdbUnrefMemTable(pSnap->pIMem); + tsdbUnrefMemTable(pSnap->pIMem, pSnap->pINode, proactive); } tsdbFSUnref(pTsdb, &pSnap->fs); + if (pSnap->pNode) taosMemoryFree(pSnap->pNode); + if (pSnap->pINode) taosMemoryFree(pSnap->pINode); taosMemoryFree(pSnap); } - tsdbTrace("vgId:%d, untake read snapshot, %s", TD_VID(pTsdb->pVnode), idStr); + tsdbTrace("vgId:%d, untake read snapshot", TD_VID(pTsdb->pVnode)); } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 08d52554c6cf7452f201e745afe2a99c632dd77f..5c7bfee5a7bc07bdf1e91015e4d3d36e5873d7ec 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -541,7 +541,7 @@ _exit: if (pReader) { taosArrayDestroy(pReader->aDelData); taosArrayDestroy(pReader->aDelIdx); - tBlockDataDestroy(&pReader->bData, 1); + tBlockDataDestroy(&pReader->bData); tsdbFSDestroy(&pReader->fs); taosMemoryFree(pReader); } @@ -568,10 +568,10 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { taosArrayDestroy(pIter->aSttBlk); } - tBlockDataDestroy(&pIter->bData, 1); + tBlockDataDestroy(&pIter->bData); } - tBlockDataDestroy(&pReader->bData, 1); + tBlockDataDestroy(&pReader->bData); tDestroyTSchema(pReader->skmTable.pTSchema); // del @@ -1357,13 +1357,13 @@ _exit: if (pWriter->aDelIdxW) taosArrayDestroy(pWriter->aDelIdxW); if (pWriter->aDelData) taosArrayDestroy(pWriter->aDelData); if (pWriter->aDelIdxR) taosArrayDestroy(pWriter->aDelIdxR); - tBlockDataDestroy(&pWriter->dWriter.sData, 1); - tBlockDataDestroy(&pWriter->dWriter.bData, 1); + tBlockDataDestroy(&pWriter->dWriter.sData); + tBlockDataDestroy(&pWriter->dWriter.bData); if (pWriter->dWriter.aSttBlk) taosArrayDestroy(pWriter->dWriter.aSttBlk); if (pWriter->dWriter.aBlockIdx) taosArrayDestroy(pWriter->dWriter.aBlockIdx); - tBlockDataDestroy(&pWriter->dReader.bData, 1); + tBlockDataDestroy(&pWriter->dReader.bData); if (pWriter->dReader.aBlockIdx) taosArrayDestroy(pWriter->dReader.aBlockIdx); - tBlockDataDestroy(&pWriter->bData, 1); + tBlockDataDestroy(&pWriter->bData); tsdbFSDestroy(&pWriter->fs); taosMemoryFree(pWriter); } @@ -1423,18 +1423,18 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { // SNAP_DATA_TSDB // Writer - tBlockDataDestroy(&pWriter->dWriter.sData, 1); - tBlockDataDestroy(&pWriter->dWriter.bData, 1); + tBlockDataDestroy(&pWriter->dWriter.sData); + tBlockDataDestroy(&pWriter->dWriter.bData); taosArrayDestroy(pWriter->dWriter.aSttBlk); tMapDataClear(&pWriter->dWriter.mDataBlk); taosArrayDestroy(pWriter->dWriter.aBlockIdx); // Reader - tBlockDataDestroy(&pWriter->dReader.bData, 1); + tBlockDataDestroy(&pWriter->dReader.bData); tMapDataClear(&pWriter->dReader.mDataBlk); taosArrayDestroy(pWriter->dReader.aBlockIdx); - tBlockDataDestroy(&pWriter->bData, 1); + tBlockDataDestroy(&pWriter->bData); tDestroyTSchema(pWriter->skmTable.pTSchema); for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) { diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 86adc1dc8005d2233b1da39fe4f70ff7e61d43a8..f4ac4b7a975b9639d865ef7bd1a9631edff7fa37 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -23,7 +23,7 @@ void tMapDataReset(SMapData *pMapData) { } void tMapDataClear(SMapData *pMapData) { - tFree((uint8_t *)pMapData->aOffset); + tFree(pMapData->aOffset); tFree(pMapData->pData); pMapData->pData = NULL; pMapData->aOffset = NULL; @@ -572,9 +572,9 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal * ASSERT(iCol > 0); - if (pRow->type == 0) { - tTSRowGetVal(pRow->pTSRow, pTSchema, iCol, pColVal); - } else if (pRow->type == 1) { + if (pRow->type == TSDBROW_ROW_FMT) { + tRowGet(pRow->pTSRow, pTSchema, iCol, pColVal); + } else if (pRow->type == TSDBROW_COL_FMT) { SColData *pColData; tBlockDataGetColData(pRow->pBlockData, pTColumn->colId, &pColData); @@ -589,60 +589,61 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal * } } -// int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow) { -// int32_t n = 0; - -// n += tPutI64(p, pRow->version); -// if (p) memcpy(p + n, pRow->pTSRow, pRow->pTSRow->len); -// n += pRow->pTSRow->len; - -// return n; -// } - int32_t tsdbRowCmprFn(const void *p1, const void *p2) { return tsdbKeyCmprFn(&TSDBROW_KEY((TSDBROW *)p1), &TSDBROW_KEY((TSDBROW *)p2)); } // STSDBRowIter ====================================================== -void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { + int32_t code = 0; + pIter->pRow = pRow; - if (pRow->type == 0) { - ASSERT(pTSchema); - pIter->pTSchema = pTSchema; - pIter->i = 1; - } else if (pRow->type == 1) { - pIter->pTSchema = NULL; - pIter->i = 0; + if (pRow->type == TSDBROW_ROW_FMT) { + code = tRowIterOpen(pRow->pTSRow, pTSchema, &pIter->pIter); + if (code) goto _exit; + } else if (pRow->type == TSDBROW_COL_FMT) { + pIter->iColData = 0; } else { ASSERT(0); } + +_exit: + return code; } -SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { - if (pIter->pRow->type == 0) { - if (pIter->i < pIter->pTSchema->numOfCols) { - tTSRowGetVal(pIter->pRow->pTSRow, pIter->pTSchema, pIter->i, &pIter->colVal); - pIter->i++; +void tsdbRowClose(STSDBRowIter *pIter) { + if (pIter->pRow->type == TSDBROW_ROW_FMT) { + tRowIterClose(&pIter->pIter); + } +} - return &pIter->colVal; +SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { + if (pIter->pRow->type == TSDBROW_ROW_FMT) { + return tRowIterNext(pIter->pIter); + } else if (pIter->pRow->type == TSDBROW_COL_FMT) { + if (pIter->iColData == 0) { + pIter->cv = COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, + (SValue){.val = pIter->pRow->pBlockData->aTSKEY[pIter->pRow->iRow]}); + ++pIter->iColData; + return &pIter->cv; } - } else { - if (pIter->i < pIter->pRow->pBlockData->nColData) { - SColData *pColData = tBlockDataGetColDataByIdx(pIter->pRow->pBlockData, pIter->i); - - tColDataGetValue(pColData, pIter->pRow->iRow, &pIter->colVal); - pIter->i++; - return &pIter->colVal; + if (pIter->iColData < pIter->pRow->pBlockData->nColData) { + tColDataGetValue(&pIter->pRow->pBlockData->aColData[pIter->iColData], pIter->pRow->iRow, &pIter->cv); + ++pIter->iColData; + return &pIter->cv; + } else { + return NULL; } + } else { + ASSERT(0); + return NULL; // suppress error report by compiler } - - return NULL; } // SRowMerger ====================================================== -int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; @@ -684,7 +685,7 @@ int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRo tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal); if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { uint8_t *pVal = pColVal->value.pData; - + pColVal->value.pData = NULL; code = tRealloc(&pColVal->value.pData, pColVal->value.nData); if (code) goto _exit; @@ -709,7 +710,7 @@ _exit: return code; } -int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; @@ -753,7 +754,7 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { code = tRealloc(&tColVal->value.pData, pColVal->value.nData); if (code) return code; - tColVal->value.nData = pColVal->value.nData; + tColVal->value.nData = pColVal->value.nData; if (pColVal->value.nData) { memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData); } @@ -771,7 +772,7 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { return code; } -int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; @@ -802,7 +803,7 @@ int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal); if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { uint8_t *pVal = pColVal->value.pData; - + pColVal->value.pData = NULL; code = tRealloc(&pColVal->value.pData, pColVal->value.nData); if (code) goto _exit; @@ -811,7 +812,7 @@ int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { memcpy(pColVal->value.pData, pVal, pColVal->value.nData); } } - + if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -822,7 +823,7 @@ _exit: return code; } -void tRowMergerClear(SRowMerger *pMerger) { +void tsdbRowMergerClear(SRowMerger *pMerger) { for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); if (IS_VAR_DATA_TYPE(pTColVal->type)) { @@ -830,10 +831,10 @@ void tRowMergerClear(SRowMerger *pMerger) { } } - taosArrayDestroy(pMerger->pArray); + taosArrayDestroy(pMerger->pArray); } -int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { +int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; @@ -853,7 +854,7 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { pTColVal->value.nData = pColVal->value.nData; if (pTColVal->value.nData) { - memcpy(pTColVal->value.pData, pColVal->value.pData, pTColVal->value.nData); + memcpy(pTColVal->value.pData, pColVal->value.pData, pTColVal->value.nData); } pTColVal->flag = 0; } else { @@ -875,7 +876,7 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { tColVal->value.nData = pColVal->value.nData; if (tColVal->value.nData) { - memcpy(tColVal->value.pData, pColVal->value.pData, tColVal->value.nData); + memcpy(tColVal->value.pData, pColVal->value.pData, tColVal->value.nData); } tColVal->flag = 0; } else { @@ -898,12 +899,8 @@ _exit: return code; } -int32_t tRowMergerGetRow(SRowMerger *pMerger, STSRow **ppRow) { - int32_t code = 0; - - code = tdSTSRowNew(pMerger->pArray, pMerger->pTSchema, ppRow); - - return code; +int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow) { + return tRowBuild(pMerger->pArray, pMerger->pTSchema, ppRow); } // delete skyline ====================================================== @@ -1022,27 +1019,49 @@ int32_t tBlockDataCreate(SBlockData *pBlockData) { pBlockData->aVersion = NULL; pBlockData->aTSKEY = NULL; pBlockData->nColData = 0; - pBlockData->aColData = taosArrayInit(0, sizeof(SColData)); - if (pBlockData->aColData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } + pBlockData->aColData = NULL; _exit: return code; } -void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear) { - tFree((uint8_t *)pBlockData->aUid); - tFree((uint8_t *)pBlockData->aVersion); - tFree((uint8_t *)pBlockData->aTSKEY); - taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataDestroy : NULL); - pBlockData->aUid = NULL; - pBlockData->aVersion = NULL; - pBlockData->aTSKEY = NULL; - pBlockData->aColData = NULL; +void tBlockDataDestroy(SBlockData *pBlockData) { + tFree(pBlockData->aUid); + tFree(pBlockData->aVersion); + tFree(pBlockData->aTSKEY); + + for (int32_t i = 0; i < pBlockData->nColData; i++) { + tColDataDestroy(&pBlockData->aColData[i]); + } + + if (pBlockData->aColData) { + taosMemoryFree(pBlockData->aColData); + pBlockData->aColData = NULL; + } } +static int32_t tBlockDataAdjustColData(SBlockData *pBlockData, int32_t nColData) { + int32_t code = 0; + + if (pBlockData->nColData > nColData) { + for (int32_t i = nColData; i < pBlockData->nColData; i++) { + tColDataDestroy(&pBlockData->aColData[i]); + } + } else if (pBlockData->nColData < nColData) { + SColData *aColData = taosMemoryRealloc(pBlockData->aColData, sizeof(SBlockData) * nColData); + if (aColData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + pBlockData->aColData = aColData; + memset(&pBlockData->aColData[pBlockData->nColData], 0, sizeof(SBlockData) * (nColData - pBlockData->nColData)); + } + pBlockData->nColData = nColData; + +_exit: + return code; +} int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid) { int32_t code = 0; @@ -1052,37 +1071,35 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, pBlockData->uid = pId->uid; pBlockData->nRow = 0; - pBlockData->nColData = 0; if (aCid) { + code = tBlockDataAdjustColData(pBlockData, nCid); + if (code) goto _exit; + int32_t iColumn = 1; STColumn *pTColumn = &pTSchema->columns[iColumn]; for (int32_t iCid = 0; iCid < nCid; iCid++) { - while (pTColumn && pTColumn->colId < aCid[iCid]) { + ASSERT(pTColumn); + while (pTColumn->colId < aCid[iCid]) { iColumn++; - pTColumn = (iColumn < pTSchema->numOfCols) ? &pTSchema->columns[iColumn] : NULL; + ASSERT(iColumn < pTSchema->numOfCols); + pTColumn = &pTSchema->columns[iColumn]; } - if (pTColumn == NULL) { - break; - } else if (pTColumn->colId == aCid[iCid]) { - SColData *pColData; - code = tBlockDataAddColData(pBlockData, &pColData); - if (code) goto _exit; - tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) ? 1 : 0); + ASSERT(pTColumn->colId == aCid[iCid]); + tColDataInit(&pBlockData->aColData[iCid], pTColumn->colId, pTColumn->type, + (pTColumn->flags & COL_SMA_ON) ? 1 : 0); - iColumn++; - pTColumn = (iColumn < pTSchema->numOfCols) ? &pTSchema->columns[iColumn] : NULL; - } + iColumn++; + pTColumn = (iColumn < pTSchema->numOfCols) ? &pTSchema->columns[iColumn] : NULL; } } else { - for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { - STColumn *pTColumn = &pTSchema->columns[iColumn]; - - SColData *pColData; - code = tBlockDataAddColData(pBlockData, &pColData); - if (code) goto _exit; + code = tBlockDataAdjustColData(pBlockData, pTSchema->numOfCols - 1); + if (code) goto _exit; - tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) ? 1 : 0); + for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { + STColumn *pTColumn = &pTSchema->columns[iColData + 1]; + tColDataInit(&pBlockData->aColData[iColData], pTColumn->colId, pTColumn->type, + (pTColumn->flags & COL_SMA_ON) ? 1 : 0); } } @@ -1093,8 +1110,6 @@ _exit: void tBlockDataReset(SBlockData *pBlockData) { pBlockData->suid = 0; pBlockData->uid = 0; - pBlockData->nRow = 0; - pBlockData->nColData = 0; } void tBlockDataClear(SBlockData *pBlockData) { @@ -1102,31 +1117,8 @@ void tBlockDataClear(SBlockData *pBlockData) { pBlockData->nRow = 0; for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - tColDataClear(pColData); - } -} - -int32_t tBlockDataAddColData(SBlockData *pBlockData, SColData **ppColData) { - int32_t code = 0; - SColData *pColData = NULL; - - if (pBlockData->nColData >= taosArrayGetSize(pBlockData->aColData)) { - if (taosArrayPush(pBlockData->aColData, &((SColData){0})) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } + tColDataClear(tBlockDataGetColDataByIdx(pBlockData, iColData)); } - pColData = (SColData *)taosArrayGet(pBlockData->aColData, pBlockData->nColData); - - pBlockData->nColData++; - - *ppColData = pColData; - return code; - -_err: - *ppColData = NULL; - return code; } static int32_t tBlockDataAppendBlockRow(SBlockData *pBlockData, SBlockData *pBlockDataFrom, int32_t iRow) { @@ -1134,17 +1126,13 @@ static int32_t tBlockDataAppendBlockRow(SBlockData *pBlockData, SBlockData *pBlo SColVal cv = {0}; int32_t iColDataFrom = 0; - SColData *pColDataFrom = - (iColDataFrom < pBlockDataFrom->nColData) ? &((SColData *)pBlockDataFrom->aColData->pData)[iColDataFrom] : NULL; + SColData *pColDataFrom = (iColDataFrom < pBlockDataFrom->nColData) ? &pBlockDataFrom->aColData[iColDataFrom] : NULL; for (int32_t iColDataTo = 0; iColDataTo < pBlockData->nColData; iColDataTo++) { - SColData *pColDataTo = &((SColData *)pBlockData->aColData->pData)[iColDataTo]; + SColData *pColDataTo = &pBlockData->aColData[iColDataTo]; while (pColDataFrom && pColDataFrom->cid < pColDataTo->cid) { - iColDataFrom++; - pColDataFrom = (iColDataFrom < pBlockDataFrom->nColData) - ? &((SColData *)pBlockDataFrom->aColData->pData)[iColDataFrom] - : NULL; + pColDataFrom = (++iColDataFrom < pBlockDataFrom->nColData) ? &pBlockDataFrom->aColData[iColDataFrom] : NULL; } if (pColDataFrom == NULL || pColDataFrom->cid > pColDataTo->cid) { @@ -1156,157 +1144,7 @@ static int32_t tBlockDataAppendBlockRow(SBlockData *pBlockData, SBlockData *pBlo code = tColDataAppendValue(pColDataTo, &cv); if (code) goto _exit; - iColDataFrom++; - pColDataFrom = (iColDataFrom < pBlockDataFrom->nColData) - ? &((SColData *)pBlockDataFrom->aColData->pData)[iColDataFrom] - : NULL; - } - } - -_exit: - return code; -} - -static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSchema *pTSchema) { - int32_t code = 0; - - int32_t iTColumn = 1; - STColumn *pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - void *pBitmap = pRow->statis ? tdGetBitmapAddrTp(pRow, pTSchema->flen) : NULL; - - for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = &((SColData *)pBlockData->aColData->pData)[iColData]; - - while (pTColumn && pTColumn->colId < pColData->cid) { - iTColumn++; - pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - } - - if (pTColumn == NULL || pTColumn->colId > pColData->cid) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } else { - ASSERT(pTColumn->type == pColData->type); - - SColVal cv = {.cid = pTColumn->colId, .type = pTColumn->type}; - - if (pRow->statis) { - TDRowValT vt = TD_VTYPE_MAX; - tdGetBitmapValTypeII(pBitmap, iTColumn - 1, &vt); - - if (vt == TD_VTYPE_NORM) { - cv.flag = CV_FLAG_VALUE; - - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); - cv.value.nData = varDataLen(pData); - cv.value.pData = varDataVal(pData); - } else { - memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); - } - - code = tColDataAppendValue(pColData, &cv); - if (code) goto _exit; - } else if (vt == TD_VTYPE_NONE) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } else if (vt == TD_VTYPE_NULL) { - code = tColDataAppendValue(pColData, &COL_VAL_NULL(pColData->cid, pColData->type)); - if (code) goto _exit; - } else { - ASSERT(0); - } - } else { - cv.flag = CV_FLAG_VALUE; - - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); - cv.value.nData = varDataLen(pData); - cv.value.pData = varDataVal(pData); - } else { - memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); - } - - code = tColDataAppendValue(pColData, &cv); - if (code) goto _exit; - } - - iTColumn++; - pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - } - } - -_exit: - return code; -} - -static int32_t tBlockDataAppendKVRow(SBlockData *pBlockData, STSRow *pRow, STSchema *pTSchema) { - int32_t code = 0; - - col_id_t kvIter = 0; - col_id_t nKvCols = tdRowGetNCols(pRow) - 1; - void *pColIdx = TD_ROW_COL_IDX(pRow); - void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow)); - int32_t iTColumn = 1; - STColumn *pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - - for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = &((SColData *)pBlockData->aColData->pData)[iColData]; - - while (pTColumn && pTColumn->colId < pColData->cid) { - iTColumn++; - pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - } - - if (pTColumn == NULL || pTColumn->colId > pColData->cid) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } else { - ASSERT(pTColumn->type == pColData->type); - - SColVal cv = {.cid = pTColumn->colId, .type = pTColumn->type}; - TDRowValT vt = TD_VTYPE_NONE; // default is NONE - SKvRowIdx *pKvIdx = NULL; - - while (kvIter < nKvCols) { - pKvIdx = (SKvRowIdx *)POINTER_SHIFT(pColIdx, kvIter * sizeof(SKvRowIdx)); - if (pKvIdx->colId == pTColumn->colId) { - tdGetBitmapValTypeII(pBitmap, kvIter, &vt); - ++kvIter; - break; - } else if (pKvIdx->colId > pTColumn->colId) { - vt = TD_VTYPE_NONE; - break; - } else { - ++kvIter; - } - } - - if (vt == TD_VTYPE_NORM) { - cv.flag = CV_FLAG_VALUE; - - void *pData = POINTER_SHIFT(pRow, pKvIdx->offset); - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - cv.value.nData = varDataLen(pData); - cv.value.pData = varDataVal(pData); - } else { - memcpy(&cv.value.val, pData, pTColumn->bytes); - } - - code = tColDataAppendValue(pColData, &cv); - if (code) goto _exit; - } else if (vt == TD_VTYPE_NONE) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } else if (vt == TD_VTYPE_NULL) { - code = tColDataAppendValue(pColData, &COL_VAL_NULL(pColData->cid, pColData->type)); - if (code) goto _exit; - } else { - ASSERT(0); - } - - iTColumn++; - pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + pColDataFrom = (++iColDataFrom < pBlockDataFrom->nColData) ? &pBlockDataFrom->aColData[iColDataFrom] : NULL; } } @@ -1336,19 +1174,14 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS pBlockData->aTSKEY[pBlockData->nRow] = TSDBROW_TS(pRow); SColVal cv = {0}; - if (pRow->type == 0) { - if (TD_IS_TP_ROW(pRow->pTSRow)) { - code = tBlockDataAppendTPRow(pBlockData, pRow->pTSRow, pTSchema); - if (code) goto _err; - } else if (TD_IS_KV_ROW(pRow->pTSRow)) { - code = tBlockDataAppendKVRow(pBlockData, pRow->pTSRow, pTSchema); - if (code) goto _err; - } else { - ASSERT(0); - } - } else { + if (pRow->type == TSDBROW_ROW_FMT) { + code = tRowAppendToColData(pRow->pTSRow, pTSchema, pBlockData->aColData, pBlockData->nColData); + if (code) goto _err; + } else if (pRow->type == TSDBROW_COL_FMT) { code = tBlockDataAppendBlockRow(pBlockData, pRow->pBlockData, pRow->iRow); if (code) goto _err; + } else { + ASSERT(0); } pBlockData->nRow++; @@ -1358,133 +1191,13 @@ _err: return code; } -int32_t tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFrom) { - int32_t code = 0; - - int32_t iColData = 0; - for (int32_t iColDataFrom = 0; iColDataFrom < pBlockDataFrom->nColData; iColDataFrom++) { - SColData *pColDataFrom = tBlockDataGetColDataByIdx(pBlockDataFrom, iColDataFrom); - - while (true) { - SColData *pColData; - if (iColData < pBlockData->nColData) { - pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - } else { - pColData = NULL; - } - - if (pColData == NULL || pColData->cid > pColDataFrom->cid) { - code = tBlockDataAddColData(pBlockData, &pColData); - if (code) goto _exit; - - tColDataInit(pColData, pColDataFrom->cid, pColDataFrom->type, pColDataFrom->smaOn); - for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } - - iColData++; - break; - } else if (pColData->cid == pColDataFrom->cid) { - iColData++; - break; - } else { - iColData++; - } - } - } - -_exit: - return code; -} - -int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData) { - int32_t code = 0; - - ASSERT(pBlockData->suid == pBlockData1->suid); - ASSERT(pBlockData->uid == pBlockData1->uid); - ASSERT(pBlockData1->nRow > 0); - ASSERT(pBlockData2->nRow > 0); - - tBlockDataClear(pBlockData); - - TSDBROW row1 = tsdbRowFromBlockData(pBlockData1, 0); - TSDBROW row2 = tsdbRowFromBlockData(pBlockData2, 0); - TSDBROW *pRow1 = &row1; - TSDBROW *pRow2 = &row2; - - while (pRow1 && pRow2) { - int32_t c = tsdbRowCmprFn(pRow1, pRow2); - - if (c < 0) { - code = tBlockDataAppendRow(pBlockData, pRow1, NULL, - pBlockData1->uid ? pBlockData1->uid : pBlockData1->aUid[pRow1->iRow]); - if (code) goto _exit; - - pRow1->iRow++; - if (pRow1->iRow < pBlockData1->nRow) { - *pRow1 = tsdbRowFromBlockData(pBlockData1, pRow1->iRow); - } else { - pRow1 = NULL; - } - } else if (c > 0) { - code = tBlockDataAppendRow(pBlockData, pRow2, NULL, - pBlockData2->uid ? pBlockData2->uid : pBlockData2->aUid[pRow2->iRow]); - if (code) goto _exit; - - pRow2->iRow++; - if (pRow2->iRow < pBlockData2->nRow) { - *pRow2 = tsdbRowFromBlockData(pBlockData2, pRow2->iRow); - } else { - pRow2 = NULL; - } - } else { - ASSERT(0); - } - } - - while (pRow1) { - code = tBlockDataAppendRow(pBlockData, pRow1, NULL, - pBlockData1->uid ? pBlockData1->uid : pBlockData1->aUid[pRow1->iRow]); - if (code) goto _exit; - - pRow1->iRow++; - if (pRow1->iRow < pBlockData1->nRow) { - *pRow1 = tsdbRowFromBlockData(pBlockData1, pRow1->iRow); - } else { - pRow1 = NULL; - } - } - - while (pRow2) { - code = tBlockDataAppendRow(pBlockData, pRow2, NULL, - pBlockData2->uid ? pBlockData2->uid : pBlockData2->aUid[pRow2->iRow]); - if (code) goto _exit; - - pRow2->iRow++; - if (pRow2->iRow < pBlockData2->nRow) { - *pRow2 = tsdbRowFromBlockData(pBlockData2, pRow2->iRow); - } else { - pRow2 = NULL; - } - } - -_exit: - return code; -} - -SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx) { - ASSERT(idx >= 0 && idx < pBlockData->nColData); - return (SColData *)taosArrayGet(pBlockData->aColData, idx); -} - void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData) { ASSERT(cid != PRIMARYKEY_TIMESTAMP_COL_ID); int32_t lidx = 0; int32_t ridx = pBlockData->nColData - 1; while (lidx <= ridx) { - int32_t midx = (lidx + ridx) / 2; + int32_t midx = (lidx + ridx) >> 1; SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, midx); int32_t c = (pColData->cid == cid) ? 0 : ((pColData->cid > cid) ? 1 : -1); @@ -1631,15 +1344,25 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin // loop to decode each column data if (hdr.szBlkCol == 0) goto _exit; + int32_t nColData = 0; int32_t nt = 0; while (nt < hdr.szBlkCol) { SBlockCol blockCol = {0}; nt += tGetBlockCol(pIn + n + nt, &blockCol); - ASSERT(nt <= hdr.szBlkCol); + ++nColData; + } + ASSERT(nt == hdr.szBlkCol); - SColData *pColData; - code = tBlockDataAddColData(pBlockData, &pColData); - if (code) goto _exit; + code = tBlockDataAdjustColData(pBlockData, nColData); + if (code) goto _exit; + + nt = 0; + int32_t iColData = 0; + while (nt < hdr.szBlkCol) { + SBlockCol blockCol = {0}; + nt += tGetBlockCol(pIn + n + nt, &blockCol); + + SColData *pColData = &pBlockData->aColData[iColData++]; tColDataInit(pColData, blockCol.cid, blockCol.type, blockCol.smaOn); if (blockCol.flag == HAS_NULL) { diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 49d5eaac43c6dbd2356bf95ca4cfb9c70cb350c7..02d076113f6cdab7066740ce071426ad06efb7d4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -26,14 +26,17 @@ static int64_t tsMaxKeyByPrecision[] = {31556995200000L, 31556995200000000L, 921 // static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); -int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *pRsp) { - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - int32_t affectedrows = 0; - int32_t numOfRows = 0; +int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2 *pRsp) { + int32_t arrSize = 0; + int32_t affectedrows = 0; + int32_t numOfRows = 0; ASSERT(pTsdb->mem != NULL); + if (pMsg) { + arrSize = taosArrayGetSize(pMsg->aSubmitTbData); + } + // scan and convert if (tsdbScanAndConvertSubmitMsg(pTsdb, pMsg) < 0) { if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) { @@ -43,18 +46,10 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp * } // loop to insert - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { - return -1; - } - while (true) { - SSubmitBlkRsp r = {0}; - tGetSubmitMsgNext(&msgIter, &pBlock); - if (pBlock == NULL) break; - if ((terrno = tsdbInsertTableData(pTsdb, version, &msgIter, pBlock, &r)) < 0) { + for (int32_t i = 0; i < arrSize; ++i) { + if ((terrno = tsdbInsertTableData(pTsdb, version, taosArrayGet(pMsg->aSubmitTbData, i), &affectedrows)) < 0) { return -1; } - - numOfRows += msgIter.numOfRows; } if (pRsp != NULL) { @@ -82,20 +77,19 @@ static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, STable *pTable, STSRow * } #endif -static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, STSRow *row, TSKEY minKey, TSKEY maxKey, +static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, TSKEY rowKey, TSKEY minKey, TSKEY maxKey, TSKEY now) { - TSKEY rowKey = TD_ROW_KEY(row); if (rowKey < minKey || rowKey > maxKey) { tsdbError("vgId:%d, table uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 " maxKey %" PRId64 " row key %" PRId64, TD_VID(pTsdb->pVnode), uid, now, minKey, maxKey, rowKey); - terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; - return -1; + return TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; } return 0; } +#if 0 int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { ASSERT(pMsg != NULL); // STsdbMeta * pMeta = pTsdb->tsdbMeta; @@ -165,4 +159,42 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { if (terrno != TSDB_CODE_SUCCESS) return -1; return 0; +} +#endif + +int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) { + int32_t code = 0; + STsdbKeepCfg *pCfg = &pTsdb->keepCfg; + TSKEY now = taosGetTimestamp(pCfg->precision); + TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2; + TSKEY maxKey = tsMaxKeyByPrecision[pCfg->precision]; + int32_t size = taosArrayGetSize(pMsg->aSubmitTbData); + + for (int32_t i = 0; i < size; ++i) { + SSubmitTbData *pData = TARRAY_GET_ELEM(pMsg->aSubmitTbData, i); + if (pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + uint64_t nColData = TARRAY_SIZE(pData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pData->aCol); + if (nColData > 0) { + int32_t nRows = aColData[0].nVal; + TSKEY *aKey = (TSKEY *)aColData[0].pData; + for (int32_t r = 0; r < nRows; ++r) { + if ((code = tsdbCheckRowRange(pTsdb, pData->uid, aKey[r], minKey, maxKey, now)) < 0) { + goto _exit; + } + } + } + } else { + int32_t nRows = taosArrayGetSize(pData->aRowP); + for (int32_t r = 0; r < nRows; ++r) { + SRow *pRow = (SRow *)taosArrayGetP(pData->aRowP, r); + if ((code = tsdbCheckRowRange(pTsdb, pData->uid, pRow->ts, minKey, maxKey, now)) < 0) { + goto _exit; + } + } + } + } + +_exit: + return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 83a414dae08bd48e41e12682922daf7055e1f5a0..80a422ca3b647ea2642a87e960e8f01ca24073c5 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -16,9 +16,7 @@ #include "vnd.h" /* ------------------------ STRUCTURES ------------------------ */ -#define VNODE_BUFPOOL_SEGMENTS 3 - -static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool) { +static int vnodeBufPoolCreate(SVnode *pVnode, int32_t id, int64_t size, SVBufPool **ppPool) { SVBufPool *pPool; pPool = taosMemoryMalloc(sizeof(SVBufPool) + size); @@ -26,6 +24,21 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool) terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + memset(pPool, 0, sizeof(SVBufPool)); + + // query handle list + taosThreadMutexInit(&pPool->mutex, NULL); + pPool->nQuery = 0; + pPool->qList.pNext = &pPool->qList; + pPool->qList.ppNext = &pPool->qList.pNext; + + pPool->pVnode = pVnode; + pPool->id = id; + pPool->ptr = pPool->node.data; + pPool->pTail = &pPool->node; + pPool->node.prev = NULL; + pPool->node.pnext = &pPool->pTail; + pPool->node.size = size; if (VND_IS_RSMA(pVnode)) { pPool->lock = taosMemoryMalloc(sizeof(TdThreadSpinlock)); @@ -44,16 +57,6 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool) pPool->lock = NULL; } - pPool->next = NULL; - pPool->pVnode = pVnode; - pPool->nRef = 0; - pPool->size = 0; - pPool->ptr = pPool->node.data; - pPool->pTail = &pPool->node; - pPool->node.prev = NULL; - pPool->node.pnext = &pPool->pTail; - pPool->node.size = size; - *ppPool = pPool; return 0; } @@ -64,27 +67,25 @@ static int vnodeBufPoolDestroy(SVBufPool *pPool) { taosThreadSpinDestroy(pPool->lock); taosMemoryFree((void *)pPool->lock); } + taosThreadMutexDestroy(&pPool->mutex); taosMemoryFree(pPool); return 0; } int vnodeOpenBufPool(SVnode *pVnode) { - SVBufPool *pPool = NULL; - int64_t size = pVnode->config.szBuf / VNODE_BUFPOOL_SEGMENTS; + int64_t size = pVnode->config.szBuf / VNODE_BUFPOOL_SEGMENTS; - ASSERT(pVnode->pPool == NULL); - - for (int i = 0; i < 3; i++) { + for (int i = 0; i < VNODE_BUFPOOL_SEGMENTS; i++) { // create pool - if (vnodeBufPoolCreate(pVnode, size, &pPool)) { + if (vnodeBufPoolCreate(pVnode, i, size, &pVnode->aBufPool[i])) { vError("vgId:%d, failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno)); vnodeCloseBufPool(pVnode); return -1; } - // add pool to vnode - pPool->next = pVnode->pPool; - pVnode->pPool = pPool; + // add to free list + pVnode->aBufPool[i]->freeNext = pVnode->freeList; + pVnode->freeList = pVnode->aBufPool[i]; } vDebug("vgId:%d, vnode buffer pool is opened, size:%" PRId64, TD_VID(pVnode), size); @@ -92,23 +93,19 @@ int vnodeOpenBufPool(SVnode *pVnode) { } int vnodeCloseBufPool(SVnode *pVnode) { - SVBufPool *pPool; - - for (pPool = pVnode->pPool; pPool; pPool = pVnode->pPool) { - pVnode->pPool = pPool->next; - vnodeBufPoolDestroy(pPool); + for (int32_t i = 0; i < VNODE_BUFPOOL_SEGMENTS; i++) { + if (pVnode->aBufPool[i]) { + vnodeBufPoolDestroy(pVnode->aBufPool[i]); + pVnode->aBufPool[i] = NULL; + } } - if (pVnode->inUse) { - vnodeBufPoolDestroy(pVnode->inUse); - pVnode->inUse = NULL; - } vDebug("vgId:%d, vnode buffer pool is closed", TD_VID(pVnode)); - return 0; } void vnodeBufPoolReset(SVBufPool *pPool) { + ASSERT(pPool->nQuery == 0); for (SVBufPoolNode *pNode = pPool->pTail; pNode->prev; pNode = pPool->pTail) { ASSERT(pNode->pnext == &pPool->pTail); pNode->prev->pnext = &pPool->pTail; @@ -215,35 +212,121 @@ void vnodeBufPoolRef(SVBufPool *pPool) { ASSERT(nRef > 0); } -void vnodeBufPoolUnRef(SVBufPool *pPool) { - if (pPool == NULL) { - return; - } - int32_t nRef = atomic_sub_fetch_32(&pPool->nRef, 1); - if (nRef == 0) { - SVnode *pVnode = pPool->pVnode; - - vnodeBufPoolReset(pPool); - - taosThreadMutexLock(&pVnode->mutex); - - int64_t size = pVnode->config.szBuf / VNODE_BUFPOOL_SEGMENTS; - if (pPool->node.size != size) { - SVBufPool *pPoolT = NULL; - if (vnodeBufPoolCreate(pVnode, size, &pPoolT) < 0) { - vWarn("vgId:%d, try to change buf pools size from %" PRId64 " to %" PRId64 " since %s", TD_VID(pVnode), - pPool->node.size, size, tstrerror(errno)); - } else { - vnodeBufPoolDestroy(pPool); - pPool = pPoolT; - vDebug("vgId:%d, change buf pools size from %" PRId64 " to %" PRId64, TD_VID(pVnode), pPool->node.size, size); - } +void vnodeBufPoolAddToFreeList(SVBufPool *pPool) { + SVnode *pVnode = pPool->pVnode; + + int64_t size = pVnode->config.szBuf / VNODE_BUFPOOL_SEGMENTS; + if (pPool->node.size != size) { + SVBufPool *pNewPool = NULL; + if (vnodeBufPoolCreate(pVnode, pPool->id, size, &pNewPool) < 0) { + vWarn("vgId:%d failed to change buffer pool of id %d size from %" PRId64 " to %" PRId64 " since %s", + TD_VID(pVnode), pPool->id, pPool->node.size, size, tstrerror(errno)); + } else { + vInfo("vgId:%d buffer pool of id %d size changed from %" PRId64 " to %" PRId64, TD_VID(pVnode), pPool->id, + pPool->node.size, size); + + vnodeBufPoolDestroy(pPool); + pPool = pNewPool; + pVnode->aBufPool[pPool->id] = pPool; } + } + + // add to free list + vDebug("vgId:%d buffer pool %p of id %d is added to free list", TD_VID(pVnode), pPool, pPool->id); + vnodeBufPoolReset(pPool); + pPool->freeNext = pVnode->freeList; + pVnode->freeList = pPool; + taosThreadCondSignal(&pVnode->poolNotEmpty); +} + +void vnodeBufPoolUnRef(SVBufPool *pPool, bool proactive) { + if (pPool == NULL) return; - pPool->next = pVnode->pPool; - pVnode->pPool = pPool; - taosThreadCondSignal(&pVnode->poolNotEmpty); + SVnode *pVnode = pPool->pVnode; - taosThreadMutexUnlock(&pVnode->mutex); + if (proactive) taosThreadMutexLock(&pVnode->mutex); + + if (atomic_sub_fetch_32(&pPool->nRef, 1) > 0) goto _exit; + + // remove from recycle queue or on-recycle position + if (pVnode->onRecycle == pPool) { + pVnode->onRecycle = NULL; + } else { + ASSERT(proactive); + + if (pPool->recyclePrev) { + pPool->recyclePrev->recycleNext = pPool->recycleNext; + } else { + pVnode->recycleHead = pPool->recycleNext; + } + + if (pPool->recycleNext) { + pPool->recycleNext->recyclePrev = pPool->recyclePrev; + } else { + pVnode->recycleTail = pPool->recyclePrev; + } + pPool->recyclePrev = pPool->recycleNext = NULL; } + + vnodeBufPoolAddToFreeList(pPool); + +_exit: + if (proactive) taosThreadMutexUnlock(&pVnode->mutex); + return; +} + +int32_t vnodeBufPoolRegisterQuery(SVBufPool *pPool, SQueryNode *pQNode) { + int32_t code = 0; + + taosThreadMutexLock(&pPool->mutex); + + pQNode->pNext = pPool->qList.pNext; + pQNode->ppNext = &pPool->qList.pNext; + pPool->qList.pNext->ppNext = &pQNode->pNext; + pPool->qList.pNext = pQNode; + pPool->nQuery++; + + taosThreadMutexUnlock(&pPool->mutex); + +_exit: + return code; } + +void vnodeBufPoolDeregisterQuery(SVBufPool *pPool, SQueryNode *pQNode, bool proactive) { + int32_t code = 0; + + if (proactive) taosThreadMutexLock(&pPool->mutex); + + pQNode->pNext->ppNext = pQNode->ppNext; + *pQNode->ppNext = pQNode->pNext; + pPool->nQuery--; + + if (proactive) taosThreadMutexUnlock(&pPool->mutex); +} + +int32_t vnodeBufPoolRecycle(SVBufPool *pPool) { + int32_t code = 0; + + SVnode *pVnode = pPool->pVnode; + + vDebug("vgId:%d recycle buffer pool %p of id %d", TD_VID(pVnode), pPool, pPool->id); + + taosThreadMutexLock(&pPool->mutex); + + SQueryNode *pNode = pPool->qList.pNext; + while (pNode != &pPool->qList) { + SQueryNode *pTNode = pNode->pNext; + + int32_t rc = pNode->reseek(pNode->pQHandle); + if (rc == 0 || rc == TSDB_CODE_VND_QUERY_BUSY) { + pNode = pTNode; + } else { + code = rc; + goto _exit; + } + } + +_exit: + taosThreadMutexUnlock(&pPool->mutex); + return code; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 69af536a4454909be7816ebda880e99e648656c3..c326c8bfacbd897fc15527edaef5cd8e77c40f5f 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -262,9 +262,10 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { tjsonGetNumberValue(info, "nodePort", pNode->nodePort, code); if (code < 0) return -1; tjsonGetStringValue(info, "nodeFqdn", pNode->nodeFqdn); - if (code < 0) return -1; tjsonGetNumberValue(info, "nodeId", pNode->nodeId, code); + if (code < 0) return -1; tjsonGetNumberValue(info, "clusterId", pNode->clusterId, code); + if (code < 0) return -1; vDebug("vgId:%d, decode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index b94d44eb7da10202dc816dbfb37a2fa58eae2db3..6a6fbcff4dedd15599166bd1078468602bae4489 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -21,41 +21,128 @@ static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData); static int vnodeCommitImpl(SCommitInfo *pInfo); -int vnodeBegin(SVnode *pVnode) { - // alloc buffer pool - taosThreadMutexLock(&pVnode->mutex); +#define WAIT_TIME_MILI_SEC 10 // miliseconds + +static int32_t vnodeTryRecycleBufPool(SVnode *pVnode) { + int32_t code = 0; + + if (pVnode->onRecycle == NULL) { + if (pVnode->recycleHead == NULL) { + vDebug("vgId:%d no recyclable buffer pool", TD_VID(pVnode)); + goto _exit; + } else { + vDebug("vgId:%d buffer pool %p of id %d on recycle queue, try to recycle", TD_VID(pVnode), pVnode->recycleHead, + pVnode->recycleHead->id); + + pVnode->onRecycle = pVnode->recycleHead; + if (pVnode->recycleHead == pVnode->recycleTail) { + pVnode->recycleHead = pVnode->recycleTail = NULL; + } else { + pVnode->recycleHead = pVnode->recycleHead->recycleNext; + pVnode->recycleHead->recyclePrev = NULL; + } + pVnode->onRecycle->recycleNext = pVnode->onRecycle->recyclePrev = NULL; + } + } - while (pVnode->pPool == NULL) { - taosThreadCondWait(&pVnode->poolNotEmpty, &pVnode->mutex); + code = vnodeBufPoolRecycle(pVnode->onRecycle); + if (code) goto _exit; + +_exit: + if (code) { + vError("vgId:%d %s failed since %s", TD_VID(pVnode), __func__, tstrerror(code)); } + return code; +} +static int32_t vnodeGetBufPoolToUse(SVnode *pVnode) { + int32_t code = 0; + int32_t lino = 0; - pVnode->inUse = pVnode->pPool; - pVnode->inUse->nRef = 1; - pVnode->pPool = pVnode->inUse->next; - pVnode->inUse->next = NULL; + taosThreadMutexLock(&pVnode->mutex); + + int32_t nTry = 0; + for (;;) { + ++nTry; + + if (pVnode->freeList) { + vDebug("vgId:%d allocate free buffer pool on %d try, pPool:%p id:%d", TD_VID(pVnode), nTry, pVnode->freeList, + pVnode->freeList->id); + + pVnode->inUse = pVnode->freeList; + pVnode->inUse->nRef = 1; + pVnode->freeList = pVnode->inUse->freeNext; + pVnode->inUse->freeNext = NULL; + break; + } else { + vDebug("vgId:%d no free buffer pool on %d try, try to recycle...", TD_VID(pVnode), nTry); + /* + code = vnodeTryRecycleBufPool(pVnode); + TSDB_CHECK_CODE(code, lino, _exit); + */ + if (pVnode->freeList == NULL) { + vDebug("vgId:%d no free buffer pool on %d try, wait %d ms...", TD_VID(pVnode), nTry, WAIT_TIME_MILI_SEC); + + struct timeval tv; + struct timespec ts; + taosGetTimeOfDay(&tv); + ts.tv_nsec = tv.tv_usec * 1000 + WAIT_TIME_MILI_SEC * 1000000; + if (ts.tv_nsec > 999999999l) { + ts.tv_sec = tv.tv_sec + 1; + ts.tv_nsec -= 1000000000l; + } else { + ts.tv_sec = tv.tv_sec; + } + + int32_t rc = taosThreadCondTimedWait(&pVnode->poolNotEmpty, &pVnode->mutex, &ts); + if (rc && rc != ETIMEDOUT) { + code = TAOS_SYSTEM_ERROR(rc); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + } + } +_exit: taosThreadMutexUnlock(&pVnode->mutex); + if (code) { + vError("vgId:%d %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } + return code; +} +int vnodeBegin(SVnode *pVnode) { + int32_t code = 0; + int32_t lino = 0; pVnode->state.commitID++; + + // alloc buffer pool + code = vnodeGetBufPoolToUse(pVnode); + TSDB_CHECK_CODE(code, lino, _exit); + // begin meta if (metaBegin(pVnode->pMeta, META_BEGIN_HEAP_BUFFERPOOL) < 0) { - vError("vgId:%d, failed to begin meta since %s", TD_VID(pVnode), tstrerror(terrno)); - return -1; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } // begin tsdb if (tsdbBegin(pVnode->pTsdb) < 0) { - vError("vgId:%d, failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); - return -1; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } // begin sma if (VND_IS_RSMA(pVnode) && smaBegin(pVnode->pSma) < 0) { - vError("vgId:%d, failed to begin sma since %s", TD_VID(pVnode), tstrerror(terrno)); - return -1; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } - return 0; +_exit: + if (code) { + terrno = code; + vError("vgId:%d %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } + return code; } void vnodeUpdCommitSched(SVnode *pVnode) { @@ -70,7 +157,7 @@ int vnodeShouldCommit(SVnode *pVnode) { } SVCommitSched *pSched = &pVnode->commitSched; - int64_t nowMs = taosGetMonoTimestampMs(); + int64_t nowMs = taosGetMonoTimestampMs(); return (((pVnode->inUse->size > pVnode->inUse->node.size) && (pSched->commitMs + SYNC_VND_COMMIT_MIN_MS < nowMs)) || (pVnode->inUse->size > 0 && pSched->commitMs + pSched->maxWaitMs < nowMs)); @@ -209,6 +296,12 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) { tsem_wait(&pVnode->canCommit); + taosThreadMutexLock(&pVnode->mutex); + ASSERT(pVnode->onCommit == NULL); + pVnode->onCommit = pVnode->inUse; + pVnode->inUse = NULL; + taosThreadMutexUnlock(&pVnode->mutex); + pVnode->state.commitTerm = pVnode->state.applyTerm; pInfo->info.config = pVnode->config; @@ -232,12 +325,11 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) { } tsdbPrepareCommit(pVnode->pTsdb); - smaPrepareAsyncCommit(pVnode->pSma); metaPrepareAsyncCommit(pVnode->pMeta); - vnodeBufPoolUnRef(pVnode->inUse); - pVnode->inUse = NULL; + code = smaPrepareAsyncCommit(pVnode->pSma); + if (code) goto _exit; _exit: if (code) { @@ -246,22 +338,51 @@ _exit: } else { vDebug("vgId:%d, %s done", TD_VID(pVnode), __func__); } + return code; } +static void vnodeReturnBufPool(SVnode *pVnode) { + taosThreadMutexLock(&pVnode->mutex); + SVBufPool *pPool = pVnode->onCommit; + int32_t nRef = atomic_sub_fetch_32(&pPool->nRef, 1); + + pVnode->onCommit = NULL; + if (nRef == 0) { + vnodeBufPoolAddToFreeList(pPool); + } else if (nRef > 0) { + vDebug("vgId:%d buffer pool %p of id %d is added to recycle queue", TD_VID(pVnode), pPool, pPool->id); + + if (pVnode->recycleTail == NULL) { + pPool->recyclePrev = pPool->recycleNext = NULL; + pVnode->recycleHead = pVnode->recycleTail = pPool; + } else { + pPool->recyclePrev = pVnode->recycleTail; + pPool->recycleNext = NULL; + pVnode->recycleTail->recycleNext = pPool; + pVnode->recycleTail = pPool; + } + } else { + ASSERT(0); + } + + taosThreadMutexUnlock(&pVnode->mutex); +} static int32_t vnodeCommitTask(void *arg) { int32_t code = 0; SCommitInfo *pInfo = (SCommitInfo *)arg; + SVnode *pVnode = pInfo->pVnode; // commit code = vnodeCommitImpl(pInfo); if (code) goto _exit; - // end commit - tsem_post(&pInfo->pVnode->canCommit); + vnodeReturnBufPool(pVnode); _exit: + // end commit + tsem_post(&pVnode->canCommit); taosMemoryFree(pInfo); return code; } @@ -282,14 +403,15 @@ int vnodeAsyncCommit(SVnode *pVnode) { } // schedule the task - vnodeScheduleTask(vnodeCommitTask, pInfo); + code = vnodeScheduleTask(vnodeCommitTask, pInfo); _exit: if (code) { if (NULL != pInfo) { taosMemoryFree(pInfo); } - vError("vgId:%d, vnode async commit failed since %s, commitId:%" PRId64, TD_VID(pVnode), tstrerror(code), + tsem_post(&pVnode->canCommit); + vError("vgId:%d, %s failed since %s, commit id:%" PRId64, TD_VID(pVnode), __func__, tstrerror(code), pVnode->state.commitID); } else { vInfo("vgId:%d, vnode async commit done, commitId:%" PRId64 " term:%" PRId64 " applied:%" PRId64, TD_VID(pVnode), diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 58d9f1a0499b90a463fed10b65497a0a41b47098..743ae17f2b87940cdd381f3ac2ebdb443f4e2e46 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -240,7 +240,7 @@ _err: if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb); if (pVnode->pSma) smaClose(pVnode->pSma); if (pVnode->pMeta) metaClose(pVnode->pMeta); - if (pVnode->pPool) vnodeCloseBufPool(pVnode); + if (pVnode->freeList) vnodeCloseBufPool(pVnode); tsem_destroy(&(pVnode->canCommit)); taosMemoryFree(pVnode); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 1199127f6d42bf2d3f7488fb120e8fa9ab09c490..8d4e70cff987c69f7b57c505c8f557bc73b70609 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -15,13 +15,13 @@ #include "vnd.h" -#define VNODE_GET_LOAD_RESET_VALS(pVar, oVal, vType, tags) \ - do { \ - int##vType##_t newVal = atomic_sub_fetch_##vType(&(pVar), (oVal)); \ - ASSERT(newVal >= 0); \ - if (newVal < 0) { \ - vWarn("vgId:%d %s, abnormal val:%" PRIi64 ", old val:%" PRIi64, TD_VID(pVnode), tags, newVal, (oVal)); \ - } \ +#define VNODE_GET_LOAD_RESET_VALS(pVar, oVal, vType, tags) \ + do { \ + int##vType##_t newVal = atomic_sub_fetch_##vType(&(pVar), (oVal)); \ + ASSERT(newVal >= 0); \ + if (newVal < 0) { \ + vWarn("vgId:%d, %s, abnormal val:%" PRIi64 ", old val:%" PRIi64, TD_VID(pVnode), tags, newVal, (oVal)); \ + } \ } while (0) int vnodeQueryOpen(SVnode *pVnode) { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 4cf6c4e55c7b8126cd4b999af58289625a9cad9b..6abc144b911720244e33b3553249b98b48550af7 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -77,52 +77,72 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { tDecoderClear(&dc); } break; case TDMT_VND_SUBMIT: { - SSubmitMsgIter msgIter = {0}; - SSubmitReq *pSubmitReq = (SSubmitReq *)pMsg->pCont; - SSubmitBlk *pBlock = NULL; - int64_t ctime = taosGetTimestampMs(); - tb_uid_t uid; - - if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { - code = terrno; + int64_t ctime = taosGetTimestampMs(); + + tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); + tStartDecode(&dc); + + uint64_t nSubmitTbData; + if (tDecodeU64v(&dc, &nSubmitTbData) < 0) { + code = TSDB_CODE_INVALID_MSG; goto _err; } - for (;;) { - tGetSubmitMsgNext(&msgIter, &pBlock); - if (pBlock == NULL) break; + for (int32_t i = 0; i < nSubmitTbData; i++) { + if (tStartDecode(&dc) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _err; + } - if (msgIter.schemaLen > 0) { - char *name = NULL; + int32_t flags; + if (tDecodeI32v(&dc, &flags) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _err; + } - tDecoderInit(&dc, pBlock->data, msgIter.schemaLen); + if (flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { + // SVCreateTbReq if (tStartDecode(&dc) < 0) { code = TSDB_CODE_INVALID_MSG; - return code; + goto _err; } if (tDecodeI32v(&dc, NULL) < 0) { code = TSDB_CODE_INVALID_MSG; - return code; + goto _err; } + + char *name = NULL; if (tDecodeCStr(&dc, &name) < 0) { code = TSDB_CODE_INVALID_MSG; - return code; + goto _err; } - uid = metaGetTableEntryUidByName(pVnode->pMeta, name); + int64_t uid = metaGetTableEntryUidByName(pVnode->pMeta, name); if (uid == 0) { uid = tGenIdPI64(); } + *(int64_t *)(dc.data + dc.pos) = uid; *(int64_t *)(dc.data + dc.pos + 8) = ctime; - pBlock->uid = htobe64(uid); tEndDecode(&dc); - tDecoderClear(&dc); + + // SSubmitTbData + int64_t suid; + if (tDecodeI64(&dc, &suid) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _err; + } + + *(int64_t *)(dc.data + dc.pos) = uid; } + + tEndDecode(&dc); } + tEndDecode(&dc); + tDecoderClear(&dc); } break; case TDMT_VND_DELETE: { int32_t size; @@ -171,13 +191,13 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp void *pReq; int32_t len; int32_t ret; - + /* if (!pVnode->inUse) { terrno = TSDB_CODE_VND_NO_AVAIL_BUFPOOL; vError("vgId:%d, not ready to write since %s", TD_VID(pVnode), terrstr()); return -1; } - + */ if (version <= pVnode->state.applied) { vError("vgId:%d, duplicate write request. version: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), version, pVnode->state.applied); @@ -236,7 +256,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp break; /* TSDB */ case TDMT_VND_SUBMIT: - if (vnodeProcessSubmitReq(pVnode, version, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err; + if (vnodeProcessSubmitReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DELETE: if (vnodeProcessDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; @@ -320,7 +340,10 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp // commit if need if (needCommit) { vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); - vnodeAsyncCommit(pVnode); + if (vnodeAsyncCommit(pVnode) < 0) { + vError("vgId:%d, failed to vnode async commit since %s.", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } // start a new one if (vnodeBegin(pVnode) < 0) { @@ -851,57 +874,199 @@ static int32_t vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, return TSDB_CODE_SUCCESS; } -static int32_t vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char *tags) { - ASSERT(pMsg != NULL); - SSubmitMsgIter msgIter = {0}; - SMeta *pMeta = pVnode->pMeta; - SSubmitBlk *pBlock = NULL; +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +#if 1 + int32_t code = 0; + terrno = 0; - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; - while (true) { - if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; - if (pBlock == NULL) break; + SSubmitReq2 *pSubmitReq = &(SSubmitReq2){0}; + SSubmitRsp2 *pSubmitRsp = &(SSubmitRsp2){0}; + SArray *newTbUids = NULL; + int32_t ret; + SEncoder ec = {0}; + + pRsp->code = TSDB_CODE_SUCCESS; + + // decode + SDecoder dc = {0}; + tDecoderInit(&dc, pReq, len); + if (tDecodeSSubmitReq2(&dc, pSubmitReq) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + tDecoderClear(&dc); - vnodeDebugPrintSingleSubmitMsg(pMeta, pBlock, &msgIter, tags); + // check + code = tsdbScanAndConvertSubmitMsg(pVnode->pTsdb, pSubmitReq); + if (code) { + goto _exit; } - return 0; -} + for (int32_t i = 0; i < TARRAY_SIZE(pSubmitReq->aSubmitTbData); ++i) { + SSubmitTbData *pSubmitTbData = taosArrayGet(pSubmitReq->aSubmitTbData, i); -static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; - SSubmitRsp submitRsp = {0}; - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SVCreateTbReq createTbReq = {0}; - SDecoder decoder = {0}; - int32_t nRows = 0; - int32_t tsize, ret; - SEncoder encoder = {0}; - SArray *newTbUids = NULL; - SVStatis statis = {0}; - bool tbCreated = false; + if (pSubmitTbData->pCreateTbReq) { + pSubmitTbData->uid = pSubmitTbData->pCreateTbReq->uid; + } else { + SMetaInfo info = {0}; + + code = metaGetInfo(pVnode->pMeta, pSubmitTbData->uid, &info, NULL); + if (code) { + code = TSDB_CODE_TDB_TABLE_NOT_EXIST; + vWarn("vgId:%d, table uid:%" PRId64 " not exists", TD_VID(pVnode), pSubmitTbData->uid); + goto _exit; + } + + if (info.suid != pSubmitTbData->suid) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (info.suid) { + metaGetInfo(pVnode->pMeta, info.suid, &info, NULL); + } + + if (pSubmitTbData->sver != info.skmVer) { + code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; + goto _exit; + } + } + + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + int32_t nColData = TARRAY_SIZE(pSubmitTbData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pSubmitTbData->aCol); + + if (nColData <= 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (aColData[0].cid != PRIMARYKEY_TIMESTAMP_COL_ID || aColData[0].type != TSDB_DATA_TYPE_TIMESTAMP || + aColData[0].nVal <= 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + for (int32_t i = 1; i < nColData; i++) { + if (aColData[i].nVal != aColData[0].nVal) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + } + } + } + + vDebug("vgId:%d, submit block size %d", TD_VID(pVnode), (int32_t)taosArrayGetSize(pSubmitReq->aSubmitTbData)); + + // loop to handle + for (int32_t i = 0; i < TARRAY_SIZE(pSubmitReq->aSubmitTbData); ++i) { + SSubmitTbData *pSubmitTbData = taosArrayGet(pSubmitReq->aSubmitTbData, i); + + // create table + if (pSubmitTbData->pCreateTbReq) { + // check (TODO: move check to create table) + code = grantCheck(TSDB_GRANT_TIMESERIES); + if (code) goto _exit; + + code = grantCheck(TSDB_GRANT_TABLE); + if (code) goto _exit; + + // alloc if need + if (pSubmitRsp->aCreateTbRsp == NULL && + (pSubmitRsp->aCreateTbRsp = taosArrayInit(TARRAY_SIZE(pSubmitReq->aSubmitTbData), sizeof(SVCreateTbRsp))) == + NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + SVCreateTbRsp *pCreateTbRsp = taosArrayReserve(pSubmitRsp->aCreateTbRsp, 1); + + // create table + if (metaCreateTable(pVnode->pMeta, version, pSubmitTbData->pCreateTbReq, &pCreateTbRsp->pMeta) == + 0) { // create table success + + if (newTbUids == NULL && + (newTbUids = taosArrayInit(TARRAY_SIZE(pSubmitReq->aSubmitTbData), sizeof(int64_t))) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + taosArrayPush(newTbUids, &pSubmitTbData->uid); + + if (pCreateTbRsp->pMeta) { + vnodeUpdateMetaRsp(pVnode, pCreateTbRsp->pMeta); + } + } else { // create table failed + if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { + code = terrno; + goto _exit; + } + pSubmitTbData->uid = pSubmitTbData->pCreateTbReq->uid; // update uid if table exist for using below + } + } + + // insert data + int32_t affectedRows; + code = tsdbInsertTableData(pVnode->pTsdb, version, pSubmitTbData, &affectedRows); + if (code) goto _exit; + + pSubmitRsp->affectedRows += affectedRows; + } + + // update table uid list + if (taosArrayGetSize(newTbUids) > 0) { + vDebug("vgId:%d, add %d table into query table list in handling submit", TD_VID(pVnode), + (int32_t)taosArrayGetSize(newTbUids)); + tqUpdateTbUidList(pVnode->pTq, newTbUids, true); + } + +_exit: + // message + pRsp->code = code; + tEncodeSize(tEncodeSSubmitRsp2, pSubmitRsp, pRsp->contLen, ret); + pRsp->pCont = rpcMallocCont(pRsp->contLen); + tEncoderInit(&ec, pRsp->pCont, pRsp->contLen); + tEncodeSSubmitRsp2(&ec, pSubmitRsp); + tEncoderClear(&ec); + + // update statistics + atomic_add_fetch_64(&pVnode->statis.nInsert, pSubmitRsp->affectedRows); + atomic_add_fetch_64(&pVnode->statis.nInsertSuccess, pSubmitRsp->affectedRows); + atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1); + if (code == 0) { + atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1); + tdProcessRSmaSubmit(pVnode->pSma, version, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT); + } + + // clear + taosArrayDestroy(newTbUids); + tDestroySSubmitReq2(pSubmitReq, TSDB_MSG_FLG_DECODE); + tDestroySSubmitRsp2(pSubmitRsp, TSDB_MSG_FLG_ENCODE); + + if (code) terrno = code; + + return code; + +#else + SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; + SSubmitRsp submitRsp = {0}; + int32_t nRows = 0; + int32_t tsize, ret; + SEncoder encoder = {0}; + SArray *newTbUids = NULL; + SVStatis statis = {0}; + bool tbCreated = false; terrno = TSDB_CODE_SUCCESS; pRsp->code = 0; pSubmitReq->version = version; statis.nBatchInsert = 1; -#ifdef TD_DEBUG_PRINT_ROW - vnodeDebugPrintSubmitMsg(pVnode, pReq, __func__); -#endif - if (tsdbScanAndConvertSubmitMsg(pVnode->pTsdb, pSubmitReq) < 0) { pRsp->code = terrno; goto _exit; } - // handle the request - if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { - pRsp->code = TSDB_CODE_INVALID_MSG; - goto _exit; - } - submitRsp.pArray = taosArrayInit(msgIter.numOfBlocks, sizeof(SSubmitBlkRsp)); newTbUids = taosArrayInit(msgIter.numOfBlocks, sizeof(int64_t)); if (!submitRsp.pArray || !newTbUids) { @@ -918,42 +1083,42 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq // create table for auto create table mode if (msgIter.schemaLen > 0) { - tDecoderInit(&decoder, pBlock->data, msgIter.schemaLen); - if (tDecodeSVCreateTbReq(&decoder, &createTbReq) < 0) { - pRsp->code = TSDB_CODE_INVALID_MSG; - tDecoderClear(&decoder); - taosArrayDestroy(createTbReq.ctb.tagName); - goto _exit; - } - - if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0) { - pRsp->code = terrno; - tDecoderClear(&decoder); - taosArrayDestroy(createTbReq.ctb.tagName); - goto _exit; - } - - if ((terrno = grantCheck(TSDB_GRANT_TABLE)) < 0) { - pRsp->code = terrno; - tDecoderClear(&decoder); - taosArrayDestroy(createTbReq.ctb.tagName); - goto _exit; - } + // tDecoderInit(&decoder, pBlock->data, msgIter.schemaLen); + // if (tDecodeSVCreateTbReq(&decoder, &createTbReq) < 0) { + // pRsp->code = TSDB_CODE_INVALID_MSG; + // tDecoderClear(&decoder); + // taosArrayDestroy(createTbReq.ctb.tagName); + // goto _exit; + // } + + // if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0) { + // pRsp->code = terrno; + // tDecoderClear(&decoder); + // taosArrayDestroy(createTbReq.ctb.tagName); + // goto _exit; + // } + + // if ((terrno = grantCheck(TSDB_GRANT_TABLE)) < 0) { + // pRsp->code = terrno; + // tDecoderClear(&decoder); + // taosArrayDestroy(createTbReq.ctb.tagName); + // goto _exit; + // } if (metaCreateTable(pVnode->pMeta, version, &createTbReq, &submitBlkRsp.pMeta) < 0) { - if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { - submitBlkRsp.code = terrno; - pRsp->code = terrno; - tDecoderClear(&decoder); - taosArrayDestroy(createTbReq.ctb.tagName); - goto _exit; - } + // if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { + // submitBlkRsp.code = terrno; + // pRsp->code = terrno; + // tDecoderClear(&decoder); + // taosArrayDestroy(createTbReq.ctb.tagName); + // goto _exit; + // } } else { if (NULL != submitBlkRsp.pMeta) { vnodeUpdateMetaRsp(pVnode, submitBlkRsp.pMeta); } - taosArrayPush(newTbUids, &createTbReq.uid); + // taosArrayPush(newTbUids, &createTbReq.uid); submitBlkRsp.uid = createTbReq.uid; submitBlkRsp.tblFName = taosMemoryMalloc(strlen(pVnode->config.dbname) + strlen(createTbReq.name) + 2); @@ -961,18 +1126,15 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq tbCreated = true; } - msgIter.uid = createTbReq.uid; - if (createTbReq.type == TSDB_CHILD_TABLE) { - msgIter.suid = createTbReq.ctb.suid; - } else { - msgIter.suid = 0; - } + // msgIter.uid = createTbReq.uid; + // if (createTbReq.type == TSDB_CHILD_TABLE) { + // msgIter.suid = createTbReq.ctb.suid; + // } else { + // msgIter.suid = 0; + // } -#ifdef TD_DEBUG_PRINT_ROW - vnodeDebugPrintSingleSubmitMsg(pVnode->pMeta, pBlock, &msgIter, "real uid"); -#endif - tDecoderClear(&decoder); - taosArrayDestroy(createTbReq.ctb.tagName); + // tDecoderClear(&decoder); + // taosArrayDestroy(createTbReq.ctb.tagName); } if (tsdbInsertTableData(pVnode->pTsdb, version, &msgIter, pBlock, &submitBlkRsp) < 0) { @@ -986,21 +1148,21 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq } } - if (taosArrayGetSize(newTbUids) > 0) { - vDebug("vgId:%d, add %d table into query table list in handling submit", TD_VID(pVnode), - (int32_t)taosArrayGetSize(newTbUids)); - } + // if (taosArrayGetSize(newTbUids) > 0) { + // vDebug("vgId:%d, add %d table into query table list in handling submit", TD_VID(pVnode), + // (int32_t)taosArrayGetSize(newTbUids)); + // } - tqUpdateTbUidList(pVnode->pTq, newTbUids, true); + // tqUpdateTbUidList(pVnode->pTq, newTbUids, true); _exit: taosArrayDestroy(newTbUids); - tEncodeSize(tEncodeSSubmitRsp, &submitRsp, tsize, ret); - pRsp->pCont = rpcMallocCont(tsize); - pRsp->contLen = tsize; - tEncoderInit(&encoder, pRsp->pCont, tsize); - tEncodeSSubmitRsp(&encoder, &submitRsp); - tEncoderClear(&encoder); + // tEncodeSize(tEncodeSSubmitRsp, &submitRsp, tsize, ret); + // pRsp->pCont = rpcMallocCont(tsize); + // pRsp->contLen = tsize; + // tEncoderInit(&encoder, pRsp->pCont, tsize); + // tEncodeSSubmitRsp(&encoder, &submitRsp); + // tEncoderClear(&encoder); taosArrayDestroyEx(submitRsp.pArray, tFreeSSubmitBlkRsp); @@ -1021,6 +1183,8 @@ _exit: vDebug("vgId:%d, submit success, index:%" PRId64, pVnode->config.vgId, version); return 0; +#endif + return 0; } static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 3e8300e05dbbb54e6000cbccd799f9715f9e2aca..836ce87fbb7bb0620567be05030527c4ca7ac101 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -582,34 +582,34 @@ typedef struct SCtgOperation { #define CTG_LOCK(type, _lock) \ do { \ if (CTG_READ == (type)) { \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before read lock"); \ CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRLockLatch(_lock); \ CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) > 0); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value after read lock"); \ } else { \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before write lock"); \ CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWLockLatch(_lock); \ CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value after write lock"); \ } \ } while (0) #define CTG_UNLOCK(type, _lock) \ do { \ if (CTG_READ == (type)) { \ - assert(atomic_load_32((_lock)) > 0); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value before read unlock"); \ CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRUnLockLatch(_lock); \ CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after read unlock"); \ } else { \ - assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value before write unlock"); \ CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWUnLockLatch(_lock); \ CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after write unlock"); \ } \ } while (0) diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index b6ff1c8b987278ee8ef223577e91771c12cefde8..a6c0d1c40157f31f8002b11e7d4ce0e95a6f3f21 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -22,7 +22,6 @@ extern SCatalogMgmt gCtgMgmt; SCtgDebug gCTGDebug = {0}; void ctgdUserCallback(SMetaData *pResult, void *param, int32_t code) { - ASSERT(*(int32_t *)param == 1); taosMemoryFree(param); qDebug("async call result: %s", tstrerror(code)); diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 7cc6c90d303c4c425358800eb646f986417cefdd..3b037e206290ff767a1f879e0b46aadf8b3e4f20 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -40,7 +40,9 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu msgNum = taosArrayGetSize(batchRsp.pRsps); } - ASSERT(taskNum == msgNum || 0 == msgNum); + if (ASSERTS(taskNum == msgNum || 0 == msgNum, "taskNum %d mis-match msgNum %d", taskNum, msgNum)) { + msgNum = 0; + } ctgDebug("QID:0x%" PRIx64 " ctg got batch %d rsp %s", pJob->queryId, cbParam->batchId, TMSG_INFO(cbParam->reqType + 1)); @@ -58,11 +60,19 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu if (msgNum > 0) { pRsp = taosArrayGet(batchRsp.pRsps, i); - taskMsg.msgType = pRsp->reqType; - taskMsg.pData = pRsp->msg; - taskMsg.len = pRsp->msgLen; - - ASSERT(pRsp->msgIdx == *msgIdx); + if (ASSERTS(pRsp->msgIdx == *msgIdx, "rsp msgIdx %d mis-match msgIdx %d", pRsp->msgIdx, *msgIdx)) { + pRsp = &rsp; + pRsp->msgIdx = *msgIdx; + pRsp->reqType = -1; + pRsp->rspCode = 0; + taskMsg.msgType = -1; + taskMsg.pData = NULL; + taskMsg.len = 0; + } else { + taskMsg.msgType = pRsp->reqType; + taskMsg.pData = pRsp->msg; + taskMsg.len = pRsp->msgLen; + } } else { pRsp = &rsp; pRsp->msgIdx = *msgIdx; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index a179ec24f91cf7b5564862796633d4ad4f965f85..c87f6953eb5fbb2515b2db54ab1e85854111eeb2 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -21,6 +21,7 @@ #include "tdatablock.h" #include "tglobal.h" #include "tgrant.h" +#include "taosdef.h" extern SConfig* tsCfg; @@ -40,7 +41,6 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->numOfCols = htonl(numOfCols); int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols); - ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); return TSDB_CODE_SUCCESS; } @@ -154,6 +154,23 @@ static int32_t buildCreateDBResultDataBlock(SSDataBlock** pOutput) { return code; } +static int32_t buildAliveResultDataBlock(SSDataBlock** pOutput) { + SSDataBlock* pBlock = createDataBlock(); + if (NULL == pBlock) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1); + int32_t code = blockDataAppendColInfo(pBlock, &infoData); + + if (TSDB_CODE_SUCCESS == code) { + *pOutput = pBlock; + } else { + blockDataDestroy(pBlock); + } + return code; +} + int64_t getValOfDiffPrecision(int8_t unit, int64_t val) { int64_t v = 0; switch (unit) { @@ -281,6 +298,108 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, S colDataAppend(pCol2, 0, buf2, false); } +#define CHECK_LEADER(n) (row[n] && (fields[n].type == TSDB_DATA_TYPE_VARCHAR && strncasecmp(row[n], "leader", varDataLen((char *)row[n] - VARSTR_HEADER_SIZE)) == 0)) +// on this row, if have leader return true else return false +bool existLeaderRole(TAOS_ROW row, TAOS_FIELD* fields, int nFields) { + // vgroup_id | db_name | tables | v1_dnode | v1_status | v2_dnode | v2_status | v3_dnode | v3_status | v4_dnode | + // v4_status | cacheload | tsma | + if (nFields != 13) { + return false; + } + + // check have leader on cloumn v*_status on 4 6 8 10 + if (CHECK_LEADER(4) || CHECK_LEADER(6) || CHECK_LEADER(8) || CHECK_LEADER(10)) { + return true; + } + + return false; +} + +// get db alive status, return 1 is alive else return 0 +int32_t getAliveStatusFromApi(int64_t* pConnId, char* dbName, int32_t* pStatus) { + char sql[128 + TSDB_DB_NAME_LEN] = "select * from information_schema.ins_vgroups"; + int32_t code; + + // filter with db name + if (dbName && dbName[0] != 0) { + char str[64 + TSDB_DB_NAME_LEN] = ""; + // test db name exist + sprintf(str, "show create database %s ;", dbName); + TAOS_RES* dbRes = taos_query(pConnId, str); + code = taos_errno(dbRes); + if (code != TSDB_CODE_SUCCESS) { + taos_free_result(dbRes); + return code; + } + taos_free_result(dbRes); + + sprintf(str, " where db_name='%s' ;", dbName); + strcat(sql, str); + } + + TAOS_RES* res = taos_query(pConnId, sql); + code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + taos_free_result(res); + return code; + } + + TAOS_ROW row = NULL; + TAOS_FIELD* fields = taos_fetch_fields(res); + int32_t nFields = taos_num_fields(res); + int32_t nAvailble = 0; + int32_t nUnAvailble = 0; + + while ((row = taos_fetch_row(res)) != NULL) { + if (existLeaderRole(row, fields, nFields)) { + nAvailble++; + } else { + nUnAvailble++; + } + } + taos_free_result(res); + + int32_t status = 0; + if (nAvailble + nUnAvailble == 0 || nUnAvailble == 0) { + status = SHOW_STATUS_AVAILABLE; + } else if (nAvailble > 0 && nUnAvailble > 0) { + status = SHOW_STATUS_HALF_AVAILABLE; + } else { + status = SHOW_STATUS_NOT_AVAILABLE; + } + + if (pStatus) { + *pStatus = status; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t setAliveResultIntoDataBlock(int64_t* pConnId, SSDataBlock* pBlock, char* dbName) { + blockDataEnsureCapacity(pBlock, 1); + pBlock->info.rows = 1; + + SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0); + int32_t status = 0; + int32_t code = getAliveStatusFromApi(pConnId, dbName, &status); + if (code == TSDB_CODE_SUCCESS) { + colDataAppend(pCol1, 0, (const char*)&status, false); + } + return code; +} + +static int32_t execShowAliveStatus(int64_t* pConnId, SShowAliveStmt* pStmt, SRetrieveTableRsp** pRsp) { + SSDataBlock* pBlock = NULL; + int32_t code = buildAliveResultDataBlock(&pBlock); + if (TSDB_CODE_SUCCESS == code) { + code = setAliveResultIntoDataBlock(pConnId, pBlock, pStmt->dbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = buildRetrieveTableRsp(pBlock, SHOW_ALIVE_RESULT_COLS, pRsp); + } + blockDataDestroy(pBlock); + return code; +} + static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveTableRsp** pRsp) { SSDataBlock* pBlock = NULL; int32_t code = buildCreateDBResultDataBlock(&pBlock); @@ -736,7 +855,7 @@ static int32_t execSelectWithoutFrom(SSelectStmt* pSelect, SRetrieveTableRsp** p return code; } -int32_t qExecCommand(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp) { +int32_t qExecCommand(int64_t* pConnId, bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp) { switch (nodeType(pStmt)) { case QUERY_NODE_DESCRIBE_STMT: return execDescribe(sysInfoUser, pStmt, pRsp); @@ -754,6 +873,9 @@ int32_t qExecCommand(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp) { return execShowLocalVariables(pRsp); case QUERY_NODE_SELECT_STMT: return execSelectWithoutFrom((SSelectStmt*)pStmt, pRsp); + case QUERY_NODE_SHOW_DB_ALIVE_STMT: + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: + return execShowAliveStatus(pConnId, (SShowAliveStmt*)pStmt, pRsp); default: break; } diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 253718048dcebb71b64ca4d4b48b3004c6d9ea3a..38222112a19a447a4a7294a6082bf8bd9edd18ce 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1666,7 +1666,6 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->numOfRows = htobe64((int64_t)rowNum); int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock)); - ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); rsp->compLen = htonl(len); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index fffe687ff628f6c537328a9b89e880806dc4a154..c122a91d7106e2dcfdd1eedc711917d50d7060ea 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -126,12 +126,14 @@ enum { typedef struct { // TODO remove prepareStatus - STqOffsetVal prepareStatus; // for tmq - STqOffsetVal lastStatus; // for tmq - SMqMetaRsp metaRsp; // for tmq fetching meta - int8_t returned; - int64_t snapshotVer; - const SSubmitReq* pReq; + STqOffsetVal prepareStatus; // for tmq + STqOffsetVal lastStatus; // for tmq + SMqMetaRsp metaRsp; // for tmq fetching meta + int8_t returned; + int64_t snapshotVer; + // const SSubmitReq* pReq; + + SPackedData submit; SSchemaWrapper* schema; char tbName[TSDB_TABLE_NAME_LEN]; @@ -355,6 +357,7 @@ typedef struct STableMergeScanInfo { SLimitInfo limitInfo; int64_t numOfRows; SScanInfo scanInfo; + int32_t scanTimes; SSDataBlock* pResBlock; SSampleExecInfo sample; // sample execution info SSortExecInfo sortExecInfo; @@ -471,9 +474,11 @@ typedef struct SStreamScanInfo { SNode* pTagIndexCond; // recover - int32_t blockRecoverContiCnt; - int32_t blockRecoverTotCnt; + int32_t blockRecoverContiCnt; + int32_t blockRecoverTotCnt; + SSDataBlock* pRecoverRes; + SSDataBlock* pCreateTbRes; } SStreamScanInfo; typedef struct { @@ -564,6 +569,8 @@ typedef struct SStreamIntervalOperatorInfo { SStreamState* pState; SWinKey delKey; uint64_t numOfDatapack; + SArray* pUpdated; + SHashObj* pUpdatedMap; } SStreamIntervalOperatorInfo; typedef struct SDataGroupInfo { @@ -610,6 +617,8 @@ typedef struct SStreamSessionAggOperatorInfo { SPhysiNode* pPhyNode; // create new child bool isFinal; bool ignoreExpiredData; + SArray* pUpdated; + SSHashObj* pStUpdated; } SStreamSessionAggOperatorInfo; typedef struct SStreamStateAggOperatorInfo { @@ -625,6 +634,8 @@ typedef struct SStreamStateAggOperatorInfo { void* pDelIterator; SArray* pChildren; // cache for children's result; bool ignoreExpiredData; + SArray* pUpdated; + SSHashObj* pSeUpdated; } SStreamStateAggOperatorInfo; typedef struct SStreamPartitionOperatorInfo { @@ -635,9 +646,11 @@ typedef struct SStreamPartitionOperatorInfo { SExprSupp tagCalSup; SHashObj* pPartitions; void* parIte; + void* pTbNameIte; SSDataBlock* pInputDataBlock; int32_t tsColIndex; SSDataBlock* pDelRes; + SSDataBlock* pCreateTbRes; } SStreamPartitionOperatorInfo; typedef struct SStreamFillSupporter { @@ -759,7 +772,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream); +SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMergeIntervalPhysiNode* pIntervalPhyNode, SExecTaskInfo* pTaskInfo); @@ -835,7 +848,6 @@ bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pS void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp, void* pTbName); uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); -void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock); int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SExprSupp* pSup, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); @@ -854,6 +866,11 @@ void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t ord int32_t qAppendTaskStopInfo(SExecTaskInfo* pTaskInfo, SExchangeOpStopInfo* pInfo); int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, int32_t order, int64_t* pData); +void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, int64_t groupId, + SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock); + +SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag); +SExprInfo* createExpr(SNodeList* pNodeList, int32_t* numOfExprs); #ifdef __cplusplus } diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 99d1968a5de525c4f1f44c2b3ebd89dbabbe4c0d..f78e3c22e13c8b99885c451cd16f76ed038b79a1 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -25,9 +25,9 @@ extern SDataSinkStat gDataSinkStat; typedef struct SSubmitRes { - int64_t affectedRows; - int32_t code; - SSubmitRsp* pRsp; + int64_t affectedRows; + int32_t code; + SSubmitRsp2* pRsp; } SSubmitRes; typedef struct SDataInserterHandle { @@ -59,22 +59,25 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) { pInserter->submitRes.code = code; if (code == TSDB_CODE_SUCCESS) { - pInserter->submitRes.pRsp = taosMemoryCalloc(1, sizeof(SSubmitRsp)); + pInserter->submitRes.pRsp = taosMemoryCalloc(1, sizeof(SSubmitRsp2)); SDecoder coder = {0}; tDecoderInit(&coder, pMsg->pData, pMsg->len); - code = tDecodeSSubmitRsp(&coder, pInserter->submitRes.pRsp); + code = tDecodeSSubmitRsp2(&coder, pInserter->submitRes.pRsp); if (code) { - tFreeSSubmitRsp(pInserter->submitRes.pRsp); + taosMemoryFree(pInserter->submitRes.pRsp); pInserter->submitRes.code = code; goto _return; } - if (pInserter->submitRes.pRsp->nBlocks > 0) { - for (int32_t i = 0; i < pInserter->submitRes.pRsp->nBlocks; ++i) { - SSubmitBlkRsp* blk = pInserter->submitRes.pRsp->pBlocks + i; - if (TSDB_CODE_SUCCESS != blk->code) { - code = blk->code; - tFreeSSubmitRsp(pInserter->submitRes.pRsp); + if (pInserter->submitRes.pRsp->affectedRows > 0) { + SArray* pCreateTbList = pInserter->submitRes.pRsp->aCreateTbRsp; + int32_t numOfTables = taosArrayGetSize(pCreateTbList); + + for (int32_t i = 0; i < numOfTables; ++i) { + SVCreateTbRsp* pRsp = taosArrayGet(pCreateTbList, i); + if (TSDB_CODE_SUCCESS != pRsp->code) { + code = pRsp->code; + taosMemoryFree(pInserter->submitRes.pRsp); pInserter->submitRes.code = code; goto _return; } @@ -82,22 +85,20 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) { } pInserter->submitRes.affectedRows += pInserter->submitRes.pRsp->affectedRows; - qDebug("submit rsp received, affectedRows:%d, total:%"PRId64, pInserter->submitRes.pRsp->affectedRows, + qDebug("submit rsp received, affectedRows:%d, total:%" PRId64, pInserter->submitRes.pRsp->affectedRows, pInserter->submitRes.affectedRows); - - tFreeSSubmitRsp(pInserter->submitRes.pRsp); + tDecoderClear(&coder); + taosMemoryFree(pInserter->submitRes.pRsp); } _return: - tsem_post(&pInserter->ready); - taosMemoryFree(pMsg->pData); - return TSDB_CODE_SUCCESS; } -static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMsg, void* pTransporter, SEpSet* pEpset) { +static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, void* pMsg, int32_t msgLen, void* pTransporter, + SEpSet* pEpset) { // send the fetch remote task result reques SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { @@ -112,7 +113,7 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMs pMsgSendInfo->param = pParam; pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->msgInfo.pData = pMsg; - pMsgSendInfo->msgInfo.len = ntohl(pMsg->length); + pMsgSendInfo->msgInfo.len = msgLen; pMsgSendInfo->msgType = TDMT_VND_SUBMIT; pMsgSendInfo->fp = inserterCallback; @@ -120,139 +121,230 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMs return asyncSendMsgToServer(pTransporter, pEpset, &transporterId, pMsgSendInfo); } -int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) { - const SArray* pBlocks = pInserter->pDataBlocks; - const STSchema* pTSchema = pInserter->pSchema; - int64_t uid = pInserter->pNode->tableId; - int64_t suid = pInserter->pNode->stableId; - int32_t vgId = pInserter->pNode->vgId; +static int32_t submitReqToMsg(int32_t vgId, SSubmitReq2* pReq, void** pData, int32_t* pLen) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t len = 0; + void* pBuf = NULL; + tEncodeSize(tEncodeSSubmitReq2, pReq, len, code); + if (TSDB_CODE_SUCCESS == code) { + SEncoder encoder; + len += sizeof(SMsgHead); + pBuf = taosMemoryMalloc(len); + if (NULL == pBuf) { + return TSDB_CODE_OUT_OF_MEMORY; + } + ((SMsgHead*)pBuf)->vgId = htonl(vgId); + ((SMsgHead*)pBuf)->contLen = htonl(len); + tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SMsgHead)), len - sizeof(SMsgHead)); + code = tEncodeSSubmitReq2(&encoder, pReq); + tEncoderClear(&encoder); + } - SSubmitReq* ret = NULL; - int32_t sz = taosArrayGetSize(pBlocks); + if (TSDB_CODE_SUCCESS == code) { + *pData = pBuf; + *pLen = len; + } else { + taosMemoryFree(pBuf); + } + return code; +} - // cal size - int32_t cap = sizeof(SSubmitReq); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i); - int32_t rows = pDataBlock->info.rows; - // TODO min - int32_t rowSize = pDataBlock->info.rowSize; - int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); +int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** ppReq, const SSDataBlock* pDataBlock, + const STSchema* pTSchema, int64_t uid, int32_t vgId, tb_uid_t suid) { + SSubmitReq2* pReq = *ppReq; + SArray* pVals = NULL; + int32_t numOfBlks = 0; - cap += sizeof(SSubmitBlk) + rows * maxLen; + terrno = TSDB_CODE_SUCCESS; + + if (NULL == pReq) { + if (!(pReq = taosMemoryMalloc(sizeof(SSubmitReq2)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } } - // assign data - // TODO - ret = taosMemoryCalloc(1, cap); - ret->header.vgId = htonl(vgId); - ret->version = htonl(pTSchema->version); - ret->length = sizeof(SSubmitReq); - ret->numOfBlocks = htonl(sz); + int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); + int32_t rows = pDataBlock->info.rows; - SSubmitBlk* blkHead = POINTER_SHIFT(ret, sizeof(SSubmitReq)); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i); + SSubmitTbData tbData = {0}; + if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow*)))) { + goto _end; + } + tbData.suid = suid; + tbData.uid = uid; + tbData.sver = pTSchema->version; - blkHead->sversion = htonl(pTSchema->version); - // TODO - blkHead->suid = htobe64(suid); - blkHead->uid = htobe64(uid); - blkHead->schemaLen = htonl(0); - - int32_t rows = 0; - int32_t dataLen = 0; - STSRow* rowData = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); - int64_t lastTs = TSKEY_MIN; - bool ignoreRow = false; - for (int32_t j = 0; j < pDataBlock->info.rows; j++) { - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTSchema->version); - tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); - tdSRowResetBuf(&rb, rowData); + if (!pVals && !(pVals = taosArrayInit(colNum, sizeof(SColVal)))) { + taosArrayDestroy(tbData.aRowP); + goto _end; + } - ignoreRow = false; - for (int32_t k = 0; k < pTSchema->numOfCols; k++) { - const STColumn* pColumn = &pTSchema->columns[k]; - SColumnInfoData* pColData = NULL; - int16_t colIdx = k; - if (!pInserter->fullOrderColList) { - int16_t* slotId = taosHashGet(pInserter->pCols, &pColumn->colId, sizeof(pColumn->colId)); - if (NULL == slotId) { - continue; - } + int64_t lastTs = TSKEY_MIN; + bool ignoreRow = false; + bool disorderTs = false; - colIdx = *slotId; - } + for (int32_t j = 0; j < rows; ++j) { // iterate by row + taosArrayClear(pVals); - pColData = taosArrayGet(pDataBlock->pDataBlock, colIdx); - if (pColData->info.type != pColumn->type) { - qError("col type mis-match, schema type:%d, type in block:%d", pColumn->type, pColData->info.type); - terrno = TSDB_CODE_APP_ERROR; - return TSDB_CODE_APP_ERROR; + int32_t offset = 0; + for (int32_t k = 0; k < pTSchema->numOfCols; ++k) { // iterate by column + int16_t colIdx = k; + const STColumn* pCol = &pTSchema->columns[k]; + if (!pInserter->fullOrderColList) { + int16_t* slotId = taosHashGet(pInserter->pCols, &pCol->colId, sizeof(pCol->colId)); + if (NULL == slotId) { + continue; } - if (colDataIsNull_s(pColData, j)) { - if (0 == k && TSDB_DATA_TYPE_TIMESTAMP == pColumn->type) { - ignoreRow = true; - break; - } + colIdx = *slotId; + } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, pColumn->offset, k); - } else { - void* data = colDataGetData(pColData, j); - if (0 == k && TSDB_DATA_TYPE_TIMESTAMP == pColumn->type) { - if (*(int64_t*)data == lastTs) { - ignoreRow = true; - break; + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, colIdx); + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY + ASSERT(pColInfoData->info.type == pCol->type); + if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + void* data = colDataGetVarData(pColInfoData, j); + SValue sv = (SValue){.nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } + break; + } + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_MEDIUMBLOB: + uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); + ASSERT(0); + break; + default: + if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { + if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); // should use pCol->type + taosArrayPush(pVals, &cv); } else { - lastTs = *(int64_t*)data; + if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId) { + if (*(int64_t*)var == lastTs) { + ignoreRow = true; + } else if (*(int64_t*)var < lastTs) { + disorderTs = true; + } else { + lastTs = *(int64_t*)var; + } + } + + SValue sv; + memcpy(&sv.val, var, tDataTypes[pCol->type].bytes); + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); } + } else { + uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); + ASSERT(0); } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); - } - } - if (!pInserter->fullOrderColList) { - rb.hasNone = true; + break; } - tdSRowEnd(&rb); if (ignoreRow) { - continue; + break; } - - rows++; - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - dataLen += rowLen; } - blkHead->dataLen = htonl(dataLen); - blkHead->numOfRows = htonl(rows); + if (ignoreRow) { + ignoreRow = false; + continue; + } - ret->length += sizeof(SSubmitBlk) + dataLen; - blkHead = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk) + dataLen); + SRow* pRow = NULL; + if ((terrno = tRowBuild(pVals, pTSchema, &pRow)) < 0) { + tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + goto _end; + } + taosArrayPush(tbData.aRowP, &pRow); } - ret->length = htonl(ret->length); + if (disorderTs) { + tRowSort(tbData.aRowP); + if ((terrno = tRowMerge(tbData.aRowP, (STSchema*)pTSchema, 0)) != 0) { + goto _end; + } + } - *pReq = ret; + taosArrayPush(pReq->aSubmitTbData, &tbData); +_end: + taosArrayDestroy(pVals); + if (terrno != 0) { + *ppReq = NULL; + if (pReq) { + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); + } + return TSDB_CODE_FAILED; + } + *ppReq = pReq; return TSDB_CODE_SUCCESS; } +int32_t dataBlocksToSubmitReq(SDataInserterHandle* pInserter, void** pMsg, int32_t* msgLen) { + const SArray* pBlocks = pInserter->pDataBlocks; + const STSchema* pTSchema = pInserter->pSchema; + int64_t uid = pInserter->pNode->tableId; + int64_t suid = pInserter->pNode->stableId; + int32_t vgId = pInserter->pNode->vgId; + int32_t sz = taosArrayGetSize(pBlocks); + int32_t code = 0; + SSubmitReq2* pReq = NULL; + + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i); + + code = buildSubmitReqFromBlock(pInserter, &pReq, pDataBlock, pTSchema, uid, vgId, suid); + if (code) { + if (pReq) { + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); + } + + return code; + } + } + + code = submitReqToMsg(vgId, pReq, pMsg, msgLen); + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); + + return code; +} + static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle; taosArrayPush(pInserter->pDataBlocks, &pInput->pData); - SSubmitReq* pMsg = NULL; - int32_t code = dataBlockToSubmit(pInserter, &pMsg); + void* pMsg = NULL; + int32_t msgLen = 0; + int32_t code = dataBlocksToSubmitReq(pInserter, &pMsg, &msgLen); if (code) { return code; } taosArrayClear(pInserter->pDataBlocks); - - code = sendSubmitRequest(pInserter, pMsg, pInserter->pParam->readHandle->pMsgCb->clientRpc, &pInserter->pNode->epSet); + + code = sendSubmitRequest(pInserter, pMsg, msgLen, pInserter->pParam->readHandle->pMsgCb->clientRpc, + &pInserter->pNode->epSet); if (code) { return code; } @@ -350,7 +442,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat inserter->pCols = taosHashInit(pInserterNode->pCols->length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); - SNode* pNode = NULL; + SNode* pNode = NULL; int32_t i = 0; FOREACH(pNode, pInserterNode->pCols) { SColumnNode* pCol = (SColumnNode*)pNode; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index a5468008aa8b6ce4fd9c52ca4a7f4965c1f0dbc8..cbb056b3b1021bc337c79d04e1276e5017428907 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1424,6 +1424,18 @@ void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode) { createExprFromOneNode(pExp, pTargetNode->pExpr, pTargetNode->slotId); } +SExprInfo* createExpr(SNodeList* pNodeList, int32_t* numOfExprs) { + *numOfExprs = LIST_LENGTH(pNodeList); + SExprInfo* pExprs = taosMemoryCalloc(*numOfExprs, sizeof(SExprInfo)); + + for (int32_t i = 0; i < (*numOfExprs); ++i) { + SExprInfo* pExp = &pExprs[i]; + createExprFromOneNode(pExp, nodesListGetNode(pNodeList, i), i + UD_TAG_COLUMN_INDEX); + } + + return pExprs; +} + SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs) { int32_t numOfFuncs = LIST_LENGTH(pNodeList); int32_t numOfGroupKeys = 0; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 814ead57f0b51428f223f65b6fe97bec51f5c8ad..6ec60ee85722d6342074cbc7e2e87fa41452b943 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -51,8 +51,8 @@ static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOf if (type == STREAM_INPUT__MERGED_SUBMIT) { for (int32_t i = 0; i < numOfBlocks; i++) { - SSubmitReq* pReq = *(void**)POINTER_SHIFT(input, i * sizeof(void*)); - taosArrayPush(pInfo->pBlockLists, &pReq); + SPackedData* pReq = POINTER_SHIFT(input, i * sizeof(SPackedData)); + taosArrayPush(pInfo->pBlockLists, pReq); } pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; } else if (type == STREAM_INPUT__DATA_SUBMIT) { @@ -61,7 +61,10 @@ static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOf } else if (type == STREAM_INPUT__DATA_BLOCK) { for (int32_t i = 0; i < numOfBlocks; ++i) { SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i]; - taosArrayPush(pInfo->pBlockLists, &pDataBlock); + SPackedData tmp = { + .pDataBlock = pDataBlock, + }; + taosArrayPush(pInfo->pBlockLists, &tmp); } pInfo->blockType = STREAM_INPUT__DATA_BLOCK; } @@ -115,18 +118,21 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu if (type == STREAM_INPUT__MERGED_SUBMIT) { // ASSERT(numOfBlocks > 1); for (int32_t i = 0; i < numOfBlocks; i++) { - SSubmitReq* pReq = *(void**)POINTER_SHIFT(input, i * sizeof(void*)); - taosArrayPush(pInfo->pBlockLists, &pReq); + SPackedData* pReq = POINTER_SHIFT(input, i * sizeof(SPackedData)); + taosArrayPush(pInfo->pBlockLists, pReq); } pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; } else if (type == STREAM_INPUT__DATA_SUBMIT) { ASSERT(numOfBlocks == 1); - taosArrayPush(pInfo->pBlockLists, &input); + taosArrayPush(pInfo->pBlockLists, input); pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; } else if (type == STREAM_INPUT__DATA_BLOCK) { for (int32_t i = 0; i < numOfBlocks; ++i) { SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i]; - taosArrayPush(pInfo->pBlockLists, &pDataBlock); + SPackedData tmp = { + .pDataBlock = pDataBlock, + }; + taosArrayPush(pInfo->pBlockLists, &tmp); } pInfo->blockType = STREAM_INPUT__DATA_BLOCK; } else { @@ -1015,11 +1021,22 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s return TSDB_CODE_SUCCESS; } -int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq) { +#if 0 +int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq, int64_t scanVer) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); ASSERT(pTaskInfo->streamInfo.pReq == NULL); pTaskInfo->streamInfo.pReq = pReq; + pTaskInfo->streamInfo.scanVer = scanVer; + return 0; +} +#endif + +int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); + ASSERT(pTaskInfo->streamInfo.submit.msgStr == NULL); + pTaskInfo->streamInfo.submit = submit; return 0; } @@ -1199,3 +1216,4 @@ void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { rpcFreeCont(pMsg->pCont); destroySendMsgInfo(pSendInfo); } + diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 448e2b0a9164dad96e71e69160f5a73a9d031904..c7ea27edadd1e4cde6aa359924850a6a6895c99f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2260,9 +2260,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type) { SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; - - bool isStream = (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type); - pOptr = createIntervalOperatorInfo(ops[0], pIntervalPhyNode, pTaskInfo, isStream); + pOptr = createIntervalOperatorInfo(ops[0], pIntervalPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { pOptr = createStreamIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) { @@ -2748,3 +2746,24 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta blockDataUpdateTsWindow(pBlock, 0); return TSDB_CODE_SUCCESS; } + +void qStreamCloseTsdbReader(void* task) { + if (task == NULL) return; + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)task; + SOperatorInfo* pOp = pTaskInfo->pRoot; + qDebug("stream close tsdb reader, reset status uid %" PRId64 " ts %" PRId64, pTaskInfo->streamInfo.lastStatus.uid, + pTaskInfo->streamInfo.lastStatus.ts); + pTaskInfo->streamInfo.lastStatus = (STqOffsetVal){0}; + while (pOp->numOfDownstream == 1 && pOp->pDownstream[0]) { + SOperatorInfo* pDownstreamOp = pOp->pDownstream[0]; + if (pDownstreamOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + SStreamScanInfo* pInfo = pDownstreamOp->info; + if (pInfo->pTableScanOp) { + STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; + tsdbReaderClose(pTSInfo->base.dataReader); + pTSInfo->base.dataReader = NULL; + return; + } + } + } +} diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 5676e19cdfa1173bd71f5d33ea05c015e05194ae..3bc00b79b1aea8bd8dc6d586ed8598b3b5672e4a 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -917,6 +917,7 @@ uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, S } static bool hasRemainPartion(SStreamPartitionOperatorInfo* pInfo) { return pInfo->parIte != NULL; } +static bool hasRemainTbName(SStreamPartitionOperatorInfo* pInfo) { return pInfo->pTbNameIte != NULL; } static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { SStreamPartitionOperatorInfo* pInfo = pOperator->info; @@ -937,40 +938,13 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { colDataAppend(pDestCol, pDest->info.rows, pSrcData, isNull); } pDest->info.rows++; - if (pInfo->tbnameCalSup.numOfExprs > 0 && i == 0) { - void* tbname = NULL; - if (streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname) == 0) { - memcpy(pDest->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); - tdbFree(tbname); - } else { - SSDataBlock* pTmpBlock = blockCopyOneRow(pSrc, rowIndex); - SSDataBlock* pResBlock = createDataBlock(); - pResBlock->info.rowSize = TSDB_TABLE_NAME_LEN; - SColumnInfoData data = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_TABLE_NAME_LEN, 0); - taosArrayPush(pResBlock->pDataBlock, &data); - blockDataEnsureCapacity(pResBlock, 1); - projectApplyFunctions(pInfo->tbnameCalSup.pExprInfo, pResBlock, pTmpBlock, pInfo->tbnameCalSup.pCtx, 1, NULL); - ASSERT(pResBlock->info.rows == 1); - ASSERT(taosArrayGetSize(pResBlock->pDataBlock) == 1); - SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, 0); - ASSERT(pCol->info.type == TSDB_DATA_TYPE_VARCHAR); - void* pData = colDataGetVarData(pCol, 0); - // TODO check tbname validity - if (pData != (void*)-1) { - memset(pDest->info.parTbName, 0, TSDB_TABLE_NAME_LEN); - int32_t len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); - memcpy(pDest->info.parTbName, varDataVal(pData), len); - /*pDest->info.parTbName[len + 1] = 0;*/ - } else { - pDest->info.parTbName[0] = 0; - } - if (pParInfo->groupId && pDest->info.parTbName[0]) { - streamStatePutParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, pDest->info.parTbName); - } - /*printf("\n\n set name %s\n\n", pDest->info.parTbName);*/ - blockDataDestroy(pTmpBlock); - blockDataDestroy(pResBlock); - } + } + pDest->info.parTbName[0] = 0; + if (pInfo->tbnameCalSup.numOfExprs > 0) { + void* tbname = NULL; + if (streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname) == 0) { + memcpy(pDest->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); + tdbFree(tbname); } } taosArrayDestroy(pParInfo->rowIds); @@ -986,6 +960,60 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { return pDest; } +void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, int64_t groupId, + SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock) { + void* pValue = NULL; + if (groupId != 0 && streamStateGetParName(pState, groupId, &pValue) != 0) { + SSDataBlock* pTmpBlock = blockCopyOneRow(pSrcBlock, rowId); + if (pTableSup->numOfExprs > 0) { + projectApplyFunctions(pTableSup->pExprInfo, pDestBlock, pTmpBlock, pTableSup->pCtx, pTableSup->numOfExprs, NULL); + SColumnInfoData* pTbCol = taosArrayGet(pDestBlock->pDataBlock, UD_TABLE_NAME_COLUMN_INDEX); + void* pData = colDataGetVarData(pTbCol, pDestBlock->info.rows - 1); + char* tbName = pSrcBlock->info.parTbName; + memset(tbName, 0, TSDB_TABLE_NAME_LEN); + int32_t len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); + memcpy(tbName, varDataVal(pData), len); + streamStatePutParName(pState, groupId, tbName); + pDestBlock->info.rows--; + } else { + void* pTbNameCol = taosArrayGet(pDestBlock->pDataBlock, UD_TABLE_NAME_COLUMN_INDEX); + colDataAppendNULL(pTbNameCol, pDestBlock->info.rows); + pSrcBlock->info.parTbName[0] = 0; + } + + if (pTagSup->numOfExprs > 0) { + projectApplyFunctions(pTagSup->pExprInfo, pDestBlock, pTmpBlock, pTagSup->pCtx, pTagSup->numOfExprs, NULL); + pDestBlock->info.rows--; + } + + void* pGpIdCol = taosArrayGet(pDestBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX); + colDataAppend(pGpIdCol, pDestBlock->info.rows, (const char*)&groupId, false); + + pDestBlock->info.rows++; + blockDataDestroy(pTmpBlock); + } + streamStateReleaseBuf(pState, NULL, pValue); +} + +static SSDataBlock* buildStreamCreateTableResult(SOperatorInfo* pOperator) { + SStreamPartitionOperatorInfo* pInfo = pOperator->info; + if ( (pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) || taosHashGetSize(pInfo->pPartitions) == 0) { + return NULL; + } + blockDataCleanup(pInfo->pCreateTbRes); + blockDataEnsureCapacity(pInfo->pCreateTbRes, taosHashGetSize(pInfo->pPartitions)); + SSDataBlock* pSrc = pInfo->pInputDataBlock; + + while (pInfo->pTbNameIte != NULL) { + SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->pTbNameIte; + int32_t rowId = *(int32_t*) taosArrayGet(pParInfo->rowIds, 0); + appendCreateTableRow(pOperator->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, + pParInfo->groupId, pSrc, rowId, pInfo->pCreateTbRes); + pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, pInfo->pTbNameIte); + } + return pInfo->pCreateTbRes->info.rows > 0 ? pInfo->pCreateTbRes : NULL; +} + static void doStreamHashPartitionImpl(SStreamPartitionOperatorInfo* pInfo, SSDataBlock* pBlock) { pInfo->pInputDataBlock = pBlock; for (int32_t i = 0; i < pBlock->info.rows; ++i) { @@ -1012,6 +1040,15 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamPartitionOperatorInfo* pInfo = pOperator->info; + SSDataBlock* pCtRes = NULL; + + if (hasRemainTbName(pInfo)) { + pCtRes = buildStreamCreateTableResult(pOperator); + if (pCtRes != NULL) { + return pCtRes; + } + } + if (hasRemainPartion(pInfo)) { return buildStreamPartitionResult(pOperator); } @@ -1039,6 +1076,7 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { return pInfo->pDelRes; } break; default: + ASSERTS(pBlock->info.type == STREAM_CREATE_CHILD_TABLE, "invalid SSDataBlock type"); return pBlock; } @@ -1056,6 +1094,11 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; pInfo->parIte = taosHashIterate(pInfo->pPartitions, NULL); + pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, NULL); + pCtRes = buildStreamCreateTableResult(pOperator); + if (pCtRes != NULL) { + return pCtRes; + } return buildStreamPartitionResult(pOperator); } @@ -1076,6 +1119,7 @@ static void destroyStreamPartitionOperatorInfo(void* param) { cleanupExprSupp(&pInfo->tagCalSup); blockDataDestroy(pInfo->pDelRes); taosHashCleanup(pInfo->pPartitions); + blockDataDestroy(pInfo->pCreateTbRes); taosMemoryFreeClear(param); } @@ -1091,6 +1135,46 @@ void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup } } +SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag) { + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + pBlock->info.hasVarCol = false; + pBlock->info.id.groupId = 0; + pBlock->info.rows = 0; + pBlock->info.type = STREAM_CREATE_CHILD_TABLE; + pBlock->info.watermark = INT64_MIN; + + pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + SColumnInfoData infoData = {0}; + infoData.info.type = TSDB_DATA_TYPE_VARCHAR; + if (tbName->numOfExprs > 0) { + infoData.info.bytes = tbName->pExprInfo->base.resSchema.bytes; + } else { + infoData.info.bytes = 1; + } + pBlock->info.rowSize += infoData.info.bytes; + // sub table name + taosArrayPush(pBlock->pDataBlock, &infoData); + + SColumnInfoData gpIdData = {0}; + gpIdData.info.type = TSDB_DATA_TYPE_UBIGINT; + gpIdData.info.bytes = 8; + pBlock->info.rowSize += gpIdData.info.bytes; + // group id + taosArrayPush(pBlock->pDataBlock, &gpIdData); + + for (int32_t i = 0; i < tag->numOfExprs; i++) { + SColumnInfoData tagCol = {0}; + tagCol.info.type = tag->pExprInfo[i].base.resSchema.type; + tagCol.info.bytes = tag->pExprInfo[i].base.resSchema.bytes; + tagCol.info.precision = tag->pExprInfo[i].base.resSchema.precision; + // tag info + taosArrayPush(pBlock->pDataBlock, &tagCol); + pBlock->info.rowSize += tagCol.info.bytes; + } + + return pBlock; +} + SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) { SStreamPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamPartitionOperatorInfo)); @@ -1110,6 +1194,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr } } + pInfo->tbnameCalSup.numOfExprs = 0; if (pPartNode->pSubtable != NULL) { SExprInfo* pSubTableExpr = taosMemoryCalloc(1, sizeof(SExprInfo)); if (pSubTableExpr == NULL) { @@ -1124,9 +1209,10 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr } } + pInfo->tagCalSup.numOfExprs = 0; if (pPartNode->pTags != NULL) { int32_t numOfTags; - SExprInfo* pTagExpr = createExprInfo(pPartNode->pTags, NULL, &numOfTags); + SExprInfo* pTagExpr = createExpr(pPartNode->pTags, &numOfTags); if (pTagExpr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _error; @@ -1137,6 +1223,12 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr } } + if (pInfo->tbnameCalSup.numOfExprs != 0 || pInfo->tagCalSup.numOfExprs != 0) { + pInfo->pCreateTbRes = buildCreateTableBlock(&pInfo->tbnameCalSup, &pInfo->tagCalSup); + } else { + pInfo->pCreateTbRes = NULL; + } + int32_t keyLen = 0; code = initGroupOptrInfo(&pInfo->partitionSup.pGroupColVals, &keyLen, &pInfo->partitionSup.keyBuf, pInfo->partitionSup.pGroupCols); @@ -1153,6 +1245,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr blockDataEnsureCapacity(pInfo->binfo.pRes, 4096); pInfo->parIte = NULL; + pInfo->pTbNameIte = NULL; pInfo->pInputDataBlock = NULL; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 4d38f2c8e93f11fa1de5e7795e4f9206697fe51f..73c3ac43111569cdf2181dda2531fc19380d5b18 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -222,7 +222,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { blockDataCleanup(pFinalRes); SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - if (pTaskInfo->streamInfo.pReq) { + if (pTaskInfo->streamInfo.submit.msgStr) { pOperator->status = OP_OPENED; } @@ -278,7 +278,8 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { // for stream interval if (pBlock->info.type == STREAM_RETRIEVE || pBlock->info.type == STREAM_DELETE_RESULT || - pBlock->info.type == STREAM_DELETE_DATA) { + pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + // printDataBlock1(pBlock, "project1"); return pBlock; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index eb38299938d55efa79314ae73a8bdbdbe29b6f42..fdfd81f52f7562c6a3d53985b9e30d8772504e02 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -306,12 +306,14 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->filterOutBlocks += 1; pCost->totalRows += pBlock->info.rows; + tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); pCost->skipBlocks += 1; + tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_SMA_LOAD) { pCost->loadBlockStatis += 1; @@ -321,6 +323,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); + tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else { qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo)); @@ -342,6 +345,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->filterOutBlocks += 1; (*status) = FUNC_DATA_REQUIRED_FILTEROUT; + tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } } @@ -356,7 +360,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; - + tsdbReleaseDataBlock(pTableScanInfo->dataReader); *status = FUNC_DATA_REQUIRED_FILTEROUT; return TSDB_CODE_SUCCESS; } @@ -1327,52 +1331,46 @@ static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, return code; } -void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { - SExprSupp* pTbNameCalSup = &pInfo->tbnameCalSup; +#if 0 +void calBlockTag(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { + SExprSupp* pTagCalSup = &pInfo->tagCalSup; SStreamState* pState = pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState; - if (pTbNameCalSup == NULL || pTbNameCalSup->numOfExprs == 0) return; + if (pTagCalSup == NULL || pTagCalSup->numOfExprs == 0) return; if (pBlock == NULL || pBlock->info.rows == 0) return; - void* tbname = NULL; - if (streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { - pBlock->info.parTbName[0] = 0; - } else { - memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); - } - tdbFree(tbname); - - SSDataBlock* pSrcBlock = blockCopyOneRow(pBlock, 0); - ASSERT(pSrcBlock->info.rows == 1); - - SSDataBlock* pResBlock = createDataBlock(); - pResBlock->info.rowSize = VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN; - SColumnInfoData data = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_TABLE_NAME_LEN, 0); - taosArrayPush(pResBlock->pDataBlock, &data); - blockDataEnsureCapacity(pResBlock, 1); - - projectApplyFunctions(pTbNameCalSup->pExprInfo, pResBlock, pSrcBlock, pTbNameCalSup->pCtx, 1, NULL); - ASSERT(pResBlock->info.rows == 1); - ASSERT(taosArrayGetSize(pResBlock->pDataBlock) == 1); - SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, 0); - ASSERT(pCol->info.type == TSDB_DATA_TYPE_VARCHAR); - - void* pData = colDataGetData(pCol, 0); - // TODO check tbname validation - if (pData != (void*)-1 && pData != NULL) { - memset(pBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN); - int32_t len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); - memcpy(pBlock->info.parTbName, varDataVal(pData), len); - /*pBlock->info.parTbName[len + 1] = 0;*/ + void* tag = NULL; + int32_t tagLen = 0; + if (streamStateGetParTag(pState, pBlock->info.id.groupId, &tag, &tagLen) == 0) { + pBlock->info.tagLen = tagLen; + void* pTag = taosMemoryRealloc(pBlock->info.pTag, tagLen); + if (pTag == NULL) { + tdbFree(tag); + taosMemoryFree(pBlock->info.pTag); + pBlock->info.pTag = NULL; + pBlock->info.tagLen = 0; + return; + } + pBlock->info.pTag = pTag; + memcpy(pBlock->info.pTag, tag, tagLen); + tdbFree(tag); + return; } else { - pBlock->info.parTbName[0] = 0; + pBlock->info.pTag = NULL; } + tdbFree(tag); +} +#endif - if (pBlock->info.id.groupId && pBlock->info.parTbName[0]) { - streamStatePutParName(pState, pBlock->info.id.groupId, pBlock->info.parTbName); +static void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { + SExprSupp* pTbNameCalSup = &pInfo->tbnameCalSup; + SStreamState* pState = pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState; + blockDataCleanup(pInfo->pCreateTbRes); + if (pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) { + pBlock->info.parTbName[0] = 0; + } else { + appendCreateTableRow(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, + pBlock->info.id.groupId, pBlock, 0, pInfo->pCreateTbRes); } - - blockDataDestroy(pSrcBlock); - blockDataDestroy(pResBlock); } void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, @@ -1508,14 +1506,18 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { qDebug("queue scan called"); - if (pTaskInfo->streamInfo.pReq != NULL) { - if (pInfo->tqReader->pMsg == NULL) { - pInfo->tqReader->pMsg = pTaskInfo->streamInfo.pReq; - const SSubmitReq* pSubmit = pInfo->tqReader->pMsg; - if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) { - qError("submit msg messed up when initing stream submit block %p", pSubmit); - pInfo->tqReader->pMsg = NULL; - pTaskInfo->streamInfo.pReq = NULL; + if (pTaskInfo->streamInfo.submit.msgStr != NULL) { + if (pInfo->tqReader->msg2.msgStr == NULL) { + /*pInfo->tqReader->pMsg = pTaskInfo->streamInfo.pReq;*/ + + /*const SSubmitReq* pSubmit = pInfo->tqReader->pMsg;*/ + /*if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) {*/ + /*void* msgStr = pTaskInfo->streamInfo.*/ + SPackedData submit = pTaskInfo->streamInfo.submit; + if (tqReaderSetSubmitReq2(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) { + qError("submit msg messed up when initing stream submit block %p", submit.msgStr); + pInfo->tqReader->msg2 = (SPackedData){0}; + pInfo->tqReader->setMsg = 0; ASSERT(0); } } @@ -1523,10 +1525,10 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { blockDataCleanup(pInfo->pRes); SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; - while (tqNextDataBlock(pInfo->tqReader)) { + while (tqNextDataBlock2(pInfo->tqReader)) { SSDataBlock block = {0}; - int32_t code = tqRetrieveDataBlock(&block, pInfo->tqReader); + int32_t code = tqRetrieveDataBlock2(&block, pInfo->tqReader); if (code != TSDB_CODE_SUCCESS || block.info.rows == 0) { continue; @@ -1539,15 +1541,17 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { } } - pInfo->tqReader->pMsg = NULL; - pTaskInfo->streamInfo.pReq = NULL; + pInfo->tqReader->msg2 = (SPackedData){0}; + pInfo->tqReader->setMsg = 0; + pTaskInfo->streamInfo.submit = (SPackedData){0}; return NULL; } if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) { SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); if (pResult && pResult->info.rows > 0) { - qDebug("queue scan tsdb return %d rows", pResult->info.rows); + qDebug("queue scan tsdb return %d rows min:%" PRId64 " max:%" PRId64, pResult->info.rows, + pResult->info.window.skey, pResult->info.window.ekey); pTaskInfo->streamInfo.returned = 1; return pResult; } else { @@ -1668,47 +1672,30 @@ static void setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { } } +static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey) { + if (pInfo->pUpdateInfo) { + checkUpdateData(pInfo, true, pInfo->pRes, true); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, endKey); + if (pInfo->pUpdateDataRes->info.rows > 0) { + pInfo->updateResIndex = 0; + if (pInfo->pUpdateDataRes->info.type == STREAM_CLEAR) { + pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES; + } else if (pInfo->pUpdateDataRes->info.type == STREAM_INVERT) { + pInfo->scanMode = STREAM_SCAN_FROM_RES; + // return pInfo->pUpdateDataRes; + } else if (pInfo->pUpdateDataRes->info.type == STREAM_DELETE_DATA) { + pInfo->scanMode = STREAM_SCAN_FROM_DELETE_DATA; + } + } + } +} + static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamScanInfo* pInfo = pOperator->info; qDebug("stream scan called"); -#if 0 - SStreamState* pState = pTaskInfo->streamInfo.pState; - if (pState) { - printf(">>>>>>>> stream write backend\n"); - SWinKey key = { - .ts = 1, - .groupId = 2, - }; - char tmp[100] = "abcdefg1"; - if (streamStatePut(pState, &key, &tmp, strlen(tmp) + 1) < 0) { - ASSERT(0); - } - - key.ts = 2; - char tmp2[100] = "abcdefg2"; - if (streamStatePut(pState, &key, &tmp2, strlen(tmp2) + 1) < 0) { - ASSERT(0); - } - - key.groupId = 5; - key.ts = 1; - char tmp3[100] = "abcdefg3"; - if (streamStatePut(pState, &key, &tmp3, strlen(tmp3) + 1) < 0) { - ASSERT(0); - } - - char* val2 = NULL; - int32_t sz; - if (streamStateGet(pState, &key, (void**)&val2, &sz) < 0) { - ASSERT(0); - } - printf("stream read %s %d\n", val2, sz); - streamFreeVal(val2); - } -#endif if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE1 || pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE2) { @@ -1743,17 +1730,32 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pInfo->blockRecoverContiCnt = 0; return NULL; } - SSDataBlock* pBlock = doTableScan(pInfo->pTableScanOp); - if (pBlock != NULL) { + + switch (pInfo->scanMode) { + case STREAM_SCAN_FROM_RES: { + pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; + printDataBlock(pInfo->pRecoverRes, "scan recover"); + return pInfo->pRecoverRes; + } break; + default: + break; + } + + pInfo->pRecoverRes = doTableScan(pInfo->pTableScanOp); + if (pInfo->pRecoverRes != NULL) { pInfo->blockRecoverContiCnt++; - calBlockTbName(pInfo, pBlock); + calBlockTbName(pInfo, pInfo->pRecoverRes); if (pInfo->pUpdateInfo) { - TSKEY maxTs = updateInfoFillBlockData(pInfo->pUpdateInfo, pBlock, pInfo->primaryTsIndex); + TSKEY maxTs = updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); } - qDebug("stream recover scan get block, rows %d", pBlock->info.rows); - printDataBlock(pBlock, "scan recover"); - return pBlock; + if (pInfo->pCreateTbRes->info.rows > 0) { + pInfo->scanMode = STREAM_SCAN_FROM_RES; + return pInfo->pCreateTbRes; + } + qDebug("stream recover scan get block, rows %d", pInfo->pRecoverRes->info.rows); + printDataBlock(pInfo->pRecoverRes, "scan recover"); + return pInfo->pRecoverRes; } pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; @@ -1778,7 +1780,8 @@ FETCH_NEXT_BLOCK: } int32_t current = pInfo->validBlockIndex++; - SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current); + SPackedData* pPacked = taosArrayGet(pInfo->pBlockLists, current); + SSDataBlock* pBlock = pPacked->pDataBlock; if (pBlock->info.id.groupId && pBlock->info.parTbName[0]) { streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, pBlock->info.parTbName); } @@ -1854,8 +1857,11 @@ FETCH_NEXT_BLOCK: qDebug("scan mode %d", pInfo->scanMode); switch (pInfo->scanMode) { case STREAM_SCAN_FROM_RES: { - blockDataDestroy(pInfo->pUpdateRes); pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; + doCheckUpdate(pInfo, pInfo->pRes->info.window.ekey); + doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); + pInfo->pRes->info.dataLoad = 1; + blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); return pInfo->pRes; } break; case STREAM_SCAN_FROM_DELETE_DATA: { @@ -1908,7 +1914,7 @@ FETCH_NEXT_BLOCK: NEXT_SUBMIT_BLK: while (1) { - if (pInfo->tqReader->pMsg == NULL) { + if (pInfo->tqReader->msg2.msgStr == NULL) { if (pInfo->validBlockIndex >= totBlockNum) { updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); doClearBufferedBlocks(pInfo); @@ -1916,22 +1922,22 @@ FETCH_NEXT_BLOCK: return NULL; } - int32_t current = pInfo->validBlockIndex++; - SSubmitReq* pSubmit = taosArrayGetP(pInfo->pBlockLists, current); - if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) { + int32_t current = pInfo->validBlockIndex++; + SPackedData* pSubmit = taosArrayGet(pInfo->pBlockLists, current); + /*if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) {*/ + if (tqReaderSetSubmitReq2(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) { qError("submit msg messed up when initing stream submit block %p, current %d, total %d", pSubmit, current, totBlockNum); - pInfo->tqReader->pMsg = NULL; continue; } } blockDataCleanup(pInfo->pRes); - while (tqNextDataBlock(pInfo->tqReader)) { + while (tqNextDataBlock2(pInfo->tqReader)) { SSDataBlock block = {0}; - int32_t code = tqRetrieveDataBlock(&block, pInfo->tqReader); + int32_t code = tqRetrieveDataBlock2(&block, pInfo->tqReader); if (code != TSDB_CODE_SUCCESS || block.info.rows == 0) { continue; @@ -1946,22 +1952,12 @@ FETCH_NEXT_BLOCK: continue; } - if (pInfo->pUpdateInfo) { - checkUpdateData(pInfo, true, pInfo->pRes, true); - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlockInfo->window.ekey); - if (pInfo->pUpdateDataRes->info.rows > 0) { - pInfo->updateResIndex = 0; - if (pInfo->pUpdateDataRes->info.type == STREAM_CLEAR) { - pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES; - } else if (pInfo->pUpdateDataRes->info.type == STREAM_INVERT) { - pInfo->scanMode = STREAM_SCAN_FROM_RES; - return pInfo->pUpdateDataRes; - } else if (pInfo->pUpdateDataRes->info.type == STREAM_DELETE_DATA) { - pInfo->scanMode = STREAM_SCAN_FROM_DELETE_DATA; - } - } + if (pInfo->pCreateTbRes->info.rows > 0) { + pInfo->scanMode = STREAM_SCAN_FROM_RES; + return pInfo->pCreateTbRes; } + doCheckUpdate(pInfo, pBlockInfo->window.ekey); doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); pInfo->pRes->info.dataLoad = 1; blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); @@ -1973,10 +1969,8 @@ FETCH_NEXT_BLOCK: if (pBlockInfo->rows > 0 || pInfo->pUpdateDataRes->info.rows > 0) { break; } else { - pInfo->tqReader->pMsg = NULL; continue; } - /*blockDataCleanup(pInfo->pRes);*/ } // record the scan action. @@ -2182,6 +2176,7 @@ static void destroyStreamScanOperatorInfo(void* param) { } cleanupExprSupp(&pStreamScan->tbnameCalSup); + cleanupExprSupp(&pStreamScan->tagCalSup); updateInfoDestroy(pStreamScan->pUpdateInfo); blockDataDestroy(pStreamScan->pRes); @@ -2189,6 +2184,7 @@ static void destroyStreamScanOperatorInfo(void* param) { blockDataDestroy(pStreamScan->pPullDataRes); blockDataDestroy(pStreamScan->pDeleteDataRes); blockDataDestroy(pStreamScan->pUpdateDataRes); + blockDataDestroy(pStreamScan->pCreateTbRes); taosArrayDestroy(pStreamScan->pBlockLists); taosMemoryFree(pStreamScan); } @@ -2244,7 +2240,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys if (pTableScanNode->pTags != NULL) { int32_t numOfTags; - SExprInfo* pTagExpr = createExprInfo(pTableScanNode->pTags, NULL, &numOfTags); + SExprInfo* pTagExpr = createExpr(pTableScanNode->pTags, &numOfTags); if (pTagExpr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _error; @@ -2255,7 +2251,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys } } - pInfo->pBlockLists = taosArrayInit(4, POINTER_BYTES); + pInfo->pBlockLists = taosArrayInit(4, sizeof(SPackedData)); if (pInfo->pBlockLists == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _error; @@ -2302,6 +2298,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->readHandle = *pHandle; pInfo->tableUid = pScanPhyNode->uid; pTaskInfo->streamInfo.snapshotVer = pHandle->version; + pInfo->pCreateTbRes = buildCreateTableBlock(&pInfo->tbnameCalSup, &pInfo->tagCalSup); + blockDataEnsureCapacity(pInfo->pCreateTbRes, 8); // set the extract column id to streamHandle tqReaderSetColIdList(pInfo->tqReader, pColIds); @@ -2518,6 +2516,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } STsdbReader* reader = pInfo->base.dataReader; + qTrace("tsdb/read-table-data: %p, enter next reader", reader); while (tsdbNextDataBlock(reader)) { if (isTaskKilled(pTaskInfo)) { T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); @@ -2552,6 +2551,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { pOperator->resultInfo.totalRows += pBlock->info.rows; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; + qTrace("tsdb/read-table-data: %p, close reader", reader); tsdbReaderClose(pInfo->base.dataReader); pInfo->base.dataReader = NULL; return pBlock; @@ -3222,7 +3222,9 @@ static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, S uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); pRes->info.id.groupId = groupId; int64_t ntbNum = metaGetNtbNum(pInfo->readHandle.meta); - fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes); + if (ntbNum != 0) { + fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes); + } } static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 94c98f41c951a258ee43140d400bfcbcbd106356..f5ceeeafe72de613bfaea86a5fded3c19dc51d9c 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -177,6 +177,10 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { colDataAppend(pDst, rows, (char*)&pSliceInfo->current, false); continue; + } else if (IS_BOOLEAN_TYPE(pExprInfo->base.resSchema.type)) { + bool isFilled = true; + colDataAppend(pDst, pResBlock->info.rows, (char*)&isFilled, false); + continue; } int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; @@ -288,6 +292,9 @@ static void addCurrentRowToResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { colDataAppend(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false); + } else if (IS_BOOLEAN_TYPE(pExprInfo->base.resSchema.type)) { + bool isFilled = false; + colDataAppend(pDst, pResBlock->info.rows, (char*)&isFilled, false); } else { int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index d78e9c4edf06329ccdb6fe1817a67c318ee3041d..68f178803d3a41c92c6962e709c9630bf22a8589 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1740,7 +1740,7 @@ void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) { } SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode, - SExecTaskInfo* pTaskInfo, bool isStream) { + SExecTaskInfo* pTaskInfo) { SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -2493,12 +2493,8 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - SOperatorInfo* downstream = pOperator->pDownstream[0]; - TSKEY maxTs = INT64_MIN; - TSKEY minTs = INT64_MAX; - - SExprSupp* pSup = &pOperator->exprSupp; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + SExprSupp* pSup = &pOperator->exprSupp; qDebug("interval status %d %s", pOperator->status, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); @@ -2554,9 +2550,14 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } } - SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); + if (!pInfo->pUpdated) { + pInfo->pUpdated = taosArrayInit(4, POINTER_BYTES); + } + if (!pInfo->pUpdatedMap) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); + } + while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -2574,35 +2575,39 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); - doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pUpdatedMap); + doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap); if (IS_FINAL_OP(pInfo)) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SStreamIntervalOperatorInfo* pChildInfo = pChildOp->info; SExprSupp* pChildSup = &pChildOp->exprSupp; doDeleteWindows(pChildOp, &pChildInfo->interval, pBlock, NULL, NULL); - rebuildIntervalWindow(pOperator, delWins, pUpdatedMap); + rebuildIntervalWindow(pOperator, delWins, pInfo->pUpdatedMap); addRetriveWindow(delWins, pInfo); taosArrayAddAll(pInfo->pDelWins, delWins); taosArrayDestroy(delWins); continue; } - removeResults(delWins, pUpdatedMap); + removeResults(delWins, pInfo->pUpdatedMap); taosArrayAddAll(pInfo->pDelWins, delWins); taosArrayDestroy(delWins); break; } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) { - getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap); + getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) { - doDeleteWindows(pOperator, &pInfo->interval, pBlock, NULL, pUpdatedMap); - if (taosArrayGetSize(pUpdated) > 0) { + doDeleteWindows(pOperator, &pInfo->interval, pBlock, NULL, pInfo->pUpdatedMap); + if (taosArrayGetSize(pInfo->pUpdated) > 0) { break; } continue; } else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_OP(pInfo)) { processPullOver(pBlock, pInfo->pPullDataMap, &pInfo->interval); continue; + } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + return pBlock; + } else { + ASSERTS(pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); } if (pInfo->scalarSupp.pExprInfo != NULL) { @@ -2610,7 +2615,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); } setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); - doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pUpdatedMap); + doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pInfo->pUpdatedMap); if (IS_FINAL_OP(pInfo)) { int32_t chIndex = getChildIndex(pBlock); int32_t size = taosArrayGetSize(pInfo->pChildren); @@ -2630,29 +2635,29 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { setInputDataBlock(&pChildOp->exprSupp, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); doStreamIntervalAggImpl(pChildOp, pBlock, pBlock->info.id.groupId, NULL); } - maxTs = TMAX(maxTs, pBlock->info.window.ekey); - maxTs = TMAX(maxTs, pBlock->info.watermark); - minTs = TMIN(minTs, pBlock->info.window.skey); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark); + pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey); } - removeDeleteResults(pUpdatedMap, pInfo->pDelWins); - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); - pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs); + removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins); if (IS_FINAL_OP(pInfo)) { closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, - pInfo->pPullDataMap, pUpdatedMap, pInfo->pDelWins, pOperator); + pInfo->pPullDataMap, pInfo->pUpdatedMap, pInfo->pDelWins, pOperator); closeChildIntervalWindow(pOperator, pInfo->pChildren, pInfo->twAggSup.maxTs); } pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs; void* pIte = NULL; - while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { - taosArrayPush(pUpdated, pIte); + while ((pIte = taosHashIterate(pInfo->pUpdatedMap, pIte)) != NULL) { + taosArrayPush(pInfo->pUpdated, pIte); } - taosHashCleanup(pUpdatedMap); - taosArraySort(pUpdated, resultrowComparAsc); + taosHashCleanup(pInfo->pUpdatedMap); + pInfo->pUpdatedMap = NULL; + taosArraySort(pInfo->pUpdated, resultrowComparAsc); - initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); + pInfo->pUpdated = NULL; blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes); @@ -2790,6 +2795,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->delKey.ts = INT64_MAX; pInfo->delKey.groupId = 0; pInfo->numOfDatapack = 0; + pInfo->pUpdated = NULL; + pInfo->pUpdatedMap = NULL; pOperator->operatorType = pPhyNode->type; pOperator->blocking = true; @@ -3419,7 +3426,6 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SExprSupp* pSup = &pOperator->exprSupp; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; - TSKEY maxTs = INT64_MIN; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -3439,10 +3445,14 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { return NULL; } - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SSHashObj* pStUpdated = tSimpleHashInit(64, hashFn); SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(16, sizeof(SSessionKey)); // SResKeyPos + if (!pInfo->pUpdated) { + pInfo->pUpdated = taosArrayInit(16, sizeof(SSessionKey)); + } + if (!pInfo->pStUpdated) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pStUpdated = tSimpleHashInit(64, hashFn); + } while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -3455,21 +3465,25 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); // gap must be 0 doDeleteTimeWindows(pAggSup, pBlock, pWins); - removeSessionResults(pStUpdated, pWins); + removeSessionResults(pInfo->pStUpdated, pWins); if (IS_FINAL_OP(pInfo)) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; // gap must be 0 doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, NULL); - rebuildSessionWindow(pOperator, pWins, pStUpdated); + rebuildSessionWindow(pOperator, pWins, pInfo->pStUpdated); } copyDeleteWindowInfo(pWins, pInfo->pStDeleted); taosArrayDestroy(pWins); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllSessionWindow(pAggSup->pResultRows, pStUpdated); + getAllSessionWindow(pAggSup->pResultRows, pInfo->pStUpdated); continue; + } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + return pBlock; + } else { + ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); } if (pInfo->scalarSupp.pExprInfo != NULL) { @@ -3478,7 +3492,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); - doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, IS_FINAL_OP(pInfo)); + doStreamSessionAggImpl(pOperator, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, IS_FINAL_OP(pInfo)); if (IS_FINAL_OP(pInfo)) { int32_t chIndex = getChildIndex(pBlock); int32_t size = taosArrayGetSize(pInfo->pChildren); @@ -3495,20 +3509,20 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { setInputDataBlock(&pChildOp->exprSupp, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); doStreamSessionAggImpl(pChildOp, pBlock, NULL, NULL, true); } - maxTs = TMAX(maxTs, pBlock->info.window.ekey); - maxTs = TMAX(maxTs, pBlock->info.watermark); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark); } - - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pStUpdated); + closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated); closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); - copyUpdateResult(pStUpdated, pUpdated); - removeSessionResults(pInfo->pStDeleted, pUpdated); - tSimpleHashCleanup(pStUpdated); - initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + copyUpdateResult(pInfo->pStUpdated, pInfo->pUpdated); + removeSessionResults(pInfo->pStDeleted, pInfo->pUpdated); + tSimpleHashCleanup(pInfo->pStUpdated); + pInfo->pStUpdated = NULL; + initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); + pInfo->pUpdated = NULL; blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); #if 0 @@ -3593,6 +3607,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->isFinal = false; pInfo->pPhyNode = pPhyNode; pInfo->ignoreExpiredData = pSessionNode->window.igExpired; + pInfo->pUpdated = NULL; + pInfo->pStUpdated = NULL; setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -3653,10 +3669,14 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { } } - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SSHashObj* pStUpdated = tSimpleHashInit(64, hashFn); SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(16, sizeof(SSessionKey)); + if (!pInfo->pUpdated) { + pInfo->pUpdated = taosArrayInit(16, sizeof(SSessionKey)); + } + if (!pInfo->pStUpdated) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pStUpdated = tSimpleHashInit(64, hashFn); + } while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -3671,13 +3691,17 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { // gap must be 0 SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, pWins); - removeSessionResults(pStUpdated, pWins); + removeSessionResults(pInfo->pStUpdated, pWins); copyDeleteWindowInfo(pWins, pInfo->pStDeleted); taosArrayDestroy(pWins); break; } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllSessionWindow(pInfo->streamAggSup.pResultRows, pStUpdated); + getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pStUpdated); continue; + } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + return pBlock; + } else { + ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); } if (pInfo->scalarSupp.pExprInfo != NULL) { @@ -3686,18 +3710,20 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); - doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, NULL, false); + doStreamSessionAggImpl(pOperator, pBlock, pInfo->pStUpdated, NULL, false); maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs; - copyUpdateResult(pStUpdated, pUpdated); - removeSessionResults(pInfo->pStDeleted, pUpdated); - tSimpleHashCleanup(pStUpdated); + copyUpdateResult(pInfo->pStUpdated, pInfo->pUpdated); + removeSessionResults(pInfo->pStDeleted, pInfo->pUpdated); + tSimpleHashCleanup(pInfo->pStUpdated); + pInfo->pStUpdated = NULL; - initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); + pInfo->pUpdated = NULL; blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); #if 0 @@ -3957,7 +3983,6 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { SExprSupp* pSup = &pOperator->exprSupp; SStreamStateAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; - int64_t maxTs = INT64_MIN; if (pOperator->status == OP_RES_TO_RETURN) { doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { @@ -3975,10 +4000,14 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { return NULL; } - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SSHashObj* pSeUpdated = tSimpleHashInit(64, hashFn); SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(16, sizeof(SSessionKey)); + if (!pInfo->pUpdated) { + pInfo->pUpdated = taosArrayInit(16, sizeof(SSessionKey)); + } + if (!pInfo->pSeUpdated) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pSeUpdated = tSimpleHashInit(64, hashFn); + } while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -3990,13 +4019,17 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { pBlock->info.type == STREAM_CLEAR) { SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, pWins); - removeSessionResults(pSeUpdated, pWins); + removeSessionResults(pInfo->pSeUpdated, pWins); copyDeleteWindowInfo(pWins, pInfo->pSeDeleted); taosArrayDestroy(pWins); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllSessionWindow(pInfo->streamAggSup.pResultRows, pSeUpdated); + getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pSeUpdated); continue; + } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + return pBlock; + } else { + ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); } if (pInfo->scalarSupp.pExprInfo != NULL) { @@ -4005,19 +4038,20 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); - doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted); - maxTs = TMAX(maxTs, pBlock->info.window.ekey); + doStreamStateAggImpl(pOperator, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); } - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pSeUpdated); - copyUpdateResult(pSeUpdated, pUpdated); - removeSessionResults(pInfo->pSeDeleted, pUpdated); - tSimpleHashCleanup(pSeUpdated); + closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pInfo->pSeUpdated); + copyUpdateResult(pInfo->pSeUpdated, pInfo->pUpdated); + removeSessionResults(pInfo->pSeDeleted, pInfo->pUpdated); + tSimpleHashCleanup(pInfo->pSeUpdated); + pInfo->pSeUpdated = NULL; - initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); + pInfo->pUpdated = NULL; blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); #if 0 @@ -4098,6 +4132,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); pInfo->pChildren = NULL; pInfo->ignoreExpiredData = pStateNode->window.igExpired; + pInfo->pUpdated = NULL; + pInfo->pSeUpdated = NULL; setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -4711,8 +4747,6 @@ _error: static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - int64_t maxTs = INT64_MIN; - int64_t minTs = INT64_MAX; SExprSupp* pSup = &pOperator->exprSupp; if (pOperator->status == OP_EXEC_DONE) { @@ -4740,9 +4774,14 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); // SResKeyPos - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); + if (!pInfo->pUpdated) { + pInfo->pUpdated = taosArrayInit(4, POINTER_BYTES); + } + if (!pInfo->pUpdatedMap) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); + } + while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); @@ -4756,11 +4795,15 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { - doDeleteWindows(pOperator, &pInfo->interval, pBlock, pInfo->pDelWins, pUpdatedMap); + doDeleteWindows(pOperator, &pInfo->interval, pBlock, pInfo->pDelWins, pInfo->pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap); + getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); continue; + } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + return pBlock; + } else { + ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); } if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) { @@ -4781,27 +4824,27 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { setInverFunction(pSup->pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.type); } - maxTs = TMAX(maxTs, pBlock->info.window.ekey); - minTs = TMIN(minTs, pBlock->info.window.skey); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); + pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey); - doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pUpdatedMap); + doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pInfo->pUpdatedMap); } - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); - pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs); pOperator->status = OP_RES_TO_RETURN; - removeDeleteResults(pUpdatedMap, pInfo->pDelWins); - closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap, + removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins); + closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pInfo->pUpdatedMap, pInfo->pDelWins, pOperator); void* pIte = NULL; - while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { - taosArrayPush(pUpdated, pIte); + while ((pIte = taosHashIterate(pInfo->pUpdatedMap, pIte)) != NULL) { + taosArrayPush(pInfo->pUpdated, pIte); } - taosArraySort(pUpdated, resultrowComparAsc); + taosArraySort(pInfo->pUpdated, resultrowComparAsc); - initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); + pInfo->pUpdated = NULL; blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - taosHashCleanup(pUpdatedMap); + taosHashCleanup(pInfo->pUpdatedMap); + pInfo->pUpdatedMap = NULL; doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); if (pInfo->pDelRes->info.rows > 0) { @@ -4902,6 +4945,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->delKey.ts = INT64_MAX; pInfo->delKey.groupId = 0; pInfo->numOfDatapack = 0; + pInfo->pUpdated = NULL; + pInfo->pUpdatedMap = NULL; setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -4922,4 +4967,3 @@ _error: pTaskInfo->code = code; return NULL; } - diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index faf7a29dd0aa34031325dc9c0b49cf46e403e78a..1a0437c26a6657c17557e4772fd91403a647bf43 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -466,7 +466,7 @@ static int32_t translateStddevMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t static int32_t translateWduration(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // pseudo column do not need to check parameters - pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; return TSDB_CODE_SUCCESS; } @@ -480,14 +480,21 @@ static int32_t translateNowToday(SFunctionNode* pFunc, char* pErrBuf, int32_t le return code; } - pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_TIMESTAMP}; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; return TSDB_CODE_SUCCESS; } static int32_t translateTimePseudoColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // pseudo column do not need to check parameters - pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_TIMESTAMP}; + pFunc->node.resType = (SDataType){.bytes =tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; + return TSDB_CODE_SUCCESS; +} + +static int32_t translateIsFilledPseudoColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + // pseudo column do not need to check parameters + + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes, .type = TSDB_DATA_TYPE_BOOL}; return TSDB_CODE_SUCCESS; } @@ -3257,6 +3264,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = NULL, .finalizeFunc = NULL }, + { + .name = "_isfilled", + .type = FUNCTION_TYPE_ISFILLED, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_INTERP_PC_FUNC, + .translateFunc = translateIsFilledPseudoColumn, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = NULL, + .finalizeFunc = NULL + }, { .name = "_tags", .type = FUNCTION_TYPE_TAGS, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 324011f2382885e79a17463924260412a4375771..224537c1f110f9d3d629320d031baf1dc6e7d108 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -860,13 +860,6 @@ int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STu return TSDB_CODE_SUCCESS; } -void releaseSource(STuplePos* pPos) { - if (pPos->pageId == -1) { - return; - } - // Todo(liuyao) relase row -} - // This function append the selectivity to subsidiaries function context directly, without fetching data // from intermediate disk based buf page void appendSelectivityValue(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t pos) { @@ -899,7 +892,6 @@ void appendSelectivityValue(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t pos) } void replaceTupleData(STuplePos* pDestPos, STuplePos* pSourcePos) { - releaseSource(pDestPos); *pDestPos = *pSourcePos; } @@ -5370,6 +5362,7 @@ int32_t blockDistFunction(SqlFunctionCtx* pCtx) { pDistInfo->numOfBlocks += p1.numOfBlocks; pDistInfo->numOfTables += p1.numOfTables; pDistInfo->numOfInmemRows += p1.numOfInmemRows; + pDistInfo->numOfVgroups += p1.numOfVgroups; pDistInfo->totalSize += p1.totalSize; pDistInfo->totalRows += p1.totalRows; pDistInfo->numOfFiles += p1.numOfFiles; @@ -5404,6 +5397,7 @@ int32_t tSerializeBlockDistInfo(void* buf, int32_t bufLen, const STableBlockDist if (tEncodeU16(&encoder, pInfo->numOfFiles) < 0) return -1; if (tEncodeU32(&encoder, pInfo->numOfBlocks) < 0) return -1; if (tEncodeU32(&encoder, pInfo->numOfTables) < 0) return -1; + if (tEncodeU32(&encoder, pInfo->numOfVgroups) < 0) return -1; if (tEncodeU64(&encoder, pInfo->totalSize) < 0) return -1; if (tEncodeU64(&encoder, pInfo->totalRows) < 0) return -1; @@ -5435,6 +5429,7 @@ int32_t tDeserializeBlockDistInfo(void* buf, int32_t bufLen, STableBlockDistInfo if (tDecodeU16(&decoder, &pInfo->numOfFiles) < 0) return -1; if (tDecodeU32(&decoder, &pInfo->numOfBlocks) < 0) return -1; if (tDecodeU32(&decoder, &pInfo->numOfTables) < 0) return -1; + if (tDecodeU32(&decoder, &pInfo->numOfVgroups) < 0) return -1; if (tDecodeU64(&decoder, &pInfo->totalSize) < 0) return -1; if (tDecodeU64(&decoder, &pInfo->totalRows) < 0) return -1; @@ -5495,7 +5490,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { colDataAppend(pColInfo, row++, st, false); len = sprintf(st + VARSTR_HEADER_SIZE, "Total_Tables=[%d] Total_Files=[%d] Total_Vgroups=[%d]", pData->numOfTables, - pData->numOfFiles, 0); + pData->numOfFiles, pData->numOfVgroups); varDataSetLen(st, len); colDataAppend(pColInfo, row++, st, false); diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 871b668cfdd70aefdfbd5a0f4fb60eed79ec6225..6577858ee693257b4ddf02f3ca1ab1d1234e40a6 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -33,7 +33,14 @@ static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) (SFilePage *)taosMemoryCalloc(1, pMemBucket->bytes * pMemBucket->pSlots[slotIdx].info.size + sizeof(SFilePage)); int32_t groupId = getGroupId(pMemBucket->numOfSlots, slotIdx, pMemBucket->times); - SArray *pIdList = *(SArray **)taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId)); + + SArray *pIdList; + void *p = taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId)); + if (p != NULL) { + pIdList = *(SArray **)p; + } else { + return NULL; + } int32_t offset = 0; for (int32_t i = 0; i < taosArrayGetSize(pIdList); ++i) { @@ -513,8 +520,17 @@ int32_t getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction resetSlotInfo(pMemBucket); int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times - 1); - SArray* list = *(SArray **)taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId)); - ASSERT(list != NULL && list->size > 0); + + SArray* list; + void *p = taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId)); + if (p != NULL) { + list = *(SArray **)p; + if (list == NULL || list->size <= 0) { + return -1; + } + } else { + return -1; + } for (int32_t f = 0; f < list->size; ++f) { int32_t *pageId = taosArrayGet(list, f); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 7b9c54ab2c122f126fed799ac650915691803796..12041bd2ae8557a1832c735b128fd4ca7861106f 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -295,6 +295,13 @@ static int32_t stateWindowNodeCopy(const SStateWindowNode* pSrc, SStateWindowNod return TSDB_CODE_SUCCESS; } +static int32_t eventWindowNodeCopy(const SEventWindowNode* pSrc, SEventWindowNode* pDst) { + CLONE_NODE_FIELD(pCol); + CLONE_NODE_FIELD(pStartCond); + CLONE_NODE_FIELD(pEndCond); + return TSDB_CODE_SUCCESS; +} + static int32_t sessionWindowNodeCopy(const SSessionWindowNode* pSrc, SSessionWindowNode* pDst) { CLONE_NODE_FIELD_EX(pCol, SColumnNode*); CLONE_NODE_FIELD_EX(pGap, SValueNode*); @@ -463,6 +470,8 @@ static int32_t logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* p CLONE_NODE_FIELD(pTspk); CLONE_NODE_FIELD(pTsEnd); CLONE_NODE_FIELD(pStateExpr); + CLONE_NODE_FIELD(pStartCond); + CLONE_NODE_FIELD(pEndCond); COPY_SCALAR_FIELD(triggerType); COPY_SCALAR_FIELD(watermark); COPY_SCALAR_FIELD(deleteMark); @@ -710,6 +719,9 @@ SNode* nodesCloneNode(const SNode* pNode) { case QUERY_NODE_STATE_WINDOW: code = stateWindowNodeCopy((const SStateWindowNode*)pNode, (SStateWindowNode*)pDst); break; + case QUERY_NODE_EVENT_WINDOW: + code = eventWindowNodeCopy((const SEventWindowNode*)pNode, (SEventWindowNode*)pDst); + break; case QUERY_NODE_SESSION_WINDOW: code = sessionWindowNodeCopy((const SSessionWindowNode*)pNode, (SSessionWindowNode*)pDst); break; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 1a4ba8e43ef6d876a42631f6564b83aac1a69265..d9b3237993732b21da75ab9eea5249e9e4019226 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -79,17 +79,23 @@ const char* nodesNodeName(ENodeType type) { return "TableOptions"; case QUERY_NODE_INDEX_OPTIONS: return "IndexOptions"; + case QUERY_NODE_EXPLAIN_OPTIONS: + return "ExplainOptions"; + case QUERY_NODE_STREAM_OPTIONS: + return "StreamOptions"; case QUERY_NODE_LEFT_VALUE: return "LeftValue"; case QUERY_NODE_WHEN_THEN: return "WhenThen"; case QUERY_NODE_CASE_WHEN: return "CaseWhen"; + case QUERY_NODE_EVENT_WINDOW: + return "EventWindow"; case QUERY_NODE_SET_OPERATOR: return "SetOperator"; case QUERY_NODE_SELECT_STMT: return "SelectStmt"; - case QUERY_NODE_VNODE_MODIF_STMT: + case QUERY_NODE_VNODE_MODIFY_STMT: return "VnodeModifStmt"; case QUERY_NODE_CREATE_DATABASE_STMT: return "CreateDatabaseStmt"; @@ -97,11 +103,15 @@ const char* nodesNodeName(ENodeType type) { return "DropDatabaseStmt"; case QUERY_NODE_ALTER_DATABASE_STMT: return "AlterDatabaseStmt"; + case QUERY_NODE_FLUSH_DATABASE_STMT: + return "FlushDatabaseStmt"; + case QUERY_NODE_TRIM_DATABASE_STMT: + return "TrimDatabaseStmt"; case QUERY_NODE_CREATE_TABLE_STMT: return "CreateTableStmt"; case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: return "CreateSubtableClause"; - case QUERY_NODE_CREATE_MULTI_TABLE_STMT: + case QUERY_NODE_CREATE_MULTI_TABLES_STMT: return "CreateMultiTableStmt"; case QUERY_NODE_DROP_TABLE_CLAUSE: return "DropTableClause"; @@ -135,12 +145,46 @@ const char* nodesNodeName(ENodeType type) { return "CreateQnodeStmt"; case QUERY_NODE_DROP_QNODE_STMT: return "DropQnodeStmt"; + case QUERY_NODE_CREATE_SNODE_STMT: + return "CreateSnodeStmt"; + case QUERY_NODE_DROP_SNODE_STMT: + return "DropSnodeStmt"; + case QUERY_NODE_CREATE_MNODE_STMT: + return "CreateMnodeStmt"; + case QUERY_NODE_DROP_MNODE_STMT: + return "DropMnodeStmt"; case QUERY_NODE_CREATE_TOPIC_STMT: return "CreateTopicStmt"; case QUERY_NODE_DROP_TOPIC_STMT: return "DropTopicStmt"; + case QUERY_NODE_DROP_CGROUP_STMT: + return "DropConsumerGroupStmt"; case QUERY_NODE_ALTER_LOCAL_STMT: return "AlterLocalStmt"; + case QUERY_NODE_EXPLAIN_STMT: + return "ExplainStmt"; + case QUERY_NODE_DESCRIBE_STMT: + return "DescribeStmt"; + case QUERY_NODE_CREATE_STREAM_STMT: + return "CreateStreamStmt"; + case QUERY_NODE_DROP_STREAM_STMT: + return "DropStreamStmt"; + case QUERY_NODE_BALANCE_VGROUP_STMT: + return "BalanceVgroupStmt"; + case QUERY_NODE_MERGE_VGROUP_STMT: + return "MergeVgroupStmt"; + case QUERY_NODE_SHOW_DB_ALIVE_STMT: + return "ShowDbAliveStmt"; + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: + return "ShowClusterAliveStmt"; + case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT: + return "RedistributeVgroupStmt"; + case QUERY_NODE_SPLIT_VGROUP_STMT: + return "SplitVgroupStmt"; + case QUERY_NODE_GRANT_STMT: + return "GrantStmt"; + case QUERY_NODE_REVOKE_STMT: + return "RevokeStmt"; case QUERY_NODE_SHOW_DNODES_STMT: return "ShowDnodesStmt"; case QUERY_NODE_SHOW_MNODES_STMT: @@ -153,6 +197,8 @@ const char* nodesNodeName(ENodeType type) { return "ShowSnodesStmt"; case QUERY_NODE_SHOW_BNODES_STMT: return "ShowBnodesStmt"; + case QUERY_NODE_SHOW_CLUSTER_STMT: + return "ShowClusterStmt"; case QUERY_NODE_SHOW_DATABASES_STMT: return "ShowDatabaseStmt"; case QUERY_NODE_SHOW_FUNCTIONS_STMT: @@ -179,8 +225,30 @@ const char* nodesNodeName(ENodeType type) { return "ShowConsumersStmt"; case QUERY_NODE_SHOW_QUERIES_STMT: return "ShowQueriesStmt"; + case QUERY_NODE_SHOW_VARIABLES_STMT: + return "ShowVariablesStmt"; + case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: + return "ShowDnodeVariablesStmt"; + case QUERY_NODE_SHOW_TRANSACTIONS_STMT: + return "ShowTransactionsStmt"; + case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT: + return "ShowSubscriptionsStmt"; case QUERY_NODE_SHOW_VNODES_STMT: return "ShowVnodeStmt"; + case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT: + return "ShowUserPrivilegesStmt"; + case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: + return "ShowCreateDatabasesStmt"; + case QUERY_NODE_SHOW_CREATE_TABLE_STMT: + return "ShowCreateTablesStmt"; + case QUERY_NODE_SHOW_CREATE_STABLE_STMT: + return "ShowCreateStablesStmt"; + case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: + return "ShowTableDistributedStmt"; + case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: + return "ShowLocalVariablesStmt"; + case QUERY_NODE_SHOW_TABLE_TAGS_STMT: + return "ShowTableTagsStmt"; case QUERY_NODE_DELETE_STMT: return "DeleteStmt"; case QUERY_NODE_INSERT_STMT: @@ -233,6 +301,10 @@ const char* nodesNodeName(ENodeType type) { return "PhysiLastRowScan"; case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: return "PhysiTableCountScan"; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + return "PhysiMergeEventWindow"; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: + return "PhysiStreamEventWindow"; case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return "PhysiProject"; case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: @@ -1358,6 +1430,23 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { return code; } +static int32_t jsonToLogicJoinNode(const SJson* pJson, void* pObj) { + SJoinLogicNode* pNode = (SJoinLogicNode*)pObj; + + int32_t code = jsonToLogicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + tjsonGetNumberValue(pJson, jkJoinLogicPlanJoinType, pNode->joinType, code); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinLogicPlanMergeCondition, &pNode->pMergeCondition); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinLogicPlanOnConditions, &pNode->pOnConditions); + } + + return code; +} + static const char* jkPhysiPlanOutputDataBlockDesc = "OutputDataBlockDesc"; static const char* jkPhysiPlanConditions = "Conditions"; static const char* jkPhysiPlanChildren = "Children"; @@ -2279,6 +2368,37 @@ static int32_t jsonToPhysiStateWindowNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkEventWindowPhysiPlanStartCond = "StartCond"; +static const char* jkEventWindowPhysiPlanEndCond = "EndCond"; + +static int32_t physiEventWindowNodeToJson(const void* pObj, SJson* pJson) { + const SEventWinodwPhysiNode* pNode = (const SEventWinodwPhysiNode*)pObj; + + int32_t code = physiWindowNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkEventWindowPhysiPlanStartCond, nodeToJson, pNode->pStartCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkEventWindowPhysiPlanEndCond, nodeToJson, pNode->pEndCond); + } + + return code; +} + +static int32_t jsonToPhysiEventWindowNode(const SJson* pJson, void* pObj) { + SEventWinodwPhysiNode* pNode = (SEventWinodwPhysiNode*)pObj; + + int32_t code = jsonToPhysiWindowNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkEventWindowPhysiPlanStartCond, &pNode->pStartCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkEventWindowPhysiPlanEndCond, &pNode->pEndCond); + } + + return code; +} + static const char* jkPartitionPhysiPlanExprs = "Exprs"; static const char* jkPartitionPhysiPlanPartitionKeys = "PartitionKeys"; static const char* jkPartitionPhysiPlanTargets = "Targets"; @@ -3539,6 +3659,51 @@ static int32_t jsonToTempTableNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkJoinTableJoinType = "JoinType"; +static const char* jkJoinTableLeft = "Left"; +static const char* jkJoinTableRight = "Right"; +static const char* jkJoinTableOnCond = "OnCond"; + +static int32_t joinTableNodeToJson(const void* pObj, SJson* pJson) { + const SJoinTableNode* pNode = (const SJoinTableNode*)pObj; + + int32_t code = tableNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkJoinTableJoinType, pNode->joinType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkJoinTableLeft, nodeToJson, pNode->pLeft); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkJoinTableRight, nodeToJson, pNode->pRight); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkJoinTableOnCond, nodeToJson, pNode->pOnCond); + } + + return code; +} + +static int32_t jsonToJoinTableNode(const SJson* pJson, void* pObj) { + SJoinTableNode* pNode = (SJoinTableNode*)pObj; + + int32_t code = jsonToTableNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + tjsonGetNumberValue(pJson, jkJoinTableJoinType, pNode->joinType, code); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinTableLeft, &pNode->pLeft); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinTableRight, &pNode->pRight); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinTableOnCond, &pNode->pOnCond); + } + + return code; +} + static const char* jkGroupingSetType = "GroupingSetType"; static const char* jkGroupingSetParameter = "Parameters"; @@ -3667,6 +3832,36 @@ static int32_t jsonToSessionWindowNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkEventWindowTsPrimaryKey = "TsPrimaryKey"; +static const char* jkEventWindowStartCond = "StartCond"; +static const char* jkEventWindowEndCond = "EndCond"; + +static int32_t eventWindowNodeToJson(const void* pObj, SJson* pJson) { + const SEventWindowNode* pNode = (const SEventWindowNode*)pObj; + + int32_t code = tjsonAddObject(pJson, jkEventWindowTsPrimaryKey, nodeToJson, pNode->pCol); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkEventWindowStartCond, nodeToJson, pNode->pStartCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkEventWindowEndCond, nodeToJson, pNode->pEndCond); + } + return code; +} + +static int32_t jsonToEventWindowNode(const SJson* pJson, void* pObj) { + SEventWindowNode* pNode = (SEventWindowNode*)pObj; + + int32_t code = jsonToNodeObject(pJson, jkEventWindowTsPrimaryKey, &pNode->pCol); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkEventWindowStartCond, &pNode->pStartCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkEventWindowEndCond, &pNode->pEndCond); + } + return code; +} + static const char* jkIntervalWindowInterval = "Interval"; static const char* jkIntervalWindowOffset = "Offset"; static const char* jkIntervalWindowSliding = "Sliding"; @@ -3863,6 +4058,45 @@ static int32_t jsonToSlotDescNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkColumnDefColName = "ColName"; +static const char* jkColumnDefDataType = "DataType"; +static const char* jkColumnDefComments = "Comments"; +static const char* jkColumnDefSma = "Sma"; + +static int32_t columnDefNodeToJson(const void* pObj, SJson* pJson) { + const SColumnDefNode* pNode = (const SColumnDefNode*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkColumnDefColName, pNode->colName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkColumnDefDataType, dataTypeToJson, &pNode->dataType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkColumnDefComments, pNode->comments); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkColumnDefSma, pNode->sma); + } + + return code; +} + +static int32_t jsonToColumnDefNode(const SJson* pJson, void* pObj) { + SColumnDefNode* pNode = (SColumnDefNode*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkColumnDefColName, pNode->colName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToObject(pJson, jkColumnDefDataType, jsonToDataType, &pNode->dataType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkColumnDefComments, pNode->comments); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkColumnDefSma, &pNode->sma); + } + + return code; +} + static const char* jkDownstreamSourceAddr = "Addr"; static const char* jkDownstreamSourceTaskId = "TaskId"; static const char* jkDownstreamSourceSchedId = "SchedId"; @@ -4053,6 +4287,190 @@ static int32_t jsonToDatabaseOptions(const SJson* pJson, void* pObj) { return code; } +static const char* jkTableOptionsComment = "Comment"; +static const char* jkTableOptionsMaxDelay = "MaxDelay"; +static const char* jkTableOptionsWatermark = "Watermark"; +static const char* jkTableOptionsDeleteMark = "DeleteMark"; +static const char* jkTableOptionsRollupFuncs = "RollupFuncs"; +static const char* jkTableOptionsTtl = "Ttl"; +static const char* jkTableOptionsSma = "Sma"; + +static int32_t tableOptionsToJson(const void* pObj, SJson* pJson) { + const STableOptions* pNode = (const STableOptions*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkTableOptionsComment, pNode->comment); + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkTableOptionsMaxDelay, pNode->pMaxDelay); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkTableOptionsWatermark, pNode->pWatermark); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkTableOptionsDeleteMark, pNode->pDeleteMark); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkTableOptionsRollupFuncs, pNode->pRollupFuncs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableOptionsTtl, pNode->ttl); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkTableOptionsSma, pNode->pSma); + } + + return code; +} + +static int32_t jsonToTableOptions(const SJson* pJson, void* pObj) { + STableOptions* pNode = (STableOptions*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkTableOptionsComment, pNode->comment); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkTableOptionsMaxDelay, &pNode->pMaxDelay); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkTableOptionsWatermark, &pNode->pWatermark); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkTableOptionsDeleteMark, &pNode->pDeleteMark); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkTableOptionsRollupFuncs, &pNode->pRollupFuncs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkTableOptionsTtl, &pNode->ttl); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkTableOptionsSma, &pNode->pSma); + } + + return code; +} + +static const char* jkIndexOptionsFuncs = "Funcs"; +static const char* jkIndexOptionsInterval = "Interval"; +static const char* jkIndexOptionsOffset = "Offset"; +static const char* jkIndexOptionsSliding = "Sliding"; +static const char* jkIndexOptionsStreamOptions = "StreamOptions"; + +static int32_t indexOptionsToJson(const void* pObj, SJson* pJson) { + const SIndexOptions* pNode = (const SIndexOptions*)pObj; + + int32_t code = nodeListToJson(pJson, jkIndexOptionsFuncs, pNode->pFuncs); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkIndexOptionsInterval, nodeToJson, pNode->pInterval); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkIndexOptionsOffset, nodeToJson, pNode->pOffset); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkIndexOptionsSliding, nodeToJson, pNode->pSliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkIndexOptionsStreamOptions, nodeToJson, pNode->pStreamOptions); + } + + return code; +} + +static int32_t jsonToIndexOptions(const SJson* pJson, void* pObj) { + SIndexOptions* pNode = (SIndexOptions*)pObj; + + int32_t code = jsonToNodeList(pJson, jkIndexOptionsFuncs, &pNode->pFuncs); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkIndexOptionsInterval, &pNode->pInterval); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkIndexOptionsOffset, &pNode->pOffset); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkIndexOptionsSliding, &pNode->pSliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkIndexOptionsStreamOptions, &pNode->pStreamOptions); + } + + return code; +} + +static const char* jkExplainOptionsVerbose = "Verbose"; +static const char* jkExplainOptionsRatio = "Ratio"; + +static int32_t explainOptionsToJson(const void* pObj, SJson* pJson) { + const SExplainOptions* pNode = (const SExplainOptions*)pObj; + + int32_t code = tjsonAddBoolToObject(pJson, jkExplainOptionsVerbose, pNode->verbose); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddDoubleToObject(pJson, jkExplainOptionsRatio, pNode->ratio); + } + + return code; +} + +static int32_t jsonToExplainOptions(const SJson* pJson, void* pObj) { + SExplainOptions* pNode = (SExplainOptions*)pObj; + + int32_t code = tjsonGetBoolValue(pJson, jkExplainOptionsVerbose, &pNode->verbose); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetDoubleValue(pJson, jkExplainOptionsRatio, &pNode->ratio); + } + + return code; +} + +static const char* jkStreamOptionsTriggerType = "TriggerType"; +static const char* jkStreamOptionsDelay = "Delay"; +static const char* jkStreamOptionsWatermark = "Watermark"; +static const char* jkStreamOptionsDeleteMark = "DeleteMark"; +static const char* jkStreamOptionsFillHistory = "FillHistory"; +static const char* jkStreamOptionsIgnoreExpired = "IgnoreExpired"; + +static int32_t streamOptionsToJson(const void* pObj, SJson* pJson) { + const SStreamOptions* pNode = (const SStreamOptions*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkStreamOptionsTriggerType, pNode->triggerType); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkStreamOptionsDelay, nodeToJson, pNode->pDelay); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkStreamOptionsWatermark, nodeToJson, pNode->pWatermark); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkStreamOptionsDeleteMark, nodeToJson, pNode->pDeleteMark); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkStreamOptionsFillHistory, pNode->fillHistory); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkStreamOptionsIgnoreExpired, pNode->ignoreExpired); + } + + return code; +} + +static int32_t jsonToStreamOptions(const SJson* pJson, void* pObj) { + SStreamOptions* pNode = (SStreamOptions*)pObj; + + int32_t code = tjsonGetTinyIntValue(pJson, jkStreamOptionsTriggerType, &pNode->triggerType); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkStreamOptionsDelay, &pNode->pDelay); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkStreamOptionsWatermark, &pNode->pWatermark); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkStreamOptionsDeleteMark, &pNode->pDeleteMark); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkStreamOptionsFillHistory, &pNode->fillHistory); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkStreamOptionsIgnoreExpired, &pNode->ignoreExpired); + } + + return code; +} + static const char* jkWhenThenWhen = "When"; static const char* jkWhenThenThen = "Then"; @@ -4338,164 +4756,1402 @@ static int32_t jsonToSelectStmt(const SJson* pJson, void* pObj) { return code; } -static const char* jkAlterDatabaseStmtDbName = "DbName"; -static const char* jkAlterDatabaseStmtOptions = "Options"; +static const char* jkVnodeModifyOpStmtSqlNodeType = "SqlNodeType"; +static const char* jkVnodeModifyOpStmtTotalRowsNum = "TotalRowsNum"; +static const char* jkVnodeModifyOpStmtTotalTbNum = "TotalTbNum"; -static int32_t alterDatabaseStmtToJson(const void* pObj, SJson* pJson) { - const SAlterDatabaseStmt* pNode = (const SAlterDatabaseStmt*)pObj; +static int32_t vnodeModifyStmtToJson(const void* pObj, SJson* pJson) { + const SVnodeModifyOpStmt* pNode = (const SVnodeModifyOpStmt*)pObj; - int32_t code = tjsonAddStringToObject(pJson, jkAlterDatabaseStmtDbName, pNode->dbName); + int32_t code = tjsonAddIntegerToObject(pJson, jkVnodeModifyOpStmtSqlNodeType, pNode->sqlNodeType); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkAlterDatabaseStmtOptions, nodeToJson, pNode->pOptions); + code = tjsonAddIntegerToObject(pJson, jkVnodeModifyOpStmtTotalRowsNum, pNode->totalRowsNum); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkVnodeModifyOpStmtTotalTbNum, pNode->totalTbNum); } return code; } -static int32_t jsonToAlterDatabaseStmt(const SJson* pJson, void* pObj) { - SAlterDatabaseStmt* pNode = (SAlterDatabaseStmt*)pObj; +static int32_t jsonToVnodeModifyStmt(const SJson* pJson, void* pObj) { + SVnodeModifyOpStmt* pNode = (SVnodeModifyOpStmt*)pObj; - int32_t code = tjsonGetStringValue(pJson, jkAlterDatabaseStmtDbName, pNode->dbName); + int32_t code = TSDB_CODE_SUCCESS; + tjsonGetNumberValue(pJson, jkVnodeModifyOpStmtSqlNodeType, pNode->sqlNodeType, code); if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkAlterDatabaseStmtOptions, (SNode**)&pNode->pOptions); + code = tjsonGetIntValue(pJson, jkVnodeModifyOpStmtTotalRowsNum, &pNode->totalRowsNum); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkVnodeModifyOpStmtTotalTbNum, &pNode->totalTbNum); } return code; } -static const char* jkAlterTableStmtDbName = "DbName"; -static const char* jkAlterTableStmtTableName = "TableName"; -static const char* jkAlterTableStmtAlterType = "AlterType"; -static const char* jkAlterTableStmtColName = "ColName"; -static const char* jkAlterTableStmtNewColName = "NewColName"; -static const char* jkAlterTableStmtOptions = "Options"; -static const char* jkAlterTableStmtNewDataType = "NewDataType"; -static const char* jkAlterTableStmtNewTagVal = "NewTagVal"; +static const char* jkCreateDatabaseStmtDbName = "DbName"; +static const char* jkCreateDatabaseStmtIgnoreExists = "IgnoreExists"; +static const char* jkCreateDatabaseStmtOptions = "Options"; -static int32_t alterTableStmtToJson(const void* pObj, SJson* pJson) { - const SAlterTableStmt* pNode = (const SAlterTableStmt*)pObj; +static int32_t createDatabaseStmtToJson(const void* pObj, SJson* pJson) { + const SCreateDatabaseStmt* pNode = (const SCreateDatabaseStmt*)pObj; - int32_t code = tjsonAddStringToObject(pJson, jkAlterTableStmtDbName, pNode->dbName); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddStringToObject(pJson, jkAlterTableStmtTableName, pNode->tableName); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkAlterTableStmtAlterType, pNode->alterType); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddStringToObject(pJson, jkAlterTableStmtColName, pNode->colName); - } + int32_t code = tjsonAddStringToObject(pJson, jkCreateDatabaseStmtDbName, pNode->dbName); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddStringToObject(pJson, jkAlterTableStmtNewColName, pNode->newColName); + code = tjsonAddBoolToObject(pJson, jkCreateDatabaseStmtIgnoreExists, pNode->ignoreExists); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkAlterTableStmtOptions, nodeToJson, pNode->pOptions); + code = tjsonAddObject(pJson, jkCreateDatabaseStmtOptions, nodeToJson, pNode->pOptions); } + + return code; +} + +static int32_t jsonToCreateDatabaseStmt(const SJson* pJson, void* pObj) { + SCreateDatabaseStmt* pNode = (SCreateDatabaseStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkCreateDatabaseStmtDbName, pNode->dbName); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkAlterTableStmtNewDataType, dataTypeToJson, &pNode->dataType); + code = tjsonGetBoolValue(pJson, jkCreateDatabaseStmtIgnoreExists, &pNode->ignoreExists); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkAlterTableStmtOptions, nodeToJson, pNode->pVal); + code = jsonToNodeObject(pJson, jkCreateDatabaseStmtOptions, (SNode**)&pNode->pOptions); } return code; } -static int32_t jsonToAlterTableStmt(const SJson* pJson, void* pObj) { - SAlterTableStmt* pNode = (SAlterTableStmt*)pObj; +static const char* jkAlterDatabaseStmtDbName = "DbName"; +static const char* jkAlterDatabaseStmtOptions = "Options"; + +static int32_t alterDatabaseStmtToJson(const void* pObj, SJson* pJson) { + const SAlterDatabaseStmt* pNode = (const SAlterDatabaseStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkAlterDatabaseStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkAlterDatabaseStmtOptions, nodeToJson, pNode->pOptions); + } + + return code; +} + +static int32_t jsonToAlterDatabaseStmt(const SJson* pJson, void* pObj) { + SAlterDatabaseStmt* pNode = (SAlterDatabaseStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkAlterDatabaseStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkAlterDatabaseStmtOptions, (SNode**)&pNode->pOptions); + } + + return code; +} + +static const char* jkTrimDatabaseStmtDbName = "DbName"; +static const char* jkTrimDatabaseStmtMaxSpeed = "MaxSpeed"; + +static int32_t trimDatabaseStmtToJson(const void* pObj, SJson* pJson) { + const STrimDatabaseStmt* pNode = (const STrimDatabaseStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkTrimDatabaseStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTrimDatabaseStmtMaxSpeed, pNode->maxSpeed); + } + + return code; +} + +static int32_t jsonToTrimDatabaseStmt(const SJson* pJson, void* pObj) { + STrimDatabaseStmt* pNode = (STrimDatabaseStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkTrimDatabaseStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkTrimDatabaseStmtMaxSpeed, &pNode->maxSpeed); + } + + return code; +} + +static const char* jkCreateTableStmtDbName = "DbName"; +static const char* jkCreateTableStmtTableName = "TableName"; +static const char* jkCreateTableStmtIgnoreExists = "IgnoreExists"; +static const char* jkCreateTableStmtCols = "Cols"; +static const char* jkCreateTableStmtTags = "Tags"; +static const char* jkCreateTableStmtOptions = "Options"; + +static int32_t createTableStmtToJson(const void* pObj, SJson* pJson) { + const SCreateTableStmt* pNode = (const SCreateTableStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkCreateTableStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateTableStmtTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkCreateTableStmtIgnoreExists, pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkCreateTableStmtCols, pNode->pCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkCreateTableStmtTags, pNode->pTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkCreateTableStmtOptions, nodeToJson, pNode->pOptions); + } + + return code; +} + +static int32_t jsonToCreateTableStmt(const SJson* pJson, void* pObj) { + SCreateTableStmt* pNode = (SCreateTableStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkCreateTableStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateTableStmtTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkCreateTableStmtIgnoreExists, &pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkCreateTableStmtCols, &pNode->pCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkCreateTableStmtTags, &pNode->pTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkCreateTableStmtOptions, (SNode**)&pNode->pOptions); + } + + return code; +} + +static const char* jkCreateSubTableClauseDbName = "DbName"; +static const char* jkCreateSubTableClauseTableName = "TableName"; +static const char* jkCreateSubTableClauseUseDbName = "UseDbName"; +static const char* jkCreateSubTableClauseUseTableName = "UseTableName"; +static const char* jkCreateSubTableClauseIgnoreExists = "IgnoreExists"; +static const char* jkCreateSubTableClauseSpecificTags = "SpecificTags"; +static const char* jkCreateSubTableClauseValsOfTags = "ValsOfTags"; +static const char* jkCreateSubTableClauseOptions = "Options"; + +static int32_t createSubTableClauseToJson(const void* pObj, SJson* pJson) { + const SCreateSubTableClause* pNode = (const SCreateSubTableClause*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkCreateSubTableClauseDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateSubTableClauseTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateSubTableClauseUseDbName, pNode->useDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateSubTableClauseUseTableName, pNode->useTableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkCreateSubTableClauseIgnoreExists, pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkCreateSubTableClauseSpecificTags, pNode->pSpecificTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkCreateSubTableClauseValsOfTags, pNode->pValsOfTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkCreateSubTableClauseOptions, nodeToJson, pNode->pOptions); + } + + return code; +} + +static int32_t jsonToCreateSubTableClause(const SJson* pJson, void* pObj) { + SCreateSubTableClause* pNode = (SCreateSubTableClause*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkCreateSubTableClauseDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateSubTableClauseTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateSubTableClauseUseDbName, pNode->useDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateSubTableClauseUseTableName, pNode->useTableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkCreateSubTableClauseIgnoreExists, &pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkCreateSubTableClauseSpecificTags, &pNode->pSpecificTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkCreateSubTableClauseValsOfTags, &pNode->pValsOfTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkCreateSubTableClauseOptions, (SNode**)&pNode->pOptions); + } + + return code; +} + +static const char* jkCreateMultiTablesStmtSubTables = "SubTables"; + +static int32_t createMultiTablesStmtToJson(const void* pObj, SJson* pJson) { + const SCreateMultiTablesStmt* pNode = (const SCreateMultiTablesStmt*)pObj; + return nodeListToJson(pJson, jkCreateMultiTablesStmtSubTables, pNode->pSubTables); +} + +static int32_t jsonToCreateMultiTablesStmt(const SJson* pJson, void* pObj) { + SCreateMultiTablesStmt* pNode = (SCreateMultiTablesStmt*)pObj; + return jsonToNodeList(pJson, jkCreateMultiTablesStmtSubTables, &pNode->pSubTables); +} + +static const char* jkDropTableClauseDbName = "DbName"; +static const char* jkDropTableClauseTableName = "TableName"; +static const char* jkDropTableClauseIgnoreNotExists = "IgnoreNotExists"; + +static int32_t dropTableClauseToJson(const void* pObj, SJson* pJson) { + const SDropTableClause* pNode = (const SDropTableClause*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkDropTableClauseDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkDropTableClauseTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkDropTableClauseIgnoreNotExists, pNode->ignoreNotExists); + } + + return code; +} + +static int32_t jsonToDropTableClause(const SJson* pJson, void* pObj) { + SDropTableClause* pNode = (SDropTableClause*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkDropTableClauseDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkDropTableClauseTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkDropTableClauseIgnoreNotExists, &pNode->ignoreNotExists); + } + + return code; +} + +static const char* jkDropTableStmtTables = "Tables"; + +static int32_t dropTableStmtToJson(const void* pObj, SJson* pJson) { + const SDropTableStmt* pNode = (const SDropTableStmt*)pObj; + return nodeListToJson(pJson, jkDropTableStmtTables, pNode->pTables); +} + +static int32_t jsonToDropTableStmt(const SJson* pJson, void* pObj) { + SDropTableStmt* pNode = (SDropTableStmt*)pObj; + return jsonToNodeList(pJson, jkDropTableStmtTables, &pNode->pTables); +} + +static const char* jkDropSuperTableStmtDbName = "DbName"; +static const char* jkDropSuperTableStmtTableName = "TableName"; +static const char* jkDropSuperTableStmtIgnoreNotExists = "IgnoreNotExists"; + +static int32_t dropStableStmtToJson(const void* pObj, SJson* pJson) { + const SDropSuperTableStmt* pNode = (const SDropSuperTableStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkDropSuperTableStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkDropSuperTableStmtTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkDropSuperTableStmtIgnoreNotExists, pNode->ignoreNotExists); + } + + return code; +} + +static int32_t jsonToDropStableStmt(const SJson* pJson, void* pObj) { + SDropSuperTableStmt* pNode = (SDropSuperTableStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkDropSuperTableStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkDropSuperTableStmtTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkDropSuperTableStmtIgnoreNotExists, &pNode->ignoreNotExists); + } + + return code; +} + +static const char* jkAlterTableStmtDbName = "DbName"; +static const char* jkAlterTableStmtTableName = "TableName"; +static const char* jkAlterTableStmtAlterType = "AlterType"; +static const char* jkAlterTableStmtColName = "ColName"; +static const char* jkAlterTableStmtNewColName = "NewColName"; +static const char* jkAlterTableStmtOptions = "Options"; +static const char* jkAlterTableStmtNewDataType = "NewDataType"; +static const char* jkAlterTableStmtNewTagVal = "NewTagVal"; + +static int32_t alterTableStmtToJson(const void* pObj, SJson* pJson) { + const SAlterTableStmt* pNode = (const SAlterTableStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkAlterTableStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkAlterTableStmtTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkAlterTableStmtAlterType, pNode->alterType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkAlterTableStmtColName, pNode->colName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkAlterTableStmtNewColName, pNode->newColName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkAlterTableStmtOptions, nodeToJson, pNode->pOptions); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkAlterTableStmtNewDataType, dataTypeToJson, &pNode->dataType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkAlterTableStmtOptions, nodeToJson, pNode->pVal); + } + + return code; +} + +static int32_t jsonToAlterTableStmt(const SJson* pJson, void* pObj) { + SAlterTableStmt* pNode = (SAlterTableStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkAlterTableStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkAlterTableStmtTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkAlterTableStmtAlterType, &pNode->alterType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkAlterTableStmtColName, pNode->colName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkAlterTableStmtNewColName, pNode->newColName); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkAlterTableStmtOptions, (SNode**)&pNode->pOptions); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToObject(pJson, jkAlterTableStmtNewDataType, jsonToDataType, &pNode->dataType); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkAlterTableStmtOptions, (SNode**)&pNode->pVal); + } + + return code; +} + +static int32_t alterStableStmtToJson(const void* pObj, SJson* pJson) { return alterTableStmtToJson(pObj, pJson); } + +static int32_t jsonToAlterStableStmt(const SJson* pJson, void* pObj) { return jsonToAlterTableStmt(pJson, pObj); } + +static const char* jkCreateUserStmtUserName = "UserName"; +static const char* jkCreateUserStmtPassword = "Password"; +static const char* jkCreateUserStmtSysinfo = "Sysinfo"; + +static int32_t createUserStmtToJson(const void* pObj, SJson* pJson) { + const SCreateUserStmt* pNode = (const SCreateUserStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkCreateUserStmtUserName, pNode->userName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateUserStmtPassword, pNode->password); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkCreateUserStmtSysinfo, pNode->sysinfo); + } + + return code; +} + +static int32_t jsonToCreateUserStmt(const SJson* pJson, void* pObj) { + SCreateUserStmt* pNode = (SCreateUserStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkCreateUserStmtUserName, pNode->userName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateUserStmtPassword, pNode->password); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkCreateUserStmtSysinfo, &pNode->sysinfo); + } + + return code; +} + +static const char* jkAlterUserStmtUserName = "UserName"; +static const char* jkAlterUserStmtAlterType = "AlterType"; +static const char* jkAlterUserStmtPassword = "Password"; +static const char* jkAlterUserStmtEnable = "Enable"; +static const char* jkAlterUserStmtSysinfo = "Sysinfo"; + +static int32_t alterUserStmtToJson(const void* pObj, SJson* pJson) { + const SAlterUserStmt* pNode = (const SAlterUserStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkAlterUserStmtUserName, pNode->userName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkAlterUserStmtAlterType, pNode->alterType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkAlterUserStmtPassword, pNode->password); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkAlterUserStmtEnable, pNode->enable); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkAlterUserStmtSysinfo, pNode->sysinfo); + } + + return code; +} + +static int32_t jsonToAlterUserStmt(const SJson* pJson, void* pObj) { + SAlterUserStmt* pNode = (SAlterUserStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkAlterUserStmtUserName, pNode->userName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkAlterUserStmtAlterType, &pNode->alterType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkAlterUserStmtPassword, pNode->password); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkAlterUserStmtEnable, &pNode->enable); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkAlterUserStmtSysinfo, &pNode->sysinfo); + } + + return code; +} + +static const char* jkDropUserStmtUserName = "UserName"; + +static int32_t dropUserStmtToJson(const void* pObj, SJson* pJson) { + const SDropUserStmt* pNode = (const SDropUserStmt*)pObj; + return tjsonAddStringToObject(pJson, jkDropUserStmtUserName, pNode->userName); +} + +static int32_t jsonToDropUserStmt(const SJson* pJson, void* pObj) { + SDropUserStmt* pNode = (SDropUserStmt*)pObj; + return tjsonGetStringValue(pJson, jkDropUserStmtUserName, pNode->userName); +} + +static const char* jkUseDatabaseStmtDbName = "DbName"; + +static int32_t useDatabaseStmtToJson(const void* pObj, SJson* pJson) { + const SUseDatabaseStmt* pNode = (const SUseDatabaseStmt*)pObj; + return tjsonAddStringToObject(pJson, jkUseDatabaseStmtDbName, pNode->dbName); +} + +static int32_t jsonToUseDatabaseStmt(const SJson* pJson, void* pObj) { + SUseDatabaseStmt* pNode = (SUseDatabaseStmt*)pObj; + return tjsonGetStringValue(pJson, jkUseDatabaseStmtDbName, pNode->dbName); +} + +static const char* jkCreateDnodeStmtFqdn = "Fqdn"; +static const char* jkCreateDnodeStmtPort = "Port"; + +static int32_t createDnodeStmtToJson(const void* pObj, SJson* pJson) { + const SCreateDnodeStmt* pNode = (const SCreateDnodeStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkCreateDnodeStmtFqdn, pNode->fqdn); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkCreateDnodeStmtPort, pNode->port); + } + + return code; +} + +static int32_t jsonToCreateDnodeStmt(const SJson* pJson, void* pObj) { + SCreateDnodeStmt* pNode = (SCreateDnodeStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkCreateDnodeStmtFqdn, pNode->fqdn); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkCreateDnodeStmtPort, &pNode->port); + } + + return code; +} + +static const char* jkAlterDnodeStmtDnodeId = "DnodeId"; +static const char* jkAlterDnodeStmtConfig = "Config"; +static const char* jkAlterDnodeStmtValue = "Value"; + +static int32_t alterDnodeStmtToJson(const void* pObj, SJson* pJson) { + const SAlterDnodeStmt* pNode = (const SAlterDnodeStmt*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkAlterDnodeStmtDnodeId, pNode->dnodeId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkAlterDnodeStmtConfig, pNode->config); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkAlterDnodeStmtValue, pNode->value); + } + + return code; +} + +static int32_t jsonToAlterDnodeStmt(const SJson* pJson, void* pObj) { + SAlterDnodeStmt* pNode = (SAlterDnodeStmt*)pObj; + + int32_t code = tjsonGetIntValue(pJson, jkAlterDnodeStmtDnodeId, &pNode->dnodeId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkAlterDnodeStmtConfig, pNode->config); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkAlterDnodeStmtValue, pNode->value); + } + + return code; +} + +static const char* jkCreateIndexStmtIndexType = "IndexType"; +static const char* jkCreateIndexStmtIgnoreExists = "IgnoreExists"; +static const char* jkCreateIndexStmtIndexDbName = "IndexDbName"; +static const char* jkCreateIndexStmtIndexName = "indexName"; +static const char* jkCreateIndexStmtDbName = "DbName"; +static const char* jkCreateIndexStmtTableName = "TableName"; +static const char* jkCreateIndexStmtCols = "Cols"; +static const char* jkCreateIndexStmtOptions = "Options"; + +static int32_t createIndexStmtToJson(const void* pObj, SJson* pJson) { + const SCreateIndexStmt* pNode = (const SCreateIndexStmt*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkCreateIndexStmtIndexType, pNode->indexType); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkCreateIndexStmtIgnoreExists, pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateIndexStmtIndexDbName, pNode->indexDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateIndexStmtIndexName, pNode->indexName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateIndexStmtDbName, pNode->dbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateIndexStmtTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkCreateIndexStmtCols, pNode->pCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkCreateIndexStmtOptions, nodeToJson, pNode->pOptions); + } + + return code; +} + +static int32_t jsonToCreateIndexStmt(const SJson* pJson, void* pObj) { + SCreateIndexStmt* pNode = (SCreateIndexStmt*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + tjsonGetNumberValue(pJson, jkCreateIndexStmtIndexType, pNode->indexType, code); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkCreateIndexStmtIgnoreExists, &pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateIndexStmtIndexDbName, pNode->indexDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateIndexStmtIndexName, pNode->indexName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateIndexStmtDbName, pNode->dbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateIndexStmtTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkCreateIndexStmtCols, &pNode->pCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkCreateIndexStmtOptions, (SNode**)&pNode->pOptions); + } + + return code; +} + +static const char* jkDropIndexStmtIgnoreNotExists = "IgnoreNotExists"; +static const char* jkDropIndexStmtIndexDbName = "IndexDbName"; +static const char* jkDropIndexStmtIndexName = "IndexName"; + +static int32_t dropIndexStmtToJson(const void* pObj, SJson* pJson) { + const SDropIndexStmt* pNode = (const SDropIndexStmt*)pObj; + + int32_t code = tjsonAddBoolToObject(pJson, jkDropIndexStmtIgnoreNotExists, pNode->ignoreNotExists); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkDropIndexStmtIndexDbName, pNode->indexDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkDropIndexStmtIndexName, pNode->indexName); + } + + return code; +} + +static int32_t jsonToDropIndexStmt(const SJson* pJson, void* pObj) { + SDropIndexStmt* pNode = (SDropIndexStmt*)pObj; + + int32_t code = tjsonGetBoolValue(pJson, jkDropIndexStmtIgnoreNotExists, &pNode->ignoreNotExists); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkDropIndexStmtIndexDbName, pNode->indexDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkDropIndexStmtIndexName, pNode->indexName); + } + + return code; +} + +static const char* jkCreateComponentNodeStmtDnodeId = "DnodeId"; + +static int32_t createComponentNodeStmtToJson(const void* pObj, SJson* pJson) { + const SCreateComponentNodeStmt* pNode = (const SCreateComponentNodeStmt*)pObj; + return tjsonAddIntegerToObject(pJson, jkCreateComponentNodeStmtDnodeId, pNode->dnodeId); +} + +static int32_t jsonToCreateComponentNodeStmt(const SJson* pJson, void* pObj) { + SCreateComponentNodeStmt* pNode = (SCreateComponentNodeStmt*)pObj; + return tjsonGetIntValue(pJson, jkCreateComponentNodeStmtDnodeId, &pNode->dnodeId); +} + +static const char* jkDropComponentNodeStmtDnodeId = "DnodeId"; + +static int32_t dropComponentNodeStmtToJson(const void* pObj, SJson* pJson) { + const SDropComponentNodeStmt* pNode = (const SDropComponentNodeStmt*)pObj; + return tjsonAddIntegerToObject(pJson, jkDropComponentNodeStmtDnodeId, pNode->dnodeId); +} + +static int32_t jsonToDropComponentNodeStmt(const SJson* pJson, void* pObj) { + SDropComponentNodeStmt* pNode = (SDropComponentNodeStmt*)pObj; + return tjsonGetIntValue(pJson, jkDropComponentNodeStmtDnodeId, &pNode->dnodeId); +} + +static int32_t createQnodeStmtToJson(const void* pObj, SJson* pJson) { + return createComponentNodeStmtToJson(pObj, pJson); +} + +static int32_t jsonToCreateQnodeStmt(const SJson* pJson, void* pObj) { + return jsonToCreateComponentNodeStmt(pJson, pObj); +} + +static int32_t dropQnodeStmtToJson(const void* pObj, SJson* pJson) { return dropComponentNodeStmtToJson(pObj, pJson); } + +static int32_t jsonToDropQnodeStmt(const SJson* pJson, void* pObj) { return jsonToDropComponentNodeStmt(pJson, pObj); } + +static int32_t createSnodeStmtToJson(const void* pObj, SJson* pJson) { + return createComponentNodeStmtToJson(pObj, pJson); +} + +static int32_t jsonToCreateSnodeStmt(const SJson* pJson, void* pObj) { + return jsonToCreateComponentNodeStmt(pJson, pObj); +} + +static int32_t dropSnodeStmtToJson(const void* pObj, SJson* pJson) { return dropComponentNodeStmtToJson(pObj, pJson); } + +static int32_t jsonToDropSnodeStmt(const SJson* pJson, void* pObj) { return jsonToDropComponentNodeStmt(pJson, pObj); } + +static int32_t createMnodeStmtToJson(const void* pObj, SJson* pJson) { + return createComponentNodeStmtToJson(pObj, pJson); +} + +static int32_t jsonToCreateMnodeStmt(const SJson* pJson, void* pObj) { + return jsonToCreateComponentNodeStmt(pJson, pObj); +} + +static int32_t dropMnodeStmtToJson(const void* pObj, SJson* pJson) { return dropComponentNodeStmtToJson(pObj, pJson); } + +static int32_t jsonToDropMnodeStmt(const SJson* pJson, void* pObj) { return jsonToDropComponentNodeStmt(pJson, pObj); } + +static const char* jkDropDnodeStmtDnodeId = "DnodeId"; +static const char* jkDropDnodeStmtFqdn = "Fqdn"; +static const char* jkDropDnodeStmtPort = "Port"; +static const char* jkDropDnodeStmtForce = "Force"; + +static int32_t dropDnodeStmtToJson(const void* pObj, SJson* pJson) { + const SDropDnodeStmt* pNode = (const SDropDnodeStmt*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkDropDnodeStmtDnodeId, pNode->dnodeId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkDropDnodeStmtFqdn, pNode->fqdn); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDropDnodeStmtPort, pNode->port); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkDropDnodeStmtForce, pNode->force); + } + + return code; +} + +static int32_t jsonToDropDnodeStmt(const SJson* pJson, void* pObj) { + SDropDnodeStmt* pNode = (SDropDnodeStmt*)pObj; + + int32_t code = tjsonGetIntValue(pJson, jkDropDnodeStmtDnodeId, &pNode->dnodeId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkDropDnodeStmtFqdn, pNode->fqdn); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkDropDnodeStmtPort, &pNode->port); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkDropDnodeStmtForce, &pNode->force); + } + + return code; +} + +static const char* jkCreateTopicStmtTopicName = "TopicName"; +static const char* jkCreateTopicStmtSubscribeDbName = "SubscribeDbName"; +static const char* jkCreateTopicStmtIgnoreExists = "IgnoreExists"; +static const char* jkCreateTopicStmtQuery = "Query"; + +static int32_t createTopicStmtToJson(const void* pObj, SJson* pJson) { + const SCreateTopicStmt* pNode = (const SCreateTopicStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkCreateTopicStmtTopicName, pNode->topicName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateTopicStmtSubscribeDbName, pNode->subDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkCreateTopicStmtIgnoreExists, pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkCreateTopicStmtQuery, nodeToJson, pNode->pQuery); + } + + return code; +} + +static int32_t jsonToCreateTopicStmt(const SJson* pJson, void* pObj) { + SCreateTopicStmt* pNode = (SCreateTopicStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkCreateTopicStmtTopicName, pNode->topicName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateTopicStmtSubscribeDbName, pNode->subDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkCreateTopicStmtIgnoreExists, &pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkCreateTopicStmtQuery, &pNode->pQuery); + } + + return code; +} + +static const char* jkDropTopicStmtTopicName = "TopicName"; +static const char* jkDropTopicStmtIgnoreNotExists = "IgnoreNotExists"; + +static int32_t dropTopicStmtToJson(const void* pObj, SJson* pJson) { + const SDropTopicStmt* pNode = (const SDropTopicStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkDropTopicStmtTopicName, pNode->topicName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkDropTopicStmtIgnoreNotExists, pNode->ignoreNotExists); + } + + return code; +} + +static int32_t jsonToDropTopicStmt(const SJson* pJson, void* pObj) { + SDropTopicStmt* pNode = (SDropTopicStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkDropTopicStmtTopicName, pNode->topicName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkDropTopicStmtIgnoreNotExists, &pNode->ignoreNotExists); + } + + return code; +} + +static const char* jkDropCGroupStmtTopicName = "TopicName"; +static const char* jkDropCGroupStmtConsumerGroup = "ConsumerGroup"; +static const char* jkDropCGroupStmtIgnoreNotExists = "IgnoreNotExists"; + +static int32_t dropConsumerGroupStmtToJson(const void* pObj, SJson* pJson) { + const SDropCGroupStmt* pNode = (const SDropCGroupStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkDropCGroupStmtTopicName, pNode->topicName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkDropCGroupStmtConsumerGroup, pNode->cgroup); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkDropCGroupStmtIgnoreNotExists, pNode->ignoreNotExists); + } + + return code; +} + +static int32_t jsonToDropConsumerGroupStmt(const SJson* pJson, void* pObj) { + SDropCGroupStmt* pNode = (SDropCGroupStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkDropCGroupStmtTopicName, pNode->topicName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkDropCGroupStmtConsumerGroup, pNode->cgroup); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkDropCGroupStmtIgnoreNotExists, &pNode->ignoreNotExists); + } + + return code; +} + +static const char* jkAlterLocalStmtConfig = "Config"; +static const char* jkAlterLocalStmtValue = "Value"; + +static int32_t alterLocalStmtToJson(const void* pObj, SJson* pJson) { + const SAlterLocalStmt* pNode = (const SAlterLocalStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkAlterLocalStmtConfig, pNode->config); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkAlterLocalStmtValue, pNode->value); + } + + return code; +} + +static int32_t jsonToAlterLocalStmt(const SJson* pJson, void* pObj) { + SAlterLocalStmt* pNode = (SAlterLocalStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkAlterLocalStmtConfig, pNode->config); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkAlterLocalStmtValue, pNode->value); + } + + return code; +} + +static const char* jkExplainStmtAnalyze = "Analyze"; +static const char* jkExplainStmtOptions = "Options"; +static const char* jkExplainStmtQuery = "Query"; + +static int32_t explainStmtToJson(const void* pObj, SJson* pJson) { + const SExplainStmt* pNode = (const SExplainStmt*)pObj; + + int32_t code = tjsonAddBoolToObject(pJson, jkExplainStmtAnalyze, pNode->analyze); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkExplainStmtOptions, nodeToJson, pNode->pOptions); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkExplainStmtQuery, nodeToJson, pNode->pQuery); + } + + return code; +} + +static int32_t jsonToExplainStmt(const SJson* pJson, void* pObj) { + SExplainStmt* pNode = (SExplainStmt*)pObj; + + int32_t code = tjsonGetBoolValue(pJson, jkExplainStmtAnalyze, &pNode->analyze); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkExplainStmtOptions, (SNode**)&pNode->pOptions); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkExplainStmtQuery, &pNode->pQuery); + } + + return code; +} + +static const char* jkDescribeStmtDbName = "DbName"; +static const char* jkDescribeStmtTableName = "TableName"; + +static int32_t describeStmtToJson(const void* pObj, SJson* pJson) { + const SDescribeStmt* pNode = (const SDescribeStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkDescribeStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkDescribeStmtTableName, pNode->tableName); + } + + return code; +} + +static int32_t jsonToDescribeStmt(const SJson* pJson, void* pObj) { + SDescribeStmt* pNode = (SDescribeStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkDescribeStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkDescribeStmtTableName, pNode->tableName); + } + + return code; +} + +static const char* jkCreateStreamStmtStreamName = "StreamName"; +static const char* jkCreateStreamStmtTargetDbName = "TargetDbName"; +static const char* jkCreateStreamStmtTargetTabName = "TargetTabName"; +static const char* jkCreateStreamStmtIgnoreExists = "IgnoreExists"; +static const char* jkCreateStreamStmtOptions = "Options"; +static const char* jkCreateStreamStmtQuery = "Query"; +static const char* jkCreateStreamStmtTags = "Tags"; +static const char* jkCreateStreamStmtSubtable = "Subtable"; + +static int32_t createStreamStmtToJson(const void* pObj, SJson* pJson) { + const SCreateStreamStmt* pNode = (const SCreateStreamStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkCreateStreamStmtStreamName, pNode->streamName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateStreamStmtTargetDbName, pNode->targetDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateStreamStmtTargetTabName, pNode->targetTabName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkCreateStreamStmtIgnoreExists, pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkCreateStreamStmtOptions, nodeToJson, pNode->pOptions); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkCreateStreamStmtQuery, nodeToJson, pNode->pQuery); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkCreateStreamStmtTags, pNode->pTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkCreateStreamStmtSubtable, nodeToJson, pNode->pSubtable); + } + + return code; +} + +static int32_t jsonToCreateStreamStmt(const SJson* pJson, void* pObj) { + SCreateStreamStmt* pNode = (SCreateStreamStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkCreateStreamStmtStreamName, pNode->streamName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateStreamStmtTargetDbName, pNode->targetDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateStreamStmtTargetTabName, pNode->targetTabName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkCreateStreamStmtIgnoreExists, &pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkCreateStreamStmtOptions, (SNode**)&pNode->pOptions); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkCreateStreamStmtQuery, &pNode->pQuery); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkCreateStreamStmtTags, &pNode->pTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkCreateStreamStmtSubtable, &pNode->pSubtable); + } + + return code; +} + +static const char* jkDropStreamStmtStreamName = "StreamName"; +static const char* jkDropStreamStmtIgnoreNotExists = "IgnoreNotExists"; + +static int32_t dropStreamStmtToJson(const void* pObj, SJson* pJson) { + const SDropStreamStmt* pNode = (const SDropStreamStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkDropStreamStmtStreamName, pNode->streamName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkDropStreamStmtIgnoreNotExists, pNode->ignoreNotExists); + } + + return code; +} + +static int32_t jsonToDropStreamStmt(const SJson* pJson, void* pObj) { + SDropStreamStmt* pNode = (SDropStreamStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkDropStreamStmtStreamName, pNode->streamName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkDropStreamStmtIgnoreNotExists, &pNode->ignoreNotExists); + } + + return code; +} + +static const char* jkMergeVgroupStmtVgroupId1 = "VgroupId1"; +static const char* jkMergeVgroupStmtVgroupId2 = "VgroupId2"; + +static int32_t mergeVgroupStmtToJson(const void* pObj, SJson* pJson) { + const SMergeVgroupStmt* pNode = (const SMergeVgroupStmt*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkMergeVgroupStmtVgroupId1, pNode->vgId1); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkMergeVgroupStmtVgroupId2, pNode->vgId2); + } + + return code; +} + +static int32_t jsonToMergeVgroupStmt(const SJson* pJson, void* pObj) { + SMergeVgroupStmt* pNode = (SMergeVgroupStmt*)pObj; + + int32_t code = tjsonGetIntValue(pJson, jkMergeVgroupStmtVgroupId1, &pNode->vgId1); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkMergeVgroupStmtVgroupId2, &pNode->vgId2); + } + + return code; +} + +static const char* jkRedistributeVgroupStmtVgroupId = "VgroupId"; +static const char* jkRedistributeVgroupStmtDnodeId1 = "DnodeId1"; +static const char* jkRedistributeVgroupStmtDnodeId2 = "DnodeId2"; +static const char* jkRedistributeVgroupStmtDnodeId3 = "DnodeId3"; +static const char* jkRedistributeVgroupStmtDnodes = "Dnodes"; + +static int32_t redistributeVgroupStmtToJson(const void* pObj, SJson* pJson) { + const SRedistributeVgroupStmt* pNode = (const SRedistributeVgroupStmt*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkRedistributeVgroupStmtVgroupId, pNode->vgId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkRedistributeVgroupStmtDnodeId1, pNode->dnodeId1); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkRedistributeVgroupStmtDnodeId2, pNode->dnodeId2); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkRedistributeVgroupStmtDnodeId3, pNode->dnodeId3); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkRedistributeVgroupStmtDnodes, pNode->pDnodes); + } + + return code; +} + +static int32_t jsonToRedistributeVgroupStmt(const SJson* pJson, void* pObj) { + SRedistributeVgroupStmt* pNode = (SRedistributeVgroupStmt*)pObj; + + int32_t code = tjsonGetIntValue(pJson, jkRedistributeVgroupStmtVgroupId, &pNode->vgId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkRedistributeVgroupStmtDnodeId1, &pNode->dnodeId1); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkRedistributeVgroupStmtDnodeId2, &pNode->dnodeId2); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkRedistributeVgroupStmtDnodeId3, &pNode->dnodeId3); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkRedistributeVgroupStmtDnodes, &pNode->pDnodes); + } - int32_t code = tjsonGetStringValue(pJson, jkAlterTableStmtDbName, pNode->dbName); + return code; +} + +static const char* jkSplitVgroupStmtVgroupId = "VgroupId"; + +static int32_t splitVgroupStmtToJson(const void* pObj, SJson* pJson) { + const SSplitVgroupStmt* pNode = (const SSplitVgroupStmt*)pObj; + return tjsonAddIntegerToObject(pJson, jkSplitVgroupStmtVgroupId, pNode->vgId); +} + +static int32_t jsonToSplitVgroupStmt(const SJson* pJson, void* pObj) { + SSplitVgroupStmt* pNode = (SSplitVgroupStmt*)pObj; + return tjsonGetIntValue(pJson, jkSplitVgroupStmtVgroupId, &pNode->vgId); +} + +static const char* jkGrantStmtUserName = "UserName"; +static const char* jkGrantStmtObjName = "ObjName"; +static const char* jkGrantStmtPrivileges = "Privileges"; + +static int32_t grantStmtToJson(const void* pObj, SJson* pJson) { + const SGrantStmt* pNode = (const SGrantStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkGrantStmtUserName, pNode->userName); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetStringValue(pJson, jkAlterTableStmtTableName, pNode->tableName); + code = tjsonAddStringToObject(pJson, jkGrantStmtObjName, pNode->objName); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetTinyIntValue(pJson, jkAlterTableStmtAlterType, &pNode->alterType); + code = tjsonAddIntegerToObject(pJson, jkGrantStmtPrivileges, pNode->privileges); } + + return code; +} + +static int32_t jsonToGrantStmt(const SJson* pJson, void* pObj) { + SGrantStmt* pNode = (SGrantStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkGrantStmtUserName, pNode->userName); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetStringValue(pJson, jkAlterTableStmtColName, pNode->colName); + code = tjsonGetStringValue(pJson, jkGrantStmtObjName, pNode->objName); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetStringValue(pJson, jkAlterTableStmtNewColName, pNode->newColName); + code = tjsonGetBigIntValue(pJson, jkGrantStmtPrivileges, &pNode->privileges); } + + return code; +} + +static int32_t revokeStmtToJson(const void* pObj, SJson* pJson) { return grantStmtToJson(pObj, pJson); } + +static int32_t jsonToRevokeStmt(const SJson* pJson, void* pObj) { return jsonToGrantStmt(pJson, pObj); } + +static const char* jkShowStmtDbName = "DbName"; +static const char* jkShowStmtTbName = "TbName"; +static const char* jkShowStmtTableCondType = "TableCondType"; + +static int32_t showStmtToJson(const void* pObj, SJson* pJson) { + const SShowStmt* pNode = (const SShowStmt*)pObj; + + int32_t code = tjsonAddObject(pJson, jkShowStmtDbName, nodeToJson, pNode->pDbName); if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkAlterTableStmtOptions, (SNode**)&pNode->pOptions); + code = tjsonAddObject(pJson, jkShowStmtTbName, nodeToJson, pNode->pTbName); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonToObject(pJson, jkAlterTableStmtNewDataType, jsonToDataType, &pNode->dataType); + code = tjsonAddIntegerToObject(pJson, jkShowStmtTableCondType, pNode->tableCondType); } + + return code; +} + +static int32_t jsonToShowStmt(const SJson* pJson, void* pObj) { + SShowStmt* pNode = (SShowStmt*)pObj; + + int32_t code = jsonToNodeObject(pJson, jkShowStmtDbName, &pNode->pDbName); if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkAlterTableStmtOptions, (SNode**)&pNode->pVal); + code = jsonToNodeObject(pJson, jkShowStmtTbName, &pNode->pTbName); + } + if (TSDB_CODE_SUCCESS == code) { + tjsonGetNumberValue(pJson, jkShowStmtTableCondType, pNode->tableCondType, code); } return code; } -static const char* jkAlterDnodeStmtDnodeId = "DnodeId"; -static const char* jkAlterDnodeStmtConfig = "Config"; -static const char* jkAlterDnodeStmtValue = "Value"; +static int32_t showDnodesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } -static int32_t alterDnodeStmtToJson(const void* pObj, SJson* pJson) { - const SAlterDnodeStmt* pNode = (const SAlterDnodeStmt*)pObj; +static int32_t jsonToShowDnodesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } - int32_t code = tjsonAddIntegerToObject(pJson, jkAlterDnodeStmtDnodeId, pNode->dnodeId); +static int32_t showMnodesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowMnodesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showQnodesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowQnodesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showClusterStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowClusterStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showDatabasesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowDatabasesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showFunctionsStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowFunctionsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showIndexesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowIndexesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showStablesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowStablesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showStreamsStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowStreamsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showTablesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowTablesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showTagsStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowTagsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showUsersStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowUsersStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showVgroupsStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowVgroupsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showConsumersStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowConsumersStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showVariablesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowVariablesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static const char* jkShowDnodeVariablesStmtDnodeId = "DnodeId"; +static const char* jkShowDnodeVariablesStmtLikePattern = "LikePattern"; + +static int32_t showDnodeVariablesStmtToJson(const void* pObj, SJson* pJson) { + const SShowDnodeVariablesStmt* pNode = (const SShowDnodeVariablesStmt*)pObj; + + int32_t code = tjsonAddObject(pJson, jkShowDnodeVariablesStmtDnodeId, nodeToJson, pNode->pDnodeId); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddStringToObject(pJson, jkAlterDnodeStmtConfig, pNode->config); + code = tjsonAddObject(pJson, jkShowDnodeVariablesStmtLikePattern, nodeToJson, pNode->pLikePattern); } + + return code; +} + +static int32_t jsonToShowDnodeVariablesStmt(const SJson* pJson, void* pObj) { + SShowDnodeVariablesStmt* pNode = (SShowDnodeVariablesStmt*)pObj; + + int32_t code = jsonToNodeObject(pJson, jkShowDnodeVariablesStmtDnodeId, &pNode->pDnodeId); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddStringToObject(pJson, jkAlterDnodeStmtValue, pNode->value); + code = jsonToNodeObject(pJson, jkShowDnodeVariablesStmtLikePattern, &pNode->pLikePattern); } return code; } -static int32_t jsonToAlterDnodeStmt(const SJson* pJson, void* pObj) { - SAlterDnodeStmt* pNode = (SAlterDnodeStmt*)pObj; +static int32_t showTransactionsStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } - int32_t code = tjsonGetIntValue(pJson, jkAlterDnodeStmtDnodeId, &pNode->dnodeId); +static int32_t jsonToShowTransactionsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showSubscriptionsStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowSubscriptionsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static const char* jkShowVnodesStmtDnodeId = "DnodeId"; +static const char* jkShowVnodesStmtDnodeEndpoint = "DnodeEndpoint"; + +static int32_t showVnodesStmtToJson(const void* pObj, SJson* pJson) { + const SShowVnodesStmt* pNode = (const SShowVnodesStmt*)pObj; + + int32_t code = tjsonAddObject(pJson, jkShowVnodesStmtDnodeId, nodeToJson, pNode->pDnodeId); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetStringValue(pJson, jkAlterDnodeStmtConfig, pNode->config); + code = tjsonAddObject(pJson, jkShowVnodesStmtDnodeEndpoint, nodeToJson, pNode->pDnodeEndpoint); } + + return code; +} + +static int32_t jsonToShowVnodesStmt(const SJson* pJson, void* pObj) { + SShowVnodesStmt* pNode = (SShowVnodesStmt*)pObj; + + int32_t code = jsonToNodeObject(pJson, jkShowVnodesStmtDnodeId, &pNode->pDnodeId); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetStringValue(pJson, jkAlterDnodeStmtValue, pNode->value); + code = jsonToNodeObject(pJson, jkShowVnodesStmtDnodeEndpoint, &pNode->pDnodeEndpoint); } return code; } -static const char* jkCreateTopicStmtTopicName = "TopicName"; -static const char* jkCreateTopicStmtSubscribeDbName = "SubscribeDbName"; -static const char* jkCreateTopicStmtIgnoreExists = "IgnoreExists"; -static const char* jkCreateTopicStmtQuery = "Query"; +static int32_t showUserPrivilegesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } -static int32_t createTopicStmtToJson(const void* pObj, SJson* pJson) { - const SCreateTopicStmt* pNode = (const SCreateTopicStmt*)pObj; +static int32_t jsonToShowUserPrivilegesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } - int32_t code = tjsonAddStringToObject(pJson, jkCreateTopicStmtTopicName, pNode->topicName); +static const char* jkShowCreateDatabaseStmtDbName = "DbName"; + +static int32_t showCreateDatabaseStmtToJson(const void* pObj, SJson* pJson) { + const SShowCreateDatabaseStmt* pNode = (const SShowCreateDatabaseStmt*)pObj; + return tjsonAddStringToObject(pJson, jkShowCreateDatabaseStmtDbName, pNode->dbName); +} + +static int32_t jsonToShowCreateDatabaseStmt(const SJson* pJson, void* pObj) { + SShowCreateDatabaseStmt* pNode = (SShowCreateDatabaseStmt*)pObj; + return tjsonGetStringValue(pJson, jkShowCreateDatabaseStmtDbName, pNode->dbName); +} + +static const char* jkShowCreateTableStmtDbName = "DbName"; +static const char* jkShowCreateTableStmtTableName = "TableName"; + +static int32_t showCreateTableStmtToJson(const void* pObj, SJson* pJson) { + const SShowCreateTableStmt* pNode = (const SShowCreateTableStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkShowCreateTableStmtDbName, pNode->dbName); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddStringToObject(pJson, jkCreateTopicStmtSubscribeDbName, pNode->subDbName); + code = tjsonAddStringToObject(pJson, jkShowCreateTableStmtTableName, pNode->tableName); } + + return code; +} + +static int32_t jsonToShowCreateTableStmt(const SJson* pJson, void* pObj) { + SShowCreateTableStmt* pNode = (SShowCreateTableStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkShowCreateTableStmtDbName, pNode->dbName); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddBoolToObject(pJson, jkCreateTopicStmtIgnoreExists, pNode->ignoreExists); + code = tjsonGetStringValue(pJson, jkShowCreateTableStmtTableName, pNode->tableName); + } + + return code; +} + +static int32_t showCreateStableStmtToJson(const void* pObj, SJson* pJson) { + return showCreateTableStmtToJson(pObj, pJson); +} + +static int32_t jsonToShowCreateStableStmt(const SJson* pJson, void* pObj) { + return jsonToShowCreateTableStmt(pJson, pObj); +} + +static const char* jkShowTableDistributedStmtDbName = "DbName"; +static const char* jkShowTableDistributedStmtTableName = "TableName"; + +static int32_t showTableDistributedStmtToJson(const void* pObj, SJson* pJson) { + const SShowTableDistributedStmt* pNode = (const SShowTableDistributedStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkShowTableDistributedStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkShowTableDistributedStmtTableName, pNode->tableName); } + + return code; +} + +static int32_t jsonToShowTableDistributedStmt(const SJson* pJson, void* pObj) { + SShowTableDistributedStmt* pNode = (SShowTableDistributedStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkShowTableDistributedStmtDbName, pNode->dbName); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkCreateTopicStmtQuery, nodeToJson, pNode->pQuery); + code = tjsonGetStringValue(pJson, jkShowTableDistributedStmtTableName, pNode->tableName); } return code; } -static int32_t jsonToCreateTopicStmt(const SJson* pJson, void* pObj) { - SCreateTopicStmt* pNode = (SCreateTopicStmt*)pObj; +static int32_t showLocalVariablesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } - int32_t code = tjsonGetStringValue(pJson, jkCreateTopicStmtTopicName, pNode->topicName); +static int32_t jsonToShowLocalVariablesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static const char* jkShowTableTagsStmtDbName = "DbName"; +static const char* jkShowTableTagsStmtTbName = "TbName"; +static const char* jkShowTableTagsStmtTags = "Tags"; + +static int32_t showTableTagsStmtToJson(const void* pObj, SJson* pJson) { + const SShowTableTagsStmt* pNode = (const SShowTableTagsStmt*)pObj; + + int32_t code = tjsonAddObject(pJson, jkShowTableTagsStmtDbName, nodeToJson, pNode->pDbName); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetStringValue(pJson, jkCreateTopicStmtSubscribeDbName, pNode->subDbName); + code = tjsonAddObject(pJson, jkShowTableTagsStmtTbName, nodeToJson, pNode->pTbName); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetBoolValue(pJson, jkCreateTopicStmtIgnoreExists, &pNode->ignoreExists); + code = nodeListToJson(pJson, jkShowTableTagsStmtTags, pNode->pTags); } + + return code; +} + +static int32_t jsonToShowTableTagsStmt(const SJson* pJson, void* pObj) { + SShowTableTagsStmt* pNode = (SShowTableTagsStmt*)pObj; + + int32_t code = jsonToNodeObject(pJson, jkShowTableTagsStmtDbName, &pNode->pDbName); if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkCreateTopicStmtQuery, &pNode->pQuery); + code = jsonToNodeObject(pJson, jkShowTableTagsStmtTbName, &pNode->pTbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkShowTableTagsStmtTags, &pNode->pTags); } return code; @@ -4568,6 +6224,45 @@ static int32_t jsonToDeleteStmt(const SJson* pJson, void* pObj) { return code; } +static const char* jkInsertStmtTable = "Table"; +static const char* jkInsertStmtCols = "Cols"; +static const char* jkInsertStmtQuery = "Query"; +static const char* jkInsertStmtPrecision = "Precision"; + +static int32_t insertStmtToJson(const void* pObj, SJson* pJson) { + const SInsertStmt* pNode = (const SInsertStmt*)pObj; + + int32_t code = tjsonAddObject(pJson, jkInsertStmtTable, nodeToJson, pNode->pTable); + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkInsertStmtCols, pNode->pCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkInsertStmtQuery, nodeToJson, pNode->pQuery); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkInsertStmtPrecision, pNode->precision); + } + + return code; +} + +static int32_t jsonToInsertStmt(const SJson* pJson, void* pObj) { + SInsertStmt* pNode = (SInsertStmt*)pObj; + + int32_t code = jsonToNodeObject(pJson, jkInsertStmtTable, &pNode->pTable); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkInsertStmtCols, &pNode->pCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkInsertStmtQuery, &pNode->pQuery); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUTinyIntValue(pJson, jkInsertStmtPrecision, &pNode->precision); + } + + return code; +} + static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { switch (nodeType(pObj)) { case QUERY_NODE_COLUMN: @@ -4585,7 +6280,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_TEMP_TABLE: return tempTableNodeToJson(pObj, pJson); case QUERY_NODE_JOIN_TABLE: - break; + return joinTableNodeToJson(pObj, pJson); case QUERY_NODE_GROUPING_SET: return groupingSetNodeToJson(pObj, pJson); case QUERY_NODE_ORDER_BY_EXPR: @@ -4611,42 +6306,169 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_SLOT_DESC: return slotDescNodeToJson(pObj, pJson); case QUERY_NODE_COLUMN_DEF: - break; + return columnDefNodeToJson(pObj, pJson); case QUERY_NODE_DOWNSTREAM_SOURCE: return downstreamSourceNodeToJson(pObj, pJson); case QUERY_NODE_DATABASE_OPTIONS: return databaseOptionsToJson(pObj, pJson); + case QUERY_NODE_TABLE_OPTIONS: + return tableOptionsToJson(pObj, pJson); + case QUERY_NODE_INDEX_OPTIONS: + return indexOptionsToJson(pObj, pJson); + case QUERY_NODE_EXPLAIN_OPTIONS: + return explainOptionsToJson(pObj, pJson); + case QUERY_NODE_STREAM_OPTIONS: + return streamOptionsToJson(pObj, pJson); case QUERY_NODE_LEFT_VALUE: return TSDB_CODE_SUCCESS; // SLeftValueNode has no fields to serialize. case QUERY_NODE_WHEN_THEN: return whenThenNodeToJson(pObj, pJson); case QUERY_NODE_CASE_WHEN: return caseWhenNodeToJson(pObj, pJson); + case QUERY_NODE_EVENT_WINDOW: + return eventWindowNodeToJson(pObj, pJson); case QUERY_NODE_SET_OPERATOR: return setOperatorToJson(pObj, pJson); case QUERY_NODE_SELECT_STMT: return selectStmtToJson(pObj, pJson); - case QUERY_NODE_VNODE_MODIF_STMT: + case QUERY_NODE_VNODE_MODIFY_STMT: + return vnodeModifyStmtToJson(pObj, pJson); case QUERY_NODE_CREATE_DATABASE_STMT: - break; + return createDatabaseStmtToJson(pObj, pJson); case QUERY_NODE_ALTER_DATABASE_STMT: return alterDatabaseStmtToJson(pObj, pJson); + case QUERY_NODE_TRIM_DATABASE_STMT: + return trimDatabaseStmtToJson(pObj, pJson); case QUERY_NODE_CREATE_TABLE_STMT: - break; + return createTableStmtToJson(pObj, pJson); + case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: + return createSubTableClauseToJson(pObj, pJson); + case QUERY_NODE_CREATE_MULTI_TABLES_STMT: + return createMultiTablesStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_TABLE_CLAUSE: + return dropTableClauseToJson(pObj, pJson); + case QUERY_NODE_DROP_TABLE_STMT: + return dropTableStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_SUPER_TABLE_STMT: + return dropStableStmtToJson(pObj, pJson); case QUERY_NODE_ALTER_TABLE_STMT: return alterTableStmtToJson(pObj, pJson); + case QUERY_NODE_ALTER_SUPER_TABLE_STMT: + return alterStableStmtToJson(pObj, pJson); + case QUERY_NODE_CREATE_USER_STMT: + return createUserStmtToJson(pObj, pJson); + case QUERY_NODE_ALTER_USER_STMT: + return alterUserStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_USER_STMT: + return dropUserStmtToJson(pObj, pJson); case QUERY_NODE_USE_DATABASE_STMT: - break; + return useDatabaseStmtToJson(pObj, pJson); + case QUERY_NODE_CREATE_DNODE_STMT: + return createDnodeStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_DNODE_STMT: + return dropDnodeStmtToJson(pObj, pJson); case QUERY_NODE_ALTER_DNODE_STMT: return alterDnodeStmtToJson(pObj, pJson); + case QUERY_NODE_CREATE_INDEX_STMT: + return createIndexStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_INDEX_STMT: + return dropIndexStmtToJson(pObj, pJson); + case QUERY_NODE_CREATE_QNODE_STMT: + return createQnodeStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_QNODE_STMT: + return dropQnodeStmtToJson(pObj, pJson); + case QUERY_NODE_CREATE_SNODE_STMT: + return createSnodeStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_SNODE_STMT: + return dropSnodeStmtToJson(pObj, pJson); + case QUERY_NODE_CREATE_MNODE_STMT: + return createMnodeStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_MNODE_STMT: + return dropMnodeStmtToJson(pObj, pJson); + case QUERY_NODE_CREATE_TOPIC_STMT: + return createTopicStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_TOPIC_STMT: + return dropTopicStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_CGROUP_STMT: + return dropConsumerGroupStmtToJson(pObj, pJson); + case QUERY_NODE_ALTER_LOCAL_STMT: + return alterLocalStmtToJson(pObj, pJson); + case QUERY_NODE_EXPLAIN_STMT: + return explainStmtToJson(pObj, pJson); + case QUERY_NODE_DESCRIBE_STMT: + return describeStmtToJson(pObj, pJson); + case QUERY_NODE_CREATE_STREAM_STMT: + return createStreamStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_STREAM_STMT: + return dropStreamStmtToJson(pObj, pJson); + case QUERY_NODE_BALANCE_VGROUP_STMT: + return TSDB_CODE_SUCCESS; // SBalanceVgroupStmt has no fields to serialize. + case QUERY_NODE_MERGE_VGROUP_STMT: + return mergeVgroupStmtToJson(pObj, pJson); + case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT: + return redistributeVgroupStmtToJson(pObj, pJson); + case QUERY_NODE_SPLIT_VGROUP_STMT: + return splitVgroupStmtToJson(pObj, pJson); + case QUERY_NODE_GRANT_STMT: + return grantStmtToJson(pObj, pJson); + case QUERY_NODE_REVOKE_STMT: + return revokeStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_DNODES_STMT: + return showDnodesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_MNODES_STMT: + return showMnodesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_QNODES_STMT: + return showQnodesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_CLUSTER_STMT: + return showClusterStmtToJson(pObj, pJson); case QUERY_NODE_SHOW_DATABASES_STMT: + return showDatabasesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_FUNCTIONS_STMT: + return showFunctionsStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_INDEXES_STMT: + return showIndexesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_STABLES_STMT: + return showStablesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_STREAMS_STMT: + return showStreamsStmtToJson(pObj, pJson); case QUERY_NODE_SHOW_TABLES_STMT: + return showTablesStmtToJson(pObj, pJson); case QUERY_NODE_SHOW_TAGS_STMT: - break; - case QUERY_NODE_CREATE_TOPIC_STMT: - return createTopicStmtToJson(pObj, pJson); + return showTagsStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_USERS_STMT: + return showUsersStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_VGROUPS_STMT: + return showVgroupsStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_CONSUMERS_STMT: + return showConsumersStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_VARIABLES_STMT: + return showVariablesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: + return showDnodeVariablesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_TRANSACTIONS_STMT: + return showTransactionsStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT: + return showSubscriptionsStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_VNODES_STMT: + return showVnodesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT: + return showUserPrivilegesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: + return showCreateDatabaseStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_CREATE_TABLE_STMT: + return showCreateTableStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_CREATE_STABLE_STMT: + return showCreateStableStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: + return showTableDistributedStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: + return showLocalVariablesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_TABLE_TAGS_STMT: + return showTableTagsStmtToJson(pObj, pJson); case QUERY_NODE_DELETE_STMT: return deleteStmtToJson(pObj, pJson); + case QUERY_NODE_INSERT_STMT: + return insertStmtToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_SCAN: return logicScanNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_JOIN: @@ -4719,6 +6541,9 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: return physiStateWindowNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: + return physiEventWindowNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_PARTITION: return physiPartitionNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: @@ -4762,6 +6587,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToRealTableNode(pJson, pObj); case QUERY_NODE_TEMP_TABLE: return jsonToTempTableNode(pJson, pObj); + case QUERY_NODE_JOIN_TABLE: + return jsonToJoinTableNode(pJson, pObj); case QUERY_NODE_GROUPING_SET: return jsonToGroupingSetNode(pJson, pObj); case QUERY_NODE_ORDER_BY_EXPR: @@ -4784,32 +6611,174 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToDataBlockDescNode(pJson, pObj); case QUERY_NODE_SLOT_DESC: return jsonToSlotDescNode(pJson, pObj); + case QUERY_NODE_COLUMN_DEF: + return jsonToColumnDefNode(pJson, pObj); case QUERY_NODE_DOWNSTREAM_SOURCE: return jsonToDownstreamSourceNode(pJson, pObj); case QUERY_NODE_DATABASE_OPTIONS: return jsonToDatabaseOptions(pJson, pObj); + case QUERY_NODE_TABLE_OPTIONS: + return jsonToTableOptions(pJson, pObj); + case QUERY_NODE_INDEX_OPTIONS: + return jsonToIndexOptions(pJson, pObj); + case QUERY_NODE_EXPLAIN_OPTIONS: + return jsonToExplainOptions(pJson, pObj); + case QUERY_NODE_STREAM_OPTIONS: + return jsonToStreamOptions(pJson, pObj); case QUERY_NODE_LEFT_VALUE: return TSDB_CODE_SUCCESS; // SLeftValueNode has no fields to deserialize. case QUERY_NODE_WHEN_THEN: return jsonToWhenThenNode(pJson, pObj); case QUERY_NODE_CASE_WHEN: return jsonToCaseWhenNode(pJson, pObj); + case QUERY_NODE_EVENT_WINDOW: + return jsonToEventWindowNode(pJson, pObj); case QUERY_NODE_SET_OPERATOR: return jsonToSetOperator(pJson, pObj); case QUERY_NODE_SELECT_STMT: return jsonToSelectStmt(pJson, pObj); + case QUERY_NODE_VNODE_MODIFY_STMT: + return jsonToVnodeModifyStmt(pJson, pObj); + case QUERY_NODE_CREATE_DATABASE_STMT: + return jsonToCreateDatabaseStmt(pJson, pObj); case QUERY_NODE_ALTER_DATABASE_STMT: return jsonToAlterDatabaseStmt(pJson, pObj); + case QUERY_NODE_TRIM_DATABASE_STMT: + return jsonToTrimDatabaseStmt(pJson, pObj); + case QUERY_NODE_CREATE_TABLE_STMT: + return jsonToCreateTableStmt(pJson, pObj); + case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: + return jsonToCreateSubTableClause(pJson, pObj); + case QUERY_NODE_CREATE_MULTI_TABLES_STMT: + return jsonToCreateMultiTablesStmt(pJson, pObj); + case QUERY_NODE_DROP_TABLE_CLAUSE: + return jsonToDropTableClause(pJson, pObj); + case QUERY_NODE_DROP_TABLE_STMT: + return jsonToDropTableStmt(pJson, pObj); + case QUERY_NODE_DROP_SUPER_TABLE_STMT: + return jsonToDropStableStmt(pJson, pObj); case QUERY_NODE_ALTER_TABLE_STMT: return jsonToAlterTableStmt(pJson, pObj); + case QUERY_NODE_ALTER_SUPER_TABLE_STMT: + return jsonToAlterStableStmt(pJson, pObj); + case QUERY_NODE_CREATE_USER_STMT: + return jsonToCreateUserStmt(pJson, pObj); + case QUERY_NODE_ALTER_USER_STMT: + return jsonToAlterUserStmt(pJson, pObj); + case QUERY_NODE_DROP_USER_STMT: + return jsonToDropUserStmt(pJson, pObj); + case QUERY_NODE_USE_DATABASE_STMT: + return jsonToUseDatabaseStmt(pJson, pObj); + case QUERY_NODE_CREATE_DNODE_STMT: + return jsonToCreateDnodeStmt(pJson, pObj); + case QUERY_NODE_DROP_DNODE_STMT: + return jsonToDropDnodeStmt(pJson, pObj); case QUERY_NODE_ALTER_DNODE_STMT: return jsonToAlterDnodeStmt(pJson, pObj); + case QUERY_NODE_CREATE_INDEX_STMT: + return jsonToCreateIndexStmt(pJson, pObj); + case QUERY_NODE_DROP_INDEX_STMT: + return jsonToDropIndexStmt(pJson, pObj); + case QUERY_NODE_CREATE_QNODE_STMT: + return jsonToCreateQnodeStmt(pJson, pObj); + case QUERY_NODE_DROP_QNODE_STMT: + return jsonToDropQnodeStmt(pJson, pObj); + case QUERY_NODE_CREATE_SNODE_STMT: + return jsonToCreateSnodeStmt(pJson, pObj); + case QUERY_NODE_DROP_SNODE_STMT: + return jsonToDropSnodeStmt(pJson, pObj); + case QUERY_NODE_CREATE_MNODE_STMT: + return jsonToCreateMnodeStmt(pJson, pObj); + case QUERY_NODE_DROP_MNODE_STMT: + return jsonToDropMnodeStmt(pJson, pObj); case QUERY_NODE_CREATE_TOPIC_STMT: return jsonToCreateTopicStmt(pJson, pObj); + case QUERY_NODE_DROP_TOPIC_STMT: + return jsonToDropTopicStmt(pJson, pObj); + case QUERY_NODE_DROP_CGROUP_STMT: + return jsonToDropConsumerGroupStmt(pJson, pObj); + case QUERY_NODE_ALTER_LOCAL_STMT: + return jsonToAlterLocalStmt(pJson, pObj); + case QUERY_NODE_EXPLAIN_STMT: + return jsonToExplainStmt(pJson, pObj); + case QUERY_NODE_DESCRIBE_STMT: + return jsonToDescribeStmt(pJson, pObj); + case QUERY_NODE_CREATE_STREAM_STMT: + return jsonToCreateStreamStmt(pJson, pObj); + case QUERY_NODE_DROP_STREAM_STMT: + return jsonToDropStreamStmt(pJson, pObj); + case QUERY_NODE_BALANCE_VGROUP_STMT: + return TSDB_CODE_SUCCESS; // SBalanceVgroupStmt has no fields to deserialize. + case QUERY_NODE_MERGE_VGROUP_STMT: + return jsonToMergeVgroupStmt(pJson, pObj); + case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT: + return jsonToRedistributeVgroupStmt(pJson, pObj); + case QUERY_NODE_SPLIT_VGROUP_STMT: + return jsonToSplitVgroupStmt(pJson, pObj); + case QUERY_NODE_GRANT_STMT: + return jsonToGrantStmt(pJson, pObj); + case QUERY_NODE_REVOKE_STMT: + return jsonToRevokeStmt(pJson, pObj); + case QUERY_NODE_SHOW_DNODES_STMT: + return jsonToShowDnodesStmt(pJson, pObj); + case QUERY_NODE_SHOW_MNODES_STMT: + return jsonToShowMnodesStmt(pJson, pObj); + case QUERY_NODE_SHOW_QNODES_STMT: + return jsonToShowQnodesStmt(pJson, pObj); + case QUERY_NODE_SHOW_CLUSTER_STMT: + return jsonToShowClusterStmt(pJson, pObj); + case QUERY_NODE_SHOW_DATABASES_STMT: + return jsonToShowDatabasesStmt(pJson, pObj); + case QUERY_NODE_SHOW_FUNCTIONS_STMT: + return jsonToShowFunctionsStmt(pJson, pObj); + case QUERY_NODE_SHOW_INDEXES_STMT: + return jsonToShowIndexesStmt(pJson, pObj); + case QUERY_NODE_SHOW_STABLES_STMT: + return jsonToShowStablesStmt(pJson, pObj); + case QUERY_NODE_SHOW_STREAMS_STMT: + return jsonToShowStreamsStmt(pJson, pObj); + case QUERY_NODE_SHOW_TABLES_STMT: + return jsonToShowTablesStmt(pJson, pObj); + case QUERY_NODE_SHOW_TAGS_STMT: + return jsonToShowTagsStmt(pJson, pObj); + case QUERY_NODE_SHOW_USERS_STMT: + return jsonToShowUsersStmt(pJson, pObj); + case QUERY_NODE_SHOW_VGROUPS_STMT: + return jsonToShowVgroupsStmt(pJson, pObj); + case QUERY_NODE_SHOW_CONSUMERS_STMT: + return jsonToShowConsumersStmt(pJson, pObj); + case QUERY_NODE_SHOW_VARIABLES_STMT: + return jsonToShowVariablesStmt(pJson, pObj); + case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: + return jsonToShowDnodeVariablesStmt(pJson, pObj); + case QUERY_NODE_SHOW_TRANSACTIONS_STMT: + return jsonToShowTransactionsStmt(pJson, pObj); + case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT: + return jsonToShowSubscriptionsStmt(pJson, pObj); + case QUERY_NODE_SHOW_VNODES_STMT: + return jsonToShowVnodesStmt(pJson, pObj); + case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT: + return jsonToShowUserPrivilegesStmt(pJson, pObj); + case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: + return jsonToShowCreateDatabaseStmt(pJson, pObj); + case QUERY_NODE_SHOW_CREATE_TABLE_STMT: + return jsonToShowCreateTableStmt(pJson, pObj); + case QUERY_NODE_SHOW_CREATE_STABLE_STMT: + return jsonToShowCreateStableStmt(pJson, pObj); + case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: + return jsonToShowTableDistributedStmt(pJson, pObj); + case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: + return jsonToShowLocalVariablesStmt(pJson, pObj); + case QUERY_NODE_SHOW_TABLE_TAGS_STMT: + return jsonToShowTableTagsStmt(pJson, pObj); case QUERY_NODE_DELETE_STMT: return jsonToDeleteStmt(pJson, pObj); + case QUERY_NODE_INSERT_STMT: + return jsonToInsertStmt(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_SCAN: return jsonToLogicScanNode(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN_JOIN: + return jsonToLogicJoinNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_AGG: return jsonToLogicAggNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_PROJECT: @@ -4878,6 +6847,9 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: return jsonToPhysiStateWindowNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: + return jsonToPhysiEventWindowNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_PARTITION: return jsonToPhysiPartitionNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index dfe2f6c8b9b78651551f6a6c7341c1e80ddb71bd..a4b77fad6928cdb46c52e44266f74c1f3ec6fb46 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2934,6 +2934,46 @@ static int32_t msgToPhysiStateWindowNode(STlvDecoder* pDecoder, void* pObj) { return code; } +enum { PHY_EVENT_CODE_WINDOW = 1, PHY_EVENT_CODE_START_COND, PHY_EVENT_CODE_END_COND }; + +static int32_t physiEventWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SEventWinodwPhysiNode* pNode = (const SEventWinodwPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_EVENT_CODE_WINDOW, physiWindowNodeToMsg, &pNode->window); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_EVENT_CODE_START_COND, nodeToMsg, pNode->pStartCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_EVENT_CODE_END_COND, nodeToMsg, pNode->pEndCond); + } + + return code; +} + +static int32_t msgToPhysiEventWindowNode(STlvDecoder* pDecoder, void* pObj) { + SEventWinodwPhysiNode* pNode = (SEventWinodwPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_EVENT_CODE_WINDOW: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiWindowNode, &pNode->window); + break; + case PHY_EVENT_CODE_START_COND: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pStartCond); + break; + case PHY_EVENT_CODE_END_COND: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pEndCond); + break; + default: + break; + } + } + + return code; +} + enum { PHY_PARTITION_CODE_BASE_NODE = 1, PHY_PARTITION_CODE_EXPR, PHY_PARTITION_CODE_KEYS, PHY_PARTITION_CODE_TARGETS }; static int32_t physiPartitionNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { @@ -3705,6 +3745,10 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: code = physiStateWindowNodeToMsg(pObj, pEncoder); break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: + code = physiEventWindowNodeToMsg(pObj, pEncoder); + break; case QUERY_NODE_PHYSICAL_PLAN_PARTITION: code = physiPartitionNodeToMsg(pObj, pEncoder); break; @@ -3844,6 +3888,10 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: code = msgToPhysiStateWindowNode(pDecoder, pObj); break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: + code = msgToPhysiEventWindowNode(pDecoder, pObj); + break; case QUERY_NODE_PHYSICAL_PLAN_PARTITION: code = msgToPhysiPartitionNode(pDecoder, pObj); break; diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index 106812d55f1cdd08120cb100e338b0b82244e752..ce575ede8a5dd2020c6af39394a72d6fa39cc348 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -165,6 +165,17 @@ static EDealRes dispatchExpr(SNode* pNode, ETraversalOrder order, FNodeWalker wa } break; } + case QUERY_NODE_EVENT_WINDOW: { + SEventWindowNode* pEvent = (SEventWindowNode*)pNode; + res = walkExpr(pEvent->pCol, order, walker, pContext); + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { + res = walkExpr(pEvent->pStartCond, order, walker, pContext); + } + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { + res = walkExpr(pEvent->pEndCond, order, walker, pContext); + } + break; + } default: break; } @@ -329,6 +340,17 @@ static EDealRes rewriteExpr(SNode** pRawNode, ETraversalOrder order, FNodeRewrit } break; } + case QUERY_NODE_EVENT_WINDOW: { + SEventWindowNode* pEvent = (SEventWindowNode*)pNode; + res = rewriteExpr(&pEvent->pCol, order, rewriter, pContext); + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { + res = rewriteExpr(&pEvent->pStartCond, order, rewriter, pContext); + } + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { + res = rewriteExpr(&pEvent->pEndCond, order, rewriter, pContext); + } + break; + } default: break; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index cd5ae7ad6e2b1e917b5b2842172b95b09ec49d20..61bb3778fa2d692816e99f43c5b09713b7673341 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -299,12 +299,14 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SWhenThenNode)); case QUERY_NODE_CASE_WHEN: return makeNode(type, sizeof(SCaseWhenNode)); + case QUERY_NODE_EVENT_WINDOW: + return makeNode(type, sizeof(SEventWindowNode)); case QUERY_NODE_SET_OPERATOR: return makeNode(type, sizeof(SSetOperator)); case QUERY_NODE_SELECT_STMT: return makeNode(type, sizeof(SSelectStmt)); - case QUERY_NODE_VNODE_MODIF_STMT: - return makeNode(type, sizeof(SVnodeModifOpStmt)); + case QUERY_NODE_VNODE_MODIFY_STMT: + return makeNode(type, sizeof(SVnodeModifyOpStmt)); case QUERY_NODE_CREATE_DATABASE_STMT: return makeNode(type, sizeof(SCreateDatabaseStmt)); case QUERY_NODE_DROP_DATABASE_STMT: @@ -319,8 +321,8 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SCreateTableStmt)); case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: return makeNode(type, sizeof(SCreateSubTableClause)); - case QUERY_NODE_CREATE_MULTI_TABLE_STMT: - return makeNode(type, sizeof(SCreateMultiTableStmt)); + case QUERY_NODE_CREATE_MULTI_TABLES_STMT: + return makeNode(type, sizeof(SCreateMultiTablesStmt)); case QUERY_NODE_DROP_TABLE_CLAUSE: return makeNode(type, sizeof(SDropTableClause)); case QUERY_NODE_DROP_TABLE_STMT: @@ -432,6 +434,9 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SShowDnodeVariablesStmt)); case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: return makeNode(type, sizeof(SShowCreateDatabaseStmt)); + case QUERY_NODE_SHOW_DB_ALIVE_STMT: + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: + return makeNode(type, sizeof(SShowAliveStmt)); case QUERY_NODE_SHOW_CREATE_TABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT: return makeNode(type, sizeof(SShowCreateTableStmt)); @@ -535,6 +540,10 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SStateWinodwPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: return makeNode(type, sizeof(SStreamStateWinodwPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + return makeNode(type, sizeof(SEventWinodwPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: + return makeNode(type, sizeof(SStreamEventWinodwPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_PARTITION: return makeNode(type, sizeof(SPartitionPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: @@ -765,16 +774,23 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_COLUMN_REF: // no pointer field break; case QUERY_NODE_WHEN_THEN: { - SWhenThenNode* pStmt = (SWhenThenNode*)pNode; - nodesDestroyNode(pStmt->pWhen); - nodesDestroyNode(pStmt->pThen); + SWhenThenNode* pWhenThen = (SWhenThenNode*)pNode; + nodesDestroyNode(pWhenThen->pWhen); + nodesDestroyNode(pWhenThen->pThen); break; } case QUERY_NODE_CASE_WHEN: { - SCaseWhenNode* pStmt = (SCaseWhenNode*)pNode; - nodesDestroyNode(pStmt->pCase); - nodesDestroyNode(pStmt->pElse); - nodesDestroyList(pStmt->pWhenThenList); + SCaseWhenNode* pCaseWhen = (SCaseWhenNode*)pNode; + nodesDestroyNode(pCaseWhen->pCase); + nodesDestroyNode(pCaseWhen->pElse); + nodesDestroyList(pCaseWhen->pWhenThenList); + break; + } + case QUERY_NODE_EVENT_WINDOW: { + SEventWindowNode* pEvent = (SEventWindowNode*)pNode; + nodesDestroyNode(pEvent->pCol); + nodesDestroyNode(pEvent->pStartCond); + nodesDestroyNode(pEvent->pEndCond); break; } case QUERY_NODE_SET_OPERATOR: { @@ -805,8 +821,8 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode((SNode*)pStmt->pSlimit); break; } - case QUERY_NODE_VNODE_MODIF_STMT: { - SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pNode; + case QUERY_NODE_VNODE_MODIFY_STMT: { + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pNode; destroyVgDataBlockArray(pStmt->pDataBlocks); taosMemoryFreeClear(pStmt->pTableMeta); taosHashCleanup(pStmt->pVgroupsHashObj); @@ -819,7 +835,8 @@ void nodesDestroyNode(SNode* pNode) { if (pStmt->freeArrayFunc) { pStmt->freeArrayFunc(pStmt->pVgDataBlocks); } - tdDestroySVCreateTbReq(&pStmt->createTblReq); + tdDestroySVCreateTbReq(pStmt->pCreateTblReq); + taosMemoryFreeClear(pStmt->pCreateTblReq); taosCloseFile(&pStmt->fp); break; } @@ -848,8 +865,8 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode((SNode*)pStmt->pOptions); break; } - case QUERY_NODE_CREATE_MULTI_TABLE_STMT: - nodesDestroyList(((SCreateMultiTableStmt*)pNode)->pSubTables); + case QUERY_NODE_CREATE_MULTI_TABLES_STMT: + nodesDestroyList(((SCreateMultiTablesStmt*)pNode)->pSubTables); break; case QUERY_NODE_DROP_TABLE_CLAUSE: // no pointer field break; @@ -946,6 +963,8 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_SHOW_USERS_STMT: case QUERY_NODE_SHOW_LICENCES_STMT: case QUERY_NODE_SHOW_VGROUPS_STMT: + case QUERY_NODE_SHOW_DB_ALIVE_STMT: + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: case QUERY_NODE_SHOW_TOPICS_STMT: case QUERY_NODE_SHOW_CONSUMERS_STMT: case QUERY_NODE_SHOW_CONNECTIONS_STMT: @@ -1232,6 +1251,14 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pPhyNode->pStateKey); break; } + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: { + SEventWinodwPhysiNode* pPhyNode = (SEventWinodwPhysiNode*)pNode; + destroyWinodwPhysiNode((SWinodwPhysiNode*)pPhyNode); + nodesDestroyNode(pPhyNode->pStartCond); + nodesDestroyNode(pPhyNode->pEndCond); + break; + } case QUERY_NODE_PHYSICAL_PLAN_PARTITION: { destroyPartitionPhysiNode((SPartitionPhysiNode*)pNode); break; diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 5fecc95f1bbe7f21091803b2fe6d705eda3efd4b..d202e60a4575c9ad3efcdbcccc938a4c8fe1ed31 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -116,6 +116,7 @@ SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const STok SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder); SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap); SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr); +SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond); SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill); SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues); @@ -175,6 +176,7 @@ SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type); SNode* createShowStmtWithCond(SAstCreateContext* pCxt, ENodeType type, SNode* pDbName, SNode* pTbName, EOperatorType tableCondType); SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName); +SNode* createShowAliveStmt(SAstCreateContext* pCxt, SNode* pDbName, ENodeType type); SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable); SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable); SNode* createShowDnodeVariablesStmt(SAstCreateContext* pCxt, SNode* pDnodeId, SNode* pLikePattern); @@ -214,7 +216,7 @@ SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool SNode* createDropFunctionStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pFuncName); SNode* createStreamOptions(SAstCreateContext* pCxt); SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pStreamName, SNode* pRealTable, - SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery); + SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery, SNodeList* pCols); SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pStreamName); SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId); SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId); diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index 5cc72f86923762454c8cae66ef9bbc7828b9ef05..7b816359f9c77b5fe370177fae2aedc67d6cc9a3 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -20,8 +20,6 @@ struct SToken; -#define IS_DATA_COL_ORDERED(spd) ((spd->orderStatus) == (int8_t)ORDER_STATUS_ORDERED) - #define NEXT_TOKEN(pSql, sToken) \ do { \ int32_t index = 0; \ @@ -37,108 +35,27 @@ struct SToken; } \ } while (0) -typedef enum EOrderStatus { - ORDER_STATUS_UNKNOWN = 0, - ORDER_STATUS_ORDERED = 1, - ORDER_STATUS_DISORDERED = 2, -} EOrderStatus; - -typedef enum EValStat { - VAL_STAT_HAS = 0x0, // 0 means has val - VAL_STAT_NONE = 0x01, // 1 means no val -} EValStat; - -typedef struct SBoundColumn { - int32_t offset; // all column offset value - int32_t toffset; // first part offset for SDataRow TODO: get offset from STSchema on future - uint8_t valStat; // EValStat. denote if current column bound or not(0 means has val, 1 means no val) -} SBoundColumn; - -typedef struct { - col_id_t schemaColIdx; - col_id_t boundIdx; - col_id_t finalIdx; -} SBoundIdxInfo; - -typedef struct SParsedDataColInfo { - col_id_t numOfCols; - col_id_t numOfBound; - uint16_t flen; // TODO: get from STSchema - uint16_t allNullLen; // TODO: get from STSchema(base on SDataRow) - uint16_t extendedVarLen; - uint16_t boundNullLen; // bound column len with all NULL value(without VarDataOffsetT/SColIdx part) - col_id_t *boundColumns; // bound column idx according to schema - SBoundColumn *cols; - SBoundIdxInfo *colIdxInfo; - int8_t orderStatus; // bound columns -} SParsedDataColInfo; - -typedef struct SInsertParseBaseContext { - SParseContext *pComCxt; - char *pSql; - SMsgBuf msg; -} SInsertParseBaseContext; - -typedef struct SInsertParseSyntaxCxt { - SParseContext *pComCxt; - char *pSql; - SMsgBuf msg; - SParseMetaCache *pMetaCache; -} SInsertParseSyntaxCxt; - -typedef struct SMemParam { - SRowBuilder *rb; - SSchema *schema; - int32_t toffset; - col_id_t colIdx; -} SMemParam; - -typedef struct { - uint8_t rowType; // default is 0, that is SDataRow - int32_t rowSize; -} SMemRowBuilder; - -typedef struct STableDataBlocks { - int8_t tsSource; // where does the UNIX timestamp come from, server or client - bool ordered; // if current rows are ordered or not - int32_t vgId; // virtual group id - int64_t prevTS; // previous timestamp, recorded to decide if the records array is ts ascending - int32_t numOfTables; // number of tables in current submit block - int32_t rowSize; // row size for current table - uint32_t nAllocSize; - uint32_t headerSize; // header for table info (uid, tid, submit metadata) - uint32_t size; - STableMeta *pTableMeta; // the tableMeta of current table, the table meta will be used during submit, keep a ref to - // avoid to be removed from cache - char *pData; - bool cloned; - int32_t createTbReqLen; - SParsedDataColInfo boundColumnInfo; - SRowBuilder rowBuilder; -} STableDataBlocks; +typedef struct SVgroupDataCxt { + int32_t vgId; + SSubmitReq2 *pData; +} SVgroupDataCxt; -int32_t insGetExtendedRowSize(STableDataBlocks *pBlock); -void insGetSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo *spd, col_id_t idx, int32_t *toffset, col_id_t *colIdx); -int32_t insSetBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks *dataBuf, int32_t numOfRows, SMsgBuf *pMsg); -int32_t insSchemaIdxCompar(const void *lhs, const void *rhs); -int32_t insBoundIdxCompar(const void *lhs, const void *rhs); -void insSetBoundColumnInfo(SParsedDataColInfo *pColList, SSchema *pSchema, col_id_t numOfCols); -void insDestroyBlockArrayList(SArray *pDataBlockList); -void insDestroyBlockHashmap(SHashObj *pDataBlockHash); -int32_t insInitRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo *pColInfo); -int32_t insGetDataBlockFromList(SHashObj *pHashList, void *id, int32_t idLen, int32_t size, int32_t startOffset, - int32_t rowSize, STableMeta *pTableMeta, STableDataBlocks **dataBlocks, - SArray *pBlockList, SVCreateTbReq *pCreateTbReq); -int32_t insMergeTableDataBlocks(SHashObj *pHashObj, SArray **pVgDataBlocks); -int32_t insBuildCreateTbMsg(STableDataBlocks *pBlocks, SVCreateTbReq *pCreateTbReq); -int32_t insAllocateMemForSize(STableDataBlocks *pDataBlock, int32_t allSize); int32_t insCreateSName(SName *pName, struct SToken *pTableName, int32_t acctId, const char *dbName, SMsgBuf *pMsgBuf); -int32_t insFindCol(struct SToken *pColname, int32_t start, int32_t end, SSchema *pSchema); +int16_t insFindCol(struct SToken *pColname, int16_t start, int16_t end, SSchema *pSchema); void insBuildCreateTbReq(SVCreateTbReq *pTbReq, const char *tname, STag *pTag, int64_t suid, const char *sname, SArray *tagName, uint8_t tagNum, int32_t ttl); -int32_t insMemRowAppend(SMsgBuf *pMsgBuf, const void *value, int32_t len, void *param); -int32_t insCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start); -int32_t insBuildOutput(SHashObj *pVgroupsHashObj, SArray *pVgDataBlocks, SArray **pDataBlocks); -void insDestroyDataBlock(STableDataBlocks *pDataBlock); +int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo *pInfo); +void insCheckTableDataOrder(STableDataCxt *pTableCxt, TSKEY tsKey); +int32_t insGetTableDataCxt(SHashObj *pHash, void *id, int32_t idLen, STableMeta *pTableMeta, + SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode); +int32_t initTableColSubmitData(STableDataCxt *pTableCxt); +int32_t insMergeTableDataCxt(SHashObj *pTableHash, SArray **pVgDataBlocks); +int32_t insBuildVgDataBlocks(SHashObj *pVgroupsHashObj, SArray *pVgDataBlocks, SArray **pDataBlocks); +void insDestroyTableDataCxtHashMap(SHashObj *pTableCxtHash); +void insDestroyVgroupDataCxt(SVgroupDataCxt *pVgCxt); +void insDestroyVgroupDataCxtList(SArray *pVgCxtList); +void insDestroyVgroupDataCxtHashMap(SHashObj *pVgCxtHash); +void insDestroyTableDataCxt(STableDataCxt *pTableCxt); +void insDestroyBoundColInfo(SBoundColInfo *pInfo); #endif // TDENGINE_PAR_INSERT_UTIL_H diff --git a/source/libs/parser/inc/parToken.h b/source/libs/parser/inc/parToken.h index fb4b46aa35094fd3ac65171c5499c038eb92d233..642979c471ab7b409bd622de26c2c2deb1879e83 100644 --- a/source/libs/parser/inc/parToken.h +++ b/source/libs/parser/inc/parToken.h @@ -175,8 +175,6 @@ _end: void taosCleanupKeywordsTable(); -SToken taosTokenDup(SToken *pToken, char *buf, int32_t len); - #ifdef __cplusplus } #endif diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 7136a555cd79852d2cb03dfa1174dc10db25deda..abb34e7e806efa72da072f258e0352a0d1fdedc7 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -433,6 +433,9 @@ cmd ::= SHOW TAGS FROM table_name_cond(A) from_db_opt(B). cmd ::= SHOW TABLE TAGS tag_list_opt(C) FROM table_name_cond(A) from_db_opt(B). { pCxt->pRootNode = createShowTableTagsStmt(pCxt, A, B, C); } cmd ::= SHOW VNODES NK_INTEGER(A). { pCxt->pRootNode = createShowVnodesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &A), NULL); } cmd ::= SHOW VNODES NK_STRING(A). { pCxt->pRootNode = createShowVnodesStmt(pCxt, NULL, createValueNode(pCxt, TSDB_DATA_TYPE_VARCHAR, &A)); } +// show alive +cmd ::= SHOW db_name_cond_opt(A) ALIVE. { pCxt->pRootNode = createShowAliveStmt(pCxt, A, QUERY_NODE_SHOW_DB_ALIVE_STMT); } +cmd ::= SHOW CLUSTER ALIVE. { pCxt->pRootNode = createShowAliveStmt(pCxt, NULL, QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT); } db_name_cond_opt(A) ::= . { A = createDefaultDatabaseCondValue(pCxt); } db_name_cond_opt(A) ::= db_name(B) NK_DOT. { A = createIdentifierValueNode(pCxt, &B); } @@ -458,17 +461,19 @@ tag_item(A) ::= column_name(B) column_alias(C). tag_item(A) ::= column_name(B) AS column_alias(C). { A = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &B), &C); } /************************************************ create index ********************************************************/ -cmd ::= CREATE SMA INDEX not_exists_opt(D) +cmd ::= CREATE SMA INDEX not_exists_opt(D) full_index_name(A) ON full_table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, D, A, B, NULL, C); } +cmd ::= CREATE INDEX not_exists_opt(D) + full_index_name(A) ON full_table_name(B) NK_LP col_name_list(C) NK_RP. { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_NORMAL, D, A, B, C, NULL); } cmd ::= DROP INDEX exists_opt(B) full_index_name(A). { pCxt->pRootNode = createDropIndexStmt(pCxt, B, A); } full_index_name(A) ::= index_name(B). { A = createRealTableNodeForIndexName(pCxt, NULL, &B); } full_index_name(A) ::= db_name(B) NK_DOT index_name(C). { A = createRealTableNodeForIndexName(pCxt, &B, &C); } -index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL +index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL NK_LP duration_literal(C) NK_RP sliding_opt(D) sma_stream_opt(E). { A = createIndexOption(pCxt, B, releaseRawExprNode(pCxt, C), NULL, D, E); } -index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL - NK_LP duration_literal(C) NK_COMMA duration_literal(D) NK_RP sliding_opt(E) +index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL + NK_LP duration_literal(C) NK_COMMA duration_literal(D) NK_RP sliding_opt(E) sma_stream_opt(F). { A = createIndexOption(pCxt, B, releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, D), E, F); } %type func_list { SNodeList* } @@ -476,7 +481,15 @@ index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL func_list(A) ::= func(B). { A = createNodeList(pCxt, B); } func_list(A) ::= func_list(B) NK_COMMA func(C). { A = addNodeToList(pCxt, B, C); } -func(A) ::= function_name(B) NK_LP expression_list(C) NK_RP. { A = createFunctionNode(pCxt, &B, C); } +func(A) ::= sma_func_name(B) NK_LP expression_list(C) NK_RP. { A = createFunctionNode(pCxt, &B, C); } + +%type sma_func_name { SToken } +%destructor sma_func_name { } +sma_func_name(A) ::= function_name(B). { A = B; } +sma_func_name(A) ::= COUNT(B). { A = B; } +sma_func_name(A) ::= FIRST(B). { A = B; } +sma_func_name(A) ::= LAST(B). { A = B; } +sma_func_name(A) ::= LAST_ROW(B). { A = B; } sma_stream_opt(A) ::= . { A = createStreamOptions(pCxt); } sma_stream_opt(A) ::= sma_stream_opt(B) WATERMARK duration_literal(C). { ((SStreamOptions*)B)->pWatermark = releaseRawExprNode(pCxt, C); A = B; } @@ -534,9 +547,15 @@ bufsize_opt(A) ::= BUFSIZE NK_INTEGER(B). /************************************************ create/drop stream **************************************************/ cmd ::= CREATE STREAM not_exists_opt(E) stream_name(A) stream_options(B) INTO - full_table_name(C) tags_def_opt(F) subtable_opt(G) AS query_or_subquery(D). { pCxt->pRootNode = createCreateStreamStmt(pCxt, E, &A, C, B, F, G, D); } + full_table_name(C) col_list_opt(H) tags_def_opt(F) subtable_opt(G) + AS query_or_subquery(D). { pCxt->pRootNode = createCreateStreamStmt(pCxt, E, &A, C, B, F, G, D, H); } cmd ::= DROP STREAM exists_opt(A) stream_name(B). { pCxt->pRootNode = createDropStreamStmt(pCxt, A, &B); } +%type col_list_opt { SNodeList* } +%destructor col_list_opt { nodesDestroyList($$); } +col_list_opt(A) ::= . { A = NULL; } +col_list_opt(A) ::= NK_LP col_name_list(B) NK_RP. { A = B; } + stream_options(A) ::= . { A = createStreamOptions(pCxt); } stream_options(A) ::= stream_options(B) TRIGGER AT_ONCE. { ((SStreamOptions*)B)->triggerType = STREAM_TRIGGER_AT_ONCE; A = B; } stream_options(A) ::= stream_options(B) TRIGGER WINDOW_CLOSE. { ((SStreamOptions*)B)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; A = B; } @@ -740,6 +759,7 @@ pseudo_column(A) ::= WSTART(B). pseudo_column(A) ::= WEND(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= WDURATION(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= IROWTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } +pseudo_column(A) ::= ISFILLED(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= QTAGS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } function_expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } @@ -971,6 +991,8 @@ twindow_clause_opt(A) ::= twindow_clause_opt(A) ::= INTERVAL NK_LP duration_literal(B) NK_COMMA duration_literal(C) NK_RP sliding_opt(D) fill_opt(E). { A = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C), D, E); } +twindow_clause_opt(A) ::= + EVENT_WINDOW START WITH search_condition(B) END WITH search_condition(C). { A = createEventWindowNode(pCxt, B, C); } sliding_opt(A) ::= . { A = NULL; } sliding_opt(A) ::= SLIDING NK_LP duration_literal(B) NK_RP. { A = releaseRawExprNode(pCxt, B); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index eaa72b137751e5fbc40e94a7ec7b28468c7b5c9d..f582adb0c6fe7d75cd0ac68c035bac5dc5c71926 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -623,6 +623,20 @@ SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr) { return (SNode*)state; } +SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond) { + CHECK_PARSER_STATUS(pCxt); + SEventWindowNode* pEvent = (SEventWindowNode*)nodesMakeNode(QUERY_NODE_EVENT_WINDOW); + CHECK_OUT_OF_MEM(pEvent); + pEvent->pCol = createPrimaryKeyCol(pCxt, NULL); + if (NULL == pEvent->pCol) { + nodesDestroyNode((SNode*)pEvent); + CHECK_OUT_OF_MEM(NULL); + } + pEvent->pStartCond = pStartCond; + pEvent->pEndCond = pEndCond; + return (SNode*)pEvent; +} + SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill) { CHECK_PARSER_STATUS(pCxt); @@ -1213,7 +1227,7 @@ SNode* createCreateSubTableClause(SAstCreateContext* pCxt, bool ignoreExists, SN SNode* createCreateMultiTableStmt(SAstCreateContext* pCxt, SNodeList* pSubTables) { CHECK_PARSER_STATUS(pCxt); - SCreateMultiTableStmt* pStmt = (SCreateMultiTableStmt*)nodesMakeNode(QUERY_NODE_CREATE_MULTI_TABLE_STMT); + SCreateMultiTablesStmt* pStmt = (SCreateMultiTablesStmt*)nodesMakeNode(QUERY_NODE_CREATE_MULTI_TABLES_STMT); CHECK_OUT_OF_MEM(pStmt); pStmt->pSubTables = pSubTables; return (SNode*)pStmt; @@ -1374,6 +1388,38 @@ SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName) { return (SNode*)pStmt; } +SNode* createShowAliveStmt(SAstCreateContext* pCxt, SNode* pNode, ENodeType type) { + CHECK_PARSER_STATUS(pCxt); + SToken dbToken = {0}; + SToken* pDbToken = NULL; + + if (pNode) { + SValueNode* pDbName = (SValueNode*)pNode; + if (pDbName->literal) { + dbToken.z = pDbName->literal; + dbToken.n = strlen(pDbName->literal); + pDbToken = &dbToken; + } + } + + if (pDbToken && !checkDbName(pCxt, pDbToken, true)) { + nodesDestroyNode(pNode); + return NULL; + } + + SShowAliveStmt* pStmt = (SShowAliveStmt*)nodesMakeNode(type); + CHECK_OUT_OF_MEM(pStmt); + + if (pDbToken) { + COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbToken); + } + if (pNode) { + nodesDestroyNode(pNode); + } + + return (SNode*)pStmt; +} + SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable) { CHECK_PARSER_STATUS(pCxt); SShowCreateTableStmt* pStmt = (SShowCreateTableStmt*)nodesMakeNode(type); @@ -1435,7 +1481,7 @@ SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const ST } SCreateUserStmt* pStmt = (SCreateUserStmt*)nodesMakeNode(QUERY_NODE_CREATE_USER_STMT); CHECK_OUT_OF_MEM(pStmt); - COPY_STRING_FORM_ID_TOKEN(pStmt->useName, pUserName); + COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName); strcpy(pStmt->password, password); pStmt->sysinfo = sysinfo; return (SNode*)pStmt; @@ -1448,7 +1494,7 @@ SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t al } SAlterUserStmt* pStmt = (SAlterUserStmt*)nodesMakeNode(QUERY_NODE_ALTER_USER_STMT); CHECK_OUT_OF_MEM(pStmt); - COPY_STRING_FORM_ID_TOKEN(pStmt->useName, pUserName); + COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName); pStmt->alterType = alterType; switch (alterType) { case TSDB_ALTER_USER_PASSWD: { @@ -1479,7 +1525,7 @@ SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName) { } SDropUserStmt* pStmt = (SDropUserStmt*)nodesMakeNode(QUERY_NODE_DROP_USER_STMT); CHECK_OUT_OF_MEM(pStmt); - COPY_STRING_FORM_ID_TOKEN(pStmt->useName, pUserName); + COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName); return (SNode*)pStmt; } @@ -1767,7 +1813,7 @@ SNode* createStreamOptions(SAstCreateContext* pCxt) { } SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pStreamName, SNode* pRealTable, - SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery) { + SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery, SNodeList* pCols) { CHECK_PARSER_STATUS(pCxt); if (!checkStreamName(pCxt, pStreamName)) { return NULL; @@ -1775,16 +1821,15 @@ SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken SCreateStreamStmt* pStmt = (SCreateStreamStmt*)nodesMakeNode(QUERY_NODE_CREATE_STREAM_STMT); CHECK_OUT_OF_MEM(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->streamName, pStreamName); - if (NULL != pRealTable) { - strcpy(pStmt->targetDbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->targetTabName, ((SRealTableNode*)pRealTable)->table.tableName); - nodesDestroyNode(pRealTable); - } + strcpy(pStmt->targetDbName, ((SRealTableNode*)pRealTable)->table.dbName); + strcpy(pStmt->targetTabName, ((SRealTableNode*)pRealTable)->table.tableName); + nodesDestroyNode(pRealTable); pStmt->ignoreExists = ignoreExists; pStmt->pOptions = (SStreamOptions*)pOptions; pStmt->pQuery = pQuery; pStmt->pTags = pTags; pStmt->pSubtable = pSubtable; + pStmt->pCols = pCols; return (SNode*)pStmt; } diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 8346ab8c05e7dd606b0223ad53663d69386d7927..fae62626fa3666909208ce9220215baea8be3810 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -252,7 +252,7 @@ static int32_t collectMetaKeyFromCreateTable(SCollectMetaKeyCxt* pCxt, SCreateTa return code; } -static int32_t collectMetaKeyFromCreateMultiTable(SCollectMetaKeyCxt* pCxt, SCreateMultiTableStmt* pStmt) { +static int32_t collectMetaKeyFromCreateMultiTable(SCollectMetaKeyCxt* pCxt, SCreateMultiTablesStmt* pStmt) { int32_t code = TSDB_CODE_SUCCESS; SNode* pNode = NULL; FOREACH(pNode, pStmt->pSubTables) { @@ -355,7 +355,12 @@ static int32_t collectMetaKeyFromDescribe(SCollectMetaKeyCxt* pCxt, SDescribeStm } static int32_t collectMetaKeyFromCreateStream(SCollectMetaKeyCxt* pCxt, SCreateStreamStmt* pStmt) { - return collectMetaKeyFromQuery(pCxt, pStmt->pQuery); + int32_t code = + reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->targetDbName, pStmt->targetTabName, pCxt->pMetaCache); + if (TSDB_CODE_SUCCESS == code) { + code = collectMetaKeyFromQuery(pCxt, pStmt->pQuery); + } + return code; } static int32_t collectMetaKeyFromShowDnodes(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { @@ -613,8 +618,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { return collectMetaKeyFromFlushDatabase(pCxt, (SFlushDatabaseStmt*)pStmt); case QUERY_NODE_CREATE_TABLE_STMT: return collectMetaKeyFromCreateTable(pCxt, (SCreateTableStmt*)pStmt); - case QUERY_NODE_CREATE_MULTI_TABLE_STMT: - return collectMetaKeyFromCreateMultiTable(pCxt, (SCreateMultiTableStmt*)pStmt); + case QUERY_NODE_CREATE_MULTI_TABLES_STMT: + return collectMetaKeyFromCreateMultiTable(pCxt, (SCreateMultiTablesStmt*)pStmt); case QUERY_NODE_DROP_TABLE_STMT: return collectMetaKeyFromDropTable(pCxt, (SDropTableStmt*)pStmt); case QUERY_NODE_ALTER_TABLE_STMT: diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index 4fd711e0ea43a0f82b09d598dca866d8ef682b04..e4de60fd051d6f7f4e781160305ffb95a3ec7d3e 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -78,7 +78,7 @@ static int32_t authSetOperator(SAuthCxt* pCxt, SSetOperator* pSetOper) { } static int32_t authDropUser(SAuthCxt* pCxt, SDropUserStmt* pStmt) { - if (!pCxt->pParseCxt->isSuperUser || 0 == strcmp(pStmt->useName, TSDB_DEFAULT_USER)) { + if (!pCxt->pParseCxt->isSuperUser || 0 == strcmp(pStmt->userName, TSDB_DEFAULT_USER)) { return TSDB_CODE_PAR_PERMISSION_DENIED; } return TSDB_CODE_SUCCESS; @@ -108,7 +108,7 @@ static int32_t authCreateTable(SAuthCxt* pCxt, SCreateTableStmt* pStmt) { return checkAuth(pCxt, pStmt->dbName, AUTH_TYPE_WRITE); } -static int32_t authCreateMultiTable(SAuthCxt* pCxt, SCreateMultiTableStmt* pStmt) { +static int32_t authCreateMultiTable(SAuthCxt* pCxt, SCreateMultiTablesStmt* pStmt) { int32_t code = TSDB_CODE_SUCCESS; SNode* pNode = NULL; FOREACH(pNode, pStmt->pSubTables) { @@ -134,8 +134,8 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { return authInsert(pCxt, (SInsertStmt*)pStmt); case QUERY_NODE_CREATE_TABLE_STMT: return authCreateTable(pCxt, (SCreateTableStmt*)pStmt); - case QUERY_NODE_CREATE_MULTI_TABLE_STMT: - return authCreateMultiTable(pCxt, (SCreateMultiTableStmt*)pStmt); + case QUERY_NODE_CREATE_MULTI_TABLES_STMT: + return authCreateMultiTable(pCxt, (SCreateMultiTablesStmt*)pStmt); case QUERY_NODE_SHOW_DNODES_STMT: case QUERY_NODE_SHOW_MNODES_STMT: case QUERY_NODE_SHOW_MODULES_STMT: @@ -145,6 +145,8 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { case QUERY_NODE_SHOW_CLUSTER_STMT: case QUERY_NODE_SHOW_LICENCES_STMT: case QUERY_NODE_SHOW_VGROUPS_STMT: + case QUERY_NODE_SHOW_DB_ALIVE_STMT: + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: case QUERY_NODE_SHOW_VNODES_STMT: diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 358baa74cb3932887894bda79a6e26e9a888c797..4742921d083257b56d7095d0e8aca8fe705900c2 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -18,6 +18,16 @@ #include "parToken.h" #include "ttime.h" +static void clearColValArray(SArray* pCols) { + int32_t num = taosArrayGetSize(pCols); + for (int32_t i = 0; i < num; ++i) { + SColVal* pCol = taosArrayGet(pCols, i); + if (TSDB_DATA_TYPE_NCHAR == pCol->type) { + taosMemoryFreeClear(pCol->value.pData); + } + } +} + int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf, int32_t msgBufLen) { SMsgBuf msg = {.buf = msgBuf, .len = msgBufLen}; @@ -45,95 +55,46 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* return TSDB_CODE_SUCCESS; } -typedef struct SmlExecTableHandle { - SParsedDataColInfo tags; // each table - SVCreateTbReq createTblReq; // each table -} SmlExecTableHandle; - -typedef struct SmlExecHandle { - SHashObj* pBlockHash; - SmlExecTableHandle tableExecHandle; - SQuery* pQuery; -} SSmlExecHandle; - -static void smlDestroyTableHandle(void* pHandle) { - SmlExecTableHandle* handle = (SmlExecTableHandle*)pHandle; - destroyBoundColumnInfo(&handle->tags); - tdDestroySVCreateTbReq(&handle->createTblReq); -} - -static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SSchema* pSchema, bool isTag) { - col_id_t nCols = pColList->numOfCols; - - pColList->numOfBound = 0; - pColList->boundNullLen = 0; - memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols); - for (col_id_t i = 0; i < nCols; ++i) { - pColList->cols[i].valStat = VAL_STAT_NONE; +static int32_t smlBoundColumnData(SArray* cols, SBoundColInfo* pBoundInfo, SSchema* pSchema, bool isTag) { + bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool)); + if (NULL == pUseCols) { + return TSDB_CODE_OUT_OF_MEMORY; } - bool isOrdered = true; - col_id_t lastColIdx = -1; // last column found + pBoundInfo->numOfBound = 0; + int16_t lastColIdx = -1; // last column found + int32_t code = TSDB_CODE_SUCCESS; + for (int i = 0; i < taosArrayGetSize(cols); ++i) { - SSmlKv* kv = taosArrayGetP(cols, i); + SSmlKv* kv = taosArrayGet(cols, i); SToken sToken = {.n = kv->keyLen, .z = (char*)kv->key}; col_id_t t = lastColIdx + 1; - col_id_t index = ((t == 0 && !isTag) ? 0 : insFindCol(&sToken, t, nCols, pSchema)); - uDebug("SML, index:%d, t:%d, ncols:%d", index, t, nCols); + col_id_t index = ((t == 0 && !isTag) ? 0 : insFindCol(&sToken, t, pBoundInfo->numOfCols, pSchema)); + uDebug("SML, index:%d, t:%d, ncols:%d", index, t, pBoundInfo->numOfCols); if (index < 0 && t > 0) { index = insFindCol(&sToken, 0, t, pSchema); - isOrdered = false; } + if (index < 0) { uError("smlBoundColumnData. index:%d", index); - return TSDB_CODE_SML_INVALID_DATA; + code = TSDB_CODE_SML_INVALID_DATA; + goto end; } - if (pColList->cols[index].valStat == VAL_STAT_HAS) { + if (pUseCols[index]) { uError("smlBoundColumnData. already set. index:%d", index); - return TSDB_CODE_SML_INVALID_DATA; + code = TSDB_CODE_SML_INVALID_DATA; + goto end; } lastColIdx = index; - pColList->cols[index].valStat = VAL_STAT_HAS; - pColList->boundColumns[pColList->numOfBound] = index; - ++pColList->numOfBound; - switch (pSchema[t].type) { - case TSDB_DATA_TYPE_BINARY: - pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + CHAR_BYTES); - break; - case TSDB_DATA_TYPE_NCHAR: - pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); - break; - default: - pColList->boundNullLen += TYPE_BYTES[pSchema[t].type]; - break; - } + pUseCols[index] = true; + pBoundInfo->pColIndex[pBoundInfo->numOfBound] = index; + ++pBoundInfo->numOfBound; } - pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; - - if (!isOrdered) { - pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); - if (NULL == pColList->colIdxInfo) { - return TSDB_CODE_OUT_OF_MEMORY; - } - SBoundIdxInfo* pColIdx = pColList->colIdxInfo; - for (col_id_t i = 0; i < pColList->numOfBound; ++i) { - pColIdx[i].schemaColIdx = pColList->boundColumns[i]; - pColIdx[i].boundIdx = i; - } - taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insSchemaIdxCompar); - for (col_id_t i = 0; i < pColList->numOfBound; ++i) { - pColIdx[i].finalIdx = i; - } - taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insBoundIdxCompar); - } - - if (pColList->numOfCols > pColList->numOfBound) { - memset(&pColList->boundColumns[pColList->numOfBound], 0, - sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); - } +end: + taosMemoryFree(pUseCols); - return TSDB_CODE_SUCCESS; + return code; } /** @@ -146,7 +107,7 @@ static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SS * @param msg * @return int32_t */ -static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* pSchema, STag** ppTag, SArray** tagName, +static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchema, STag** ppTag, SArray** tagName, SMsgBuf* msg) { SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal)); if (!pTagArray) { @@ -159,8 +120,8 @@ static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* p int32_t code = TSDB_CODE_SUCCESS; for (int i = 0; i < tags->numOfBound; ++i) { - SSchema* pTagSchema = &pSchema[tags->boundColumns[i]]; - SSmlKv* kv = taosArrayGetP(cols, i); + SSchema* pTagSchema = &pSchema[tags->pColIndex[i]]; + SSmlKv* kv = taosArrayGet(cols, i); taosArrayPush(*tagName, pTagSchema->name); STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; @@ -207,153 +168,253 @@ end: return code; } -int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta, - char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int16_t msgBufLen) { - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; +STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) { + STableDataCxt* pTableCxt = NULL; + SVCreateTbReq* pCreateTbReq = NULL; + int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, + sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false); + if (ret != TSDB_CODE_SUCCESS) { + return NULL; + } - SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle; - smlDestroyTableHandle(&smlHandle->tableExecHandle); // free for each table - SSchema* pTagsSchema = getTableTagSchema(pTableMeta); - insSetBoundColumnInfo(&smlHandle->tableExecHandle.tags, pTagsSchema, getNumOfTags(pTableMeta)); - int ret = smlBoundColumnData(tags, &smlHandle->tableExecHandle.tags, pTagsSchema, true); + ret = initTableColSubmitData(pTableCxt); if (ret != TSDB_CODE_SUCCESS) { - buildInvalidOperationMsg(&pBuf, "bound tags error"); + return NULL; + } + return pTableCxt; +} + +int32_t smlBuildRow(STableDataCxt* pTableCxt) { + SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + int ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); + if (TSDB_CODE_SUCCESS != ret) { return ret; } - STag* pTag = NULL; - SArray* tagName = NULL; - ret = smlBuildTagRow(tags, &smlHandle->tableExecHandle.tags, pTagsSchema, &pTag, &tagName, &pBuf); + insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow)); + return TSDB_CODE_SUCCESS; +} + +int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32_t index) { + int ret = TSDB_CODE_SUCCESS; + SSchema* pColSchema = schema + index; + SColVal* pVal = taosArrayGet(pTableCxt->pValues, index); + SSmlKv* kv = (SSmlKv*)data; + if(kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || kv->type != pColSchema->type){ + ret = TSDB_CODE_SML_INVALID_DATA; + goto end; + } + if (kv->type == TSDB_DATA_TYPE_NCHAR) { + int32_t len = 0; + int64_t size = pColSchema->bytes - VARSTR_HEADER_SIZE; + if(size <= 0){ + ret = TSDB_CODE_SML_INVALID_DATA; + goto end; + } + char* pUcs4 = taosMemoryCalloc(1, size); + if (NULL == pUcs4) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, size, &len)) { + if (errno == E2BIG) { + taosMemoryFree(pUcs4); + ret = TSDB_CODE_PAR_VALUE_TOO_LONG; + goto end; + } + taosMemoryFree(pUcs4); + ret = TSDB_CODE_TSC_INVALID_VALUE; + goto end; + } + pVal->value.pData = pUcs4; + pVal->value.nData = len; + } else if (kv->type == TSDB_DATA_TYPE_BINARY) { + pVal->value.nData = kv->length; + pVal->value.pData = (uint8_t*)kv->value; + } else { + memcpy(&pVal->value.val, &(kv->value), kv->length); + } + pVal->flag = CV_FLAG_VALUE; + +end: + return ret; +} + +int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, + STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, + char* msgBuf, int16_t msgBufLen) { + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + + SSchema* pTagsSchema = getTableTagSchema(pTableMeta); + SBoundColInfo bindTags = {0}; + SVCreateTbReq* pCreateTblReq = NULL; + SArray* tagName = NULL; + + insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags); + int ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true); if (ret != TSDB_CODE_SUCCESS) { - taosArrayDestroy(tagName); - return ret; + buildInvalidOperationMsg(&pBuf, "bound tags error"); + goto end; } - insBuildCreateTbReq(&smlHandle->tableExecHandle.createTblReq, tableName, pTag, pTableMeta->suid, NULL, tagName, - pTableMeta->tableInfo.numOfTags, ttl); - taosArrayDestroy(tagName); + STag* pTag = NULL; + + ret = smlBuildTagRow(tags, &bindTags, pTagsSchema, &pTag, &tagName, &pBuf); + if (ret != TSDB_CODE_SUCCESS) { + goto end; + } - smlHandle->tableExecHandle.createTblReq.ctb.stbName = taosMemoryMalloc(sTableNameLen + 1); - memcpy(smlHandle->tableExecHandle.createTblReq.ctb.stbName, sTableName, sTableNameLen); - smlHandle->tableExecHandle.createTblReq.ctb.stbName[sTableNameLen] = 0; + pCreateTblReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == pCreateTblReq) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + insBuildCreateTbReq(pCreateTblReq, tableName, pTag, pTableMeta->suid, NULL, tagName, pTableMeta->tableInfo.numOfTags, + ttl); + + pCreateTblReq->ctb.stbName = taosMemoryCalloc(1, sTableNameLen + 1); + memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen); + + if (dataFormat) { + STableDataCxt** pTableCxt = (STableDataCxt**)taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, + &pTableMeta->uid, sizeof(pTableMeta->uid)); + if (NULL == pTableCxt) { + ret = buildInvalidOperationMsg(&pBuf, "dataformat true. get tableDataCtx error"); + goto end; + } + (*pTableCxt)->pData->flags |= SUBMIT_REQ_AUTO_CREATE_TABLE; + (*pTableCxt)->pData->pCreateTbReq = pCreateTblReq; + (*pTableCxt)->pMeta->uid = pTableMeta->uid; + (*pTableCxt)->pMeta->vgId = pTableMeta->vgId; + pCreateTblReq = NULL; + goto end; + } - STableDataBlocks* pDataBlock = NULL; - ret = insGetDataBlockFromList(smlHandle->pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid), - TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize, - pTableMeta, &pDataBlock, NULL, &smlHandle->tableExecHandle.createTblReq); + STableDataCxt* pTableCxt = NULL; + ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, + sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false); if (ret != TSDB_CODE_SUCCESS) { - buildInvalidOperationMsg(&pBuf, "create data block error"); - return ret; + buildInvalidOperationMsg(&pBuf, "insGetTableDataCxt error"); + goto end; } SSchema* pSchema = getTableColumnSchema(pTableMeta); - - ret = smlBoundColumnData(colsSchema, &pDataBlock->boundColumnInfo, pSchema, false); + ret = smlBoundColumnData(colsSchema, &pTableCxt->boundColsInfo, pSchema, false); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "bound cols error"); - return ret; + goto end; } - int32_t extendedRowSize = insGetExtendedRowSize(pDataBlock); - SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; - SRowBuilder* pBuilder = &pDataBlock->rowBuilder; - SMemParam param = {.rb = pBuilder}; - insInitRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo); + ret = initTableColSubmitData(pTableCxt); + if (ret != TSDB_CODE_SUCCESS) { + buildInvalidOperationMsg(&pBuf, "initTableColSubmitData error"); + goto end; + } int32_t rowNum = taosArrayGetSize(cols); if (rowNum <= 0) { - return buildInvalidOperationMsg(&pBuf, "cols size <= 0"); - } - ret = insAllocateMemForSize(pDataBlock, extendedRowSize * rowNum); - if (ret != TSDB_CODE_SUCCESS) { - buildInvalidOperationMsg(&pBuf, "allocate memory error"); - return ret; + ret = buildInvalidOperationMsg(&pBuf, "cols size <= 0"); + goto end; } + for (int32_t r = 0; r < rowNum; ++r) { - STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header - tdSRowResetBuf(pBuilder, row); - void* rowData = taosArrayGetP(cols, r); - size_t rowDataSize = 0; - if (format) { - rowDataSize = taosArrayGetSize(rowData); - } + void* rowData = taosArrayGetP(cols, r); // 1. set the parsed value from sql string - for (int c = 0, j = 0; c < spd->numOfBound; ++c) { - SSchema* pColSchema = &pSchema[spd->boundColumns[c]]; - - param.schema = pColSchema; - insGetSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); - - SSmlKv* kv = NULL; - if (format) { - if (j < rowDataSize) { - kv = taosArrayGetP(rowData, j); - if (rowDataSize != spd->numOfBound && j != 0 && - (kv->keyLen != strlen(pColSchema->name) || strncmp(kv->key, pColSchema->name, kv->keyLen) != 0)) { - kv = NULL; - } else { - j++; - } - } - } else { - void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); - if (p) kv = *p; + for (int c = 0; c < pTableCxt->boundColsInfo.numOfBound; ++c) { + SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]]; + SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]); + void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); + if (p == NULL) { + continue; } - - if (kv) { - int32_t colLen = kv->length; - if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { - uDebug("SML:data before:%" PRId64 ", precision:%d", kv->i, pTableMeta->tableInfo.precision); - kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision); - uDebug("SML:data after:%" PRId64 ", precision:%d", kv->i, pTableMeta->tableInfo.precision); + SSmlKv* kv = *(SSmlKv**)p; + if(kv->type != pColSchema->type){ + ret = buildInvalidOperationMsg(&pBuf, "kv type not equal to col type"); + goto end; + } + if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { + kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision); + } + if (kv->type == TSDB_DATA_TYPE_NCHAR) { + int32_t len = 0; + char* pUcs4 = taosMemoryCalloc(1, pColSchema->bytes - VARSTR_HEADER_SIZE); + if (NULL == pUcs4) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; } - - if (IS_VAR_DATA_TYPE(kv->type)) { - insMemRowAppend(&pBuf, kv->value, colLen, ¶m); - } else { - insMemRowAppend(&pBuf, &(kv->value), colLen, ¶m); + if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len)) { + if (errno == E2BIG) { + buildInvalidOperationMsg(&pBuf, "value too long"); + ret = TSDB_CODE_PAR_VALUE_TOO_LONG; + goto end; + } + ret = buildInvalidOperationMsg(&pBuf, strerror(errno)); + goto end; } + pVal->value.pData = pUcs4; + pVal->value.nData = len; + } else if (kv->type == TSDB_DATA_TYPE_BINARY) { + pVal->value.nData = kv->length; + pVal->value.pData = (uint8_t*)kv->value; } else { - pBuilder->hasNone = true; - } - - if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { - TSKEY tsKey = TD_ROW_KEY(row); - insCheckTimestamp(pDataBlock, (const char*)&tsKey); + memcpy(&pVal->value.val, &(kv->value), kv->length); } + pVal->flag = CV_FLAG_VALUE; } - // set the null value for the columns that do not assign values - if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - pBuilder->hasNone = true; + SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); + if (TSDB_CODE_SUCCESS != ret) { + buildInvalidOperationMsg(&pBuf, "tRowBuild error"); + goto end; } - - tdSRowEnd(pBuilder); - pDataBlock->size += extendedRowSize; + insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow)); + clearColValArray(pTableCxt->pValues); } - SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); - return insSetBlockInfo(pBlocks, pDataBlock, rowNum, &pBuf); +end: + insDestroyBoundColInfo(&bindTags); + tdDestroySVCreateTbReq(pCreateTblReq); + taosMemoryFree(pCreateTblReq); + taosArrayDestroy(tagName); + return ret; } -void* smlInitHandle(SQuery* pQuery) { - SSmlExecHandle* handle = taosMemoryCalloc(1, sizeof(SSmlExecHandle)); - if (!handle) return NULL; - handle->pBlockHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - handle->pQuery = pQuery; - - return handle; -} +SQuery* smlInitHandle() { + SQuery* pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); + if (NULL == pQuery) { + uError("create pQuery error"); + return NULL; + } + pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + pQuery->haveResultSet = false; + pQuery->msgType = TDMT_VND_SUBMIT; + SVnodeModifyOpStmt* stmt = (SVnodeModifyOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT); + if (NULL == stmt) { + uError("create SVnodeModifyOpStmt error"); + qDestroyQuery(pQuery); + return NULL; + } + stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + stmt->freeHashFunc = insDestroyTableDataCxtHashMap; + stmt->freeArrayFunc = insDestroyVgroupDataCxtList; -void smlDestroyHandle(void* pHandle) { - if (!pHandle) return; - SSmlExecHandle* handle = (SSmlExecHandle*)pHandle; - insDestroyBlockHashmap(handle->pBlockHash); - smlDestroyTableHandle(&handle->tableExecHandle); - taosMemoryFree(handle); + pQuery->pRoot = (SNode*)stmt; + return pQuery; } -int32_t smlBuildOutput(void* handle, SHashObj* pVgHash) { - SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle; - return qBuildStmtOutput(smlHandle->pQuery, pVgHash, smlHandle->pBlockHash); +int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) { + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)(handle)->pRoot; + // merge according to vgId + int32_t code = insMergeTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks); + if (code != TSDB_CODE_SUCCESS) { + uError("insMergeTableDataCxt failed"); + return code; + } + code = insBuildVgDataBlocks(pVgHash, pStmt->pVgDataBlocks, &pStmt->pDataBlocks); + if (code != TSDB_CODE_SUCCESS) { + uError("insBuildVgDataBlocks failed"); + return code; + } + return code; } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 98c6aed829128140217d2c1ba243afac7b3ecb01..3c5d4d506a72d9a9e3d8bda482cbf1010c8acb3f 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -38,13 +38,13 @@ } while (TK_NK_SPACE == (token).type) typedef struct SInsertParseContext { - SParseContext* pComCxt; - SMsgBuf msg; - char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW]; - SParsedDataColInfo tags; // for stmt - bool missCache; - bool usingDuplicateTable; - bool forceUpdate; + SParseContext* pComCxt; + SMsgBuf msg; + char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW]; + SBoundColInfo tags; // for stmt + bool missCache; + bool usingDuplicateTable; + bool forceUpdate; } SInsertParseContext; typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param); @@ -155,7 +155,7 @@ static int32_t ignoreUsingClause(SInsertParseContext* pCxt, const char** pSql) { return code; } -static int32_t parseDuplicateUsingClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, bool* pDuplicate) { +static int32_t parseDuplicateUsingClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool* pDuplicate) { *pDuplicate = false; char tbFName[TSDB_TABLE_FNAME_LEN]; @@ -173,21 +173,19 @@ static int32_t parseDuplicateUsingClause(SInsertParseContext* pCxt, SVnodeModifO } // pStmt->pSql -> field1_name, ...) -static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, bool isTags, - SParsedDataColInfo* pColList, SSchema* pSchema) { - col_id_t nCols = pColList->numOfCols; - - pColList->numOfBound = 0; - pColList->boundNullLen = 0; - memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols); - for (col_id_t i = 0; i < nCols; ++i) { - pColList->cols[i].valStat = VAL_STAT_NONE; +static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, bool isTags, SSchema* pSchema, + SBoundColInfo* pBoundInfo) { + bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool)); + if (NULL == pUseCols) { + return TSDB_CODE_OUT_OF_MEMORY; } - SToken token; - bool isOrdered = true; - col_id_t lastColIdx = -1; // last column found - while (1) { + pBoundInfo->numOfBound = 0; + + int16_t lastColIdx = -1; // last column found + int32_t code = TSDB_CODE_SUCCESS; + while (TSDB_CODE_SUCCESS == code) { + SToken token; NEXT_TOKEN(*pSql, token); if (TK_NK_RP == token.type) { @@ -199,64 +197,30 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, b token.z = tmpTokenBuf; token.n = strdequote(token.z); - col_id_t t = lastColIdx + 1; - col_id_t index = insFindCol(&token, t, nCols, pSchema); + int16_t t = lastColIdx + 1; + int16_t index = insFindCol(&token, t, pBoundInfo->numOfCols, pSchema); if (index < 0 && t > 0) { index = insFindCol(&token, 0, t, pSchema); - isOrdered = false; } if (index < 0) { - return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMN, token.z); - } - if (pColList->cols[index].valStat == VAL_STAT_HAS) { - return buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", token.z); - } - lastColIdx = index; - pColList->cols[index].valStat = VAL_STAT_HAS; - pColList->boundColumns[pColList->numOfBound] = index; - ++pColList->numOfBound; - switch (pSchema[t].type) { - case TSDB_DATA_TYPE_BINARY: - pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + CHAR_BYTES); - break; - case TSDB_DATA_TYPE_NCHAR: - pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); - break; - default: - pColList->boundNullLen += TYPE_BYTES[pSchema[t].type]; - break; + code = generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMN, token.z); + } else if (pUseCols[index]) { + code = buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", token.z); + } else { + lastColIdx = index; + pUseCols[index] = true; + pBoundInfo->pColIndex[pBoundInfo->numOfBound] = index; + ++pBoundInfo->numOfBound; } } - if (!isTags && pColList->cols[0].valStat == VAL_STAT_NONE) { - return buildInvalidOperationMsg(&pCxt->msg, "primary timestamp column can not be null"); + if (TSDB_CODE_SUCCESS == code && !isTags && !pUseCols[0]) { + code = buildInvalidOperationMsg(&pCxt->msg, "primary timestamp column can not be null"); } - pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; - - if (!isOrdered) { - pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); - if (NULL == pColList->colIdxInfo) { - return TSDB_CODE_OUT_OF_MEMORY; - } - SBoundIdxInfo* pColIdx = pColList->colIdxInfo; - for (col_id_t i = 0; i < pColList->numOfBound; ++i) { - pColIdx[i].schemaColIdx = pColList->boundColumns[i]; - pColIdx[i].boundIdx = i; - } - taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insSchemaIdxCompar); - for (col_id_t i = 0; i < pColList->numOfBound; ++i) { - pColIdx[i].finalIdx = i; - } - taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insBoundIdxCompar); - } - - if (pColList->numOfCols > pColList->numOfBound) { - memset(&pColList->boundColumns[pColList->numOfBound], 0, - sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); - } + taosMemoryFree(pUseCols); - return TSDB_CODE_SUCCESS; + return code; } static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t* time, SMsgBuf* pMsgBuf) { @@ -518,9 +482,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, // input pStmt->pSql: [(tag1_name, ...)] TAGS (tag1_value, ...) ... // output pStmt->pSql: TAGS (tag1_value, ...) ... -static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { - SSchema* pTagsSchema = getTableTagSchema(pStmt->pTableMeta); - insSetBoundColumnInfo(&pCxt->tags, pTagsSchema, getNumOfTags(pStmt->pTableMeta)); +static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + insInitBoundColsInfo(getNumOfTags(pStmt->pTableMeta), &pCxt->tags); SToken token; int32_t index = 0; @@ -530,10 +493,10 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt } pStmt->pSql += index; - return parseBoundColumns(pCxt, &pStmt->pSql, true, &pCxt->tags, pTagsSchema); + return parseBoundColumns(pCxt, &pStmt->pSql, true, getTableTagSchema(pStmt->pTableMeta), &pCxt->tags); } -static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SSchema* pTagSchema, SToken* pToken, +static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SSchema* pTagSchema, SToken* pToken, SArray* pTagName, SArray* pTagVals, STag** pTag) { if (!isNullValue(pTagSchema->type, pToken)) { taosArrayPush(pTagName, pTagSchema->name); @@ -561,10 +524,15 @@ static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt return code; } -static void buildCreateTbReq(SVnodeModifOpStmt* pStmt, STag* pTag, SArray* pTagName) { - insBuildCreateTbReq(&pStmt->createTblReq, pStmt->targetTableName.tname, pTag, pStmt->pTableMeta->suid, +static int32_t buildCreateTbReq(SVnodeModifyOpStmt* pStmt, STag* pTag, SArray* pTagName) { + pStmt->pCreateTblReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == pStmt->pCreateTblReq) { + return TSDB_CODE_OUT_OF_MEMORY; + } + insBuildCreateTbReq(pStmt->pCreateTblReq, pStmt->targetTableName.tname, pTag, pStmt->pTableMeta->suid, pStmt->usingTableName.tname, pTagName, pStmt->pTableMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); + return TSDB_CODE_SUCCESS; } static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf) { @@ -591,7 +559,7 @@ static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMs } // pSql -> tag1_value, ...) -static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { int32_t code = TSDB_CODE_SUCCESS; SSchema* pSchema = getTableTagSchema(pStmt->pTableMeta); SArray* pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal)); @@ -618,7 +586,7 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* break; } - SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i]]; + SSchema* pTagSchema = &pSchema[pCxt->tags.pColIndex[i]]; isJson = pTagSchema->type == TSDB_DATA_TYPE_JSON; code = checkAndTrimValue(&token, pCxt->tmpTokenBuf, &pCxt->msg); if (TSDB_CODE_SUCCESS == code) { @@ -631,7 +599,7 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* } if (TSDB_CODE_SUCCESS == code && !isParseBindParam) { - buildCreateTbReq(pStmt, pTag, pTagName); + code = buildCreateTbReq(pStmt, pTag, pTagName); pTag = NULL; } @@ -649,7 +617,7 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* // input pStmt->pSql: TAGS (tag1_value, ...) [table_options] ... // output pStmt->pSql: [table_options] ... -static int32_t parseTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { SToken token; NEXT_TOKEN(pStmt->pSql, token); if (TK_TAGS != token.type) { @@ -673,7 +641,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt return code; } -static int32_t storeTableMeta(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t storeTableMeta(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { pStmt->pTableMeta->suid = pStmt->pTableMeta->uid; pStmt->pTableMeta->uid = pStmt->totalTbNum; pStmt->pTableMeta->tableType = TSDB_CHILD_TABLE; @@ -688,7 +656,7 @@ static int32_t storeTableMeta(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStm return taosHashPut(pStmt->pSubTableHashObj, tbFName, strlen(tbFName), &pBackup, POINTER_BYTES); } -static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { do { int32_t index = 0; SToken token; @@ -699,8 +667,8 @@ static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifOpStmt* p if (TK_NK_INTEGER != token.type) { return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", token.z); } - pStmt->createTblReq.ttl = taosStr2Int32(token.z, NULL, 10); - if (pStmt->createTblReq.ttl < 0) { + pStmt->pCreateTblReq->ttl = taosStr2Int32(token.z, NULL, 10); + if (pStmt->pCreateTblReq->ttl < 0) { return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", token.z); } } else if (TK_COMMENT == token.type) { @@ -713,11 +681,11 @@ static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifOpStmt* p return buildSyntaxErrMsg(&pCxt->msg, "comment too long", token.z); } int32_t len = trimString(token.z, token.n, pCxt->tmpTokenBuf, TSDB_TB_COMMENT_LEN); - pStmt->createTblReq.comment = strndup(pCxt->tmpTokenBuf, len); - if (NULL == pStmt->createTblReq.comment) { + pStmt->pCreateTblReq->comment = strndup(pCxt->tmpTokenBuf, len); + if (NULL == pStmt->pCreateTblReq->comment) { return TSDB_CODE_OUT_OF_MEMORY; } - pStmt->createTblReq.commentLen = len; + pStmt->pCreateTblReq->commentLen = len; } else { break; } @@ -731,7 +699,7 @@ static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifOpStmt* p // output pStmt->pSql: // 1. [(field1_name, ...)] // 2. VALUES ... | FILE ... -static int32_t parseUsingClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseUsingClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { if (!pStmt->usingTableProcessing || pCxt->usingDuplicateTable) { return TSDB_CODE_SUCCESS; } @@ -805,7 +773,7 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, bool isSt return code; } -static int32_t getTableVgroup(SParseContext* pCxt, SVnodeModifOpStmt* pStmt, bool isStb, bool* pMissCache) { +static int32_t getTableVgroup(SParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool isStb, bool* pMissCache) { int32_t code = TSDB_CODE_SUCCESS; SVgroupInfo vg; bool exists = true; @@ -830,7 +798,7 @@ static int32_t getTableVgroup(SParseContext* pCxt, SVnodeModifOpStmt* pStmt, boo return code; } -static int32_t getTableMetaAndVgroupImpl(SParseContext* pCxt, SVnodeModifOpStmt* pStmt, bool* pMissCache) { +static int32_t getTableMetaAndVgroupImpl(SParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool* pMissCache) { SVgroupInfo vg; int32_t code = catalogGetCachedTableVgMeta(pCxt->pCatalog, &pStmt->targetTableName, &vg, &pStmt->pTableMeta); if (TSDB_CODE_SUCCESS == code) { @@ -842,7 +810,7 @@ static int32_t getTableMetaAndVgroupImpl(SParseContext* pCxt, SVnodeModifOpStmt* return code; } -static int32_t getTableMetaAndVgroup(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, bool* pMissCache) { +static int32_t getTableMetaAndVgroup(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool* pMissCache) { SParseContext* pComCxt = pCxt->pComCxt; int32_t code = TSDB_CODE_SUCCESS; if (pComCxt->async) { @@ -868,7 +836,7 @@ static int32_t collectUseDatabase(const SName* pName, SHashObj* pDbs) { return taosHashPut(pDbs, dbFName, strlen(dbFName), dbFName, sizeof(dbFName)); } -static int32_t getTargetTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t getTargetTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { if (pCxt->forceUpdate) { pCxt->missCache = true; return TSDB_CODE_SUCCESS; @@ -887,11 +855,11 @@ static int32_t getTargetTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt return code; } -static int32_t preParseUsingTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) { +static int32_t preParseUsingTableName(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pTbName) { return insCreateSName(&pStmt->usingTableName, pTbName, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); } -static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { if (pCxt->forceUpdate) { pCxt->missCache = true; return TSDB_CODE_SUCCESS; @@ -913,7 +881,7 @@ static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt* return code; } -static int32_t parseUsingTableNameImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseUsingTableNameImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { SToken token; NEXT_TOKEN(pStmt->pSql, token); int32_t code = preParseUsingTableName(pCxt, pStmt, &token); @@ -932,7 +900,7 @@ static int32_t parseUsingTableNameImpl(SInsertParseContext* pCxt, SVnodeModifOpS // output pStmt->pSql: // 1. [(tag1_name, ...)] TAGS (tag1_value, ...) [table_options]] ... // 2. VALUES ... | FILE ... -static int32_t parseUsingTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseUsingTableName(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { SToken token; int32_t index = 0; NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index); @@ -950,7 +918,7 @@ static int32_t parseUsingTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt* return code; } -static int32_t preParseTargetTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) { +static int32_t preParseTargetTableName(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pTbName) { return insCreateSName(&pStmt->targetTableName, pTbName, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); } @@ -961,7 +929,7 @@ static int32_t preParseTargetTableName(SInsertParseContext* pCxt, SVnodeModifOpS // output pStmt->pSql: // 1. [ USING ... ] ... // 2. VALUES ... | FILE ... -static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { SToken token; int32_t index = 0; NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index); @@ -975,26 +943,22 @@ static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModif return skipParentheses(pCxt, &pStmt->pSql); } -static int32_t getTableDataBlocks(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks** pDataBuf) { +static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt** pTableCxt) { if (pCxt->pComCxt->async) { - uint64_t uid = pStmt->pTableMeta->uid; - if (pStmt->usingTableProcessing) { - pStmt->pTableMeta->uid = 0; - } - - return insGetDataBlockFromList( - pStmt->pTableBlockHashObj, &uid, sizeof(pStmt->pTableMeta->uid), TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), - getTableInfo(pStmt->pTableMeta).rowSize, pStmt->pTableMeta, pDataBuf, NULL, &pStmt->createTblReq); + return insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), + pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false); } + char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(&pStmt->targetTableName, tbFName); - return insGetDataBlockFromList(pStmt->pTableBlockHashObj, tbFName, strlen(tbFName), TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), getTableInfo(pStmt->pTableMeta).rowSize, pStmt->pTableMeta, - pDataBuf, NULL, &pStmt->createTblReq); + if (pStmt->usingTableProcessing) { + pStmt->pTableMeta->uid = 0; + } + return insGetTableDataCxt(pStmt->pTableBlockHashObj, tbFName, strlen(tbFName), pStmt->pTableMeta, + &pStmt->pCreateTblReq, pTableCxt, NULL != pCxt->pComCxt->pStmtCb); } -static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, - STableDataBlocks* pDataBuf) { +static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { SToken token; int32_t index = 0; NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index); @@ -1004,13 +968,30 @@ static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifOpS return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } // pStmt->pSql -> field1_name, ...) - return parseBoundColumns(pCxt, &pStmt->pSql, false, &pDataBuf->boundColumnInfo, - getTableColumnSchema(pStmt->pTableMeta)); + return parseBoundColumns(pCxt, &pStmt->pSql, false, getTableColumnSchema(pStmt->pTableMeta), + &pTableCxt->boundColsInfo); } if (NULL != pStmt->pBoundCols) { - return parseBoundColumns(pCxt, &pStmt->pBoundCols, false, &pDataBuf->boundColumnInfo, - getTableColumnSchema(pStmt->pTableMeta)); + return parseBoundColumns(pCxt, &pStmt->pBoundCols, false, getTableColumnSchema(pStmt->pTableMeta), + &pTableCxt->boundColsInfo); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t initTableColSubmitData(STableDataCxt* pTableCxt) { + if (0 == (pTableCxt->pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT)) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < pTableCxt->boundColsInfo.numOfBound; ++i) { + SSchema* pSchema = &pTableCxt->pMeta->schema[pTableCxt->boundColsInfo.pColIndex[i]]; + SColData* pCol = taosArrayReserve(pTableCxt->pData->aCol, 1); + if (NULL == pCol) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tColDataInit(pCol, pSchema->colId, pSchema->type, 0); } return TSDB_CODE_SUCCESS; @@ -1020,14 +1001,17 @@ static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifOpS // 1. [(tag1_name, ...)] ... // 2. VALUES ... | FILE ... // output pStmt->pSql: VALUES ... | FILE ... -static int32_t parseSchemaClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, - STableDataBlocks** pDataBuf) { +static int32_t parseSchemaClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, + STableDataCxt** pTableCxt) { int32_t code = parseUsingClauseBottom(pCxt, pStmt); if (TSDB_CODE_SUCCESS == code) { - code = getTableDataBlocks(pCxt, pStmt, pDataBuf); + code = getTableDataCxt(pCxt, pStmt, pTableCxt); } if (TSDB_CODE_SUCCESS == code) { - code = parseBoundColumnsClause(pCxt, pStmt, *pDataBuf); + code = parseBoundColumnsClause(pCxt, pStmt, *pTableCxt); + } + if (TSDB_CODE_SUCCESS == code) { + code = initTableColSubmitData(*pTableCxt); } return code; } @@ -1036,7 +1020,7 @@ static int32_t parseSchemaClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpS // output pStmt->pSql: // 1. [(tag1_name, ...)] ... // 2. VALUES ... | FILE ... -static int32_t parseSchemaClauseTop(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) { +static int32_t parseSchemaClauseTop(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pTbName) { int32_t code = preParseTargetTableName(pCxt, pStmt, pTbName); if (TSDB_CODE_SUCCESS == code) { // option: [(field1_name, ...)] @@ -1050,108 +1034,88 @@ static int32_t parseSchemaClauseTop(SInsertParseContext* pCxt, SVnodeModifOpStmt } static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, SToken* pToken, SSchema* pSchema, - int16_t timePrec, _row_append_fn_t func, void* param) { - int64_t iv; - uint64_t uv; - char* endptr = NULL; - + int16_t timePrec, SColVal* pVal) { switch (pSchema->type) { case TSDB_DATA_TYPE_BOOL: { if ((pToken->type == TK_NK_BOOL || pToken->type == TK_NK_STRING) && (pToken->n != 0)) { if (strncmp(pToken->z, "true", pToken->n) == 0) { - return func(&pCxt->msg, &TRUE_VALUE, pSchema->bytes, param); + pVal->value.val = TRUE_VALUE; } else if (strncmp(pToken->z, "false", pToken->n) == 0) { - return func(&pCxt->msg, &FALSE_VALUE, pSchema->bytes, param); + pVal->value.val = FALSE_VALUE; } else { return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); } } else if (pToken->type == TK_NK_INTEGER) { - return func(&pCxt->msg, ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), - pSchema->bytes, param); + pVal->value.val = ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? FALSE_VALUE : TRUE_VALUE); } else if (pToken->type == TK_NK_FLOAT) { - return func(&pCxt->msg, ((taosStr2Double(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, - param); + pVal->value.val = ((taosStr2Double(pToken->z, NULL) == 0) ? FALSE_VALUE : TRUE_VALUE); } else { return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); } + break; } - case TSDB_DATA_TYPE_TINYINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z); - } else if (!IS_VALID_TINYINT(iv)) { + } else if (!IS_VALID_TINYINT(pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "tinyint data overflow", pToken->z); } - - uint8_t tmpVal = (uint8_t)iv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_UTINYINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z); - } else if (uv > UINT8_MAX) { + } else if (pVal->value.val > UINT8_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned tinyint data overflow", pToken->z); } - uint8_t tmpVal = (uint8_t)uv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_SMALLINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z); - } else if (!IS_VALID_SMALLINT(iv)) { + } else if (!IS_VALID_SMALLINT(pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "smallint data overflow", pToken->z); } - int16_t tmpVal = (int16_t)iv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_USMALLINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z); - } else if (uv > UINT16_MAX) { + } else if (pVal->value.val > UINT16_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned smallint data overflow", pToken->z); } - uint16_t tmpVal = (uint16_t)uv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_INT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z); - } else if (!IS_VALID_INT(iv)) { + } else if (!IS_VALID_INT(pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "int data overflow", pToken->z); } - int32_t tmpVal = (int32_t)iv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_UINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); - } else if (uv > UINT32_MAX) { + } else if (pVal->value.val > UINT32_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned int data overflow", pToken->z); } - uint32_t tmpVal = (uint32_t)uv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_BIGINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z); } - return func(&pCxt->msg, &iv, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_UBIGINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z); } - return func(&pCxt->msg, &uv, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_FLOAT: { + char* endptr = NULL; double dv; if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); @@ -1160,11 +1124,12 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, isnan(dv)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } - float tmpVal = (float)dv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + float f = dv; + memcpy(&pVal->value.val, &f, sizeof(f)); + break; } - case TSDB_DATA_TYPE_DOUBLE: { + char* endptr = NULL; double dv; if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); @@ -1172,49 +1137,77 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); } - return func(&pCxt->msg, &dv, pSchema->bytes, param); + pVal->value.val = *(int64_t*)&dv; + break; } - case TSDB_DATA_TYPE_BINARY: { // Too long values will raise the invalid sql error message if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } - - return func(&pCxt->msg, pToken->z, pToken->n, param); + pVal->value.pData = taosMemoryMalloc(pToken->n); + if (NULL == pVal->value.pData) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(pVal->value.pData, pToken->z, pToken->n); + pVal->value.nData = pToken->n; + break; } - case TSDB_DATA_TYPE_NCHAR: { - return func(&pCxt->msg, pToken->z, pToken->n, param); + // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' + int32_t len = 0; + char* pUcs4 = taosMemoryCalloc(1, pSchema->bytes - VARSTR_HEADER_SIZE); + if (NULL == pUcs4) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)pUcs4, pSchema->bytes - VARSTR_HEADER_SIZE, &len)) { + taosMemoryFree(pUcs4); + if (errno == E2BIG) { + return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + } + char buf[512] = {0}; + snprintf(buf, tListLen(buf), "%s", strerror(errno)); + return buildSyntaxErrMsg(&pCxt->msg, buf, pToken->z); + } + pVal->value.pData = pUcs4; + pVal->value.nData = len; + break; } case TSDB_DATA_TYPE_JSON: { if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { return buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", pToken->z); } - return func(&pCxt->msg, pToken->z, pToken->n, param); + pVal->value.pData = taosMemoryMalloc(pToken->n); + if (NULL == pVal->value.pData) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(pVal->value.pData, pToken->z, pToken->n); + pVal->value.nData = pToken->n; + break; } case TSDB_DATA_TYPE_TIMESTAMP: { - int64_t tmpVal; - if (parseTime(pSql, pToken, timePrec, &tmpVal, &pCxt->msg) != TSDB_CODE_SUCCESS) { + if (parseTime(pSql, pToken, timePrec, &pVal->value.val, &pCxt->msg) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp", pToken->z); } - - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } + default: + return TSDB_CODE_FAILED; } - return TSDB_CODE_FAILED; + pVal->flag = CV_FLAG_VALUE; + return TSDB_CODE_SUCCESS; } static int32_t parseValueToken(SInsertParseContext* pCxt, const char** pSql, SToken* pToken, SSchema* pSchema, - int16_t timePrec, _row_append_fn_t func, void* param) { + int16_t timePrec, SColVal* pVal) { int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg); if (TSDB_CODE_SUCCESS == code && isNullValue(pSchema->type, pToken)) { if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { return buildSyntaxErrMsg(&pCxt->msg, "primary timestamp should not be null", pToken->z); } - - return func(&pCxt->msg, NULL, 0, param); + pVal->flag = CV_FLAG_NULL; + return TSDB_CODE_SUCCESS; } if (TSDB_CODE_SUCCESS == code && IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { @@ -1222,26 +1215,34 @@ static int32_t parseValueToken(SInsertParseContext* pCxt, const char** pSql, STo } if (TSDB_CODE_SUCCESS == code) { - code = parseValueTokenImpl(pCxt, pSql, pToken, pSchema, timePrec, func, param); + code = parseValueTokenImpl(pCxt, pSql, pToken, pSchema, timePrec, pVal); } return code; } -static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataBlocks* pDataBuf, bool* pGotRow, +static void clearColValArray(SArray* pCols) { + int32_t num = taosArrayGetSize(pCols); + for (int32_t i = 0; i < num; ++i) { + SColVal* pCol = taosArrayGet(pCols, i); + if (IS_VAR_DATA_TYPE(pCol->type)) { + taosMemoryFreeClear(pCol->value.pData); + } + } +} + +static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataCxt* pTableCxt, bool* pGotRow, SToken* pToken) { - SRowBuilder* pBuilder = &pDataBuf->rowBuilder; - STSRow* row = (STSRow*)(pDataBuf->pData + pDataBuf->size); // skip the SSubmitBlk header - SParsedDataColInfo* pCols = &pDataBuf->boundColumnInfo; - bool isParseBindParam = false; - SSchema* pSchemas = getTableColumnSchema(pDataBuf->pTableMeta); - SMemParam param = {.rb = pBuilder}; - - int32_t code = tdSRowResetBuf(pBuilder, row); + SBoundColInfo* pCols = &pTableCxt->boundColsInfo; + bool isParseBindParam = false; + SSchema* pSchemas = getTableColumnSchema(pTableCxt->pMeta); + + int32_t code = TSDB_CODE_SUCCESS; // 1. set the parsed value from sql string for (int i = 0; i < pCols->numOfBound && TSDB_CODE_SUCCESS == code; ++i) { NEXT_TOKEN_WITH_PREV(*pSql, *pToken); - SSchema* pSchema = &pSchemas[pCols->boundColumns[i]]; + SSchema* pSchema = &pSchemas[pCols->pColIndex[i]]; + SColVal* pVal = taosArrayGet(pTableCxt->pValues, pCols->pColIndex[i]); if (pToken->type == TK_NK_QUESTION) { isParseBindParam = true; @@ -1260,10 +1261,7 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataB } if (TSDB_CODE_SUCCESS == code) { - param.schema = pSchema; - insGetSTSRowAppendInfo(pBuilder->rowType, pCols, i, ¶m.toffset, ¶m.colIdx); - code = parseValueToken(pCxt, pSql, pToken, pSchema, getTableInfo(pDataBuf->pTableMeta).precision, insMemRowAppend, - ¶m); + code = parseValueToken(pCxt, pSql, pToken, pSchema, getTableInfo(pTableCxt->pMeta).precision, pVal); } if (TSDB_CODE_SUCCESS == code && i < pCols->numOfBound - 1) { @@ -1274,65 +1272,28 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataB } } - if (TSDB_CODE_SUCCESS == code) { - TSKEY tsKey = TD_ROW_KEY(row); - code = insCheckTimestamp(pDataBuf, (const char*)&tsKey); - } - if (TSDB_CODE_SUCCESS == code && !isParseBindParam) { - // set the null value for the columns that do not assign values - if ((pCols->numOfBound < pCols->numOfCols) && TD_IS_TP_ROW(row)) { - pBuilder->hasNone = true; + SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + code = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); + if (TSDB_CODE_SUCCESS == code) { + insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow)); } + } - tdSRowEnd(pBuilder); - + if (TSDB_CODE_SUCCESS == code && !isParseBindParam) { *pGotRow = true; - -#ifdef TD_DEBUG_PRINT_ROW - STSchema* pSTSchema = tdGetSTSChemaFromSSChema(schema, spd->numOfCols, 1); - tdSRowPrint(row, pSTSchema, __func__); - taosMemoryFree(pSTSchema); -#endif } - return code; -} + clearColValArray(pTableCxt->pValues); -static int32_t allocateMemIfNeed(STableDataBlocks* pDataBlock, int32_t rowSize, int32_t* numOfRows) { - size_t remain = pDataBlock->nAllocSize - pDataBlock->size; - const int factor = 5; - uint32_t nAllocSizeOld = pDataBlock->nAllocSize; - - // expand the allocated size - if (remain < rowSize * factor) { - while (remain < rowSize * factor) { - pDataBlock->nAllocSize = (uint32_t)(pDataBlock->nAllocSize * 1.5); - remain = pDataBlock->nAllocSize - pDataBlock->size; - } - - char* tmp = taosMemoryRealloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize); - if (tmp != NULL) { - pDataBlock->pData = tmp; - memset(pDataBlock->pData + pDataBlock->size, 0, pDataBlock->nAllocSize - pDataBlock->size); - } else { - // do nothing, if allocate more memory failed - pDataBlock->nAllocSize = nAllocSizeOld; - *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize; - return TSDB_CODE_OUT_OF_MEMORY; - } - } - - *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize; - return TSDB_CODE_SUCCESS; + return code; } // pSql -> (field1_value, ...) [(field1_value2, ...) ...] -static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf, - int32_t maxRows, int32_t* pNumOfRows, SToken* pToken) { - int32_t code = insInitRowBuilder(&pDataBuf->rowBuilder, pDataBuf->pTableMeta->sversion, &pDataBuf->boundColumnInfo); +static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, + int32_t* pNumOfRows, SToken* pToken) { + int32_t code = TSDB_CODE_SUCCESS; - int32_t extendedRowSize = insGetExtendedRowSize(pDataBuf); (*pNumOfRows) = 0; while (TSDB_CODE_SUCCESS == code) { int32_t index = 0; @@ -1342,13 +1303,9 @@ static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, } pStmt->pSql += index; - if ((*pNumOfRows) >= maxRows || pDataBuf->size + extendedRowSize >= pDataBuf->nAllocSize) { - code = allocateMemIfNeed(pDataBuf, extendedRowSize, &maxRows); - } - bool gotRow = false; if (TSDB_CODE_SUCCESS == code) { - code = parseOneRow(pCxt, &pStmt->pSql, pDataBuf, &gotRow, pToken); + code = parseOneRow(pCxt, &pStmt->pSql, pTableCxt, &gotRow, pToken); } if (TSDB_CODE_SUCCESS == code) { @@ -1361,7 +1318,6 @@ static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, } if (TSDB_CODE_SUCCESS == code && gotRow) { - pDataBuf->size += extendedRowSize; (*pNumOfRows)++; } } @@ -1374,19 +1330,11 @@ static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, } // VALUES (field1_value, ...) [(field1_value2, ...) ...] -static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf, +static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, SToken* pToken) { - int32_t maxNumOfRows = 0; int32_t numOfRows = 0; - int32_t code = allocateMemIfNeed(pDataBuf, insGetExtendedRowSize(pDataBuf), &maxNumOfRows); + int32_t code = parseValues(pCxt, pStmt, pTableCxt, &numOfRows, pToken); if (TSDB_CODE_SUCCESS == code) { - code = parseValues(pCxt, pStmt, pDataBuf, maxNumOfRows, &numOfRows, pToken); - } - if (TSDB_CODE_SUCCESS == code) { - code = insSetBlockInfo((SSubmitBlk*)(pDataBuf->pData), pDataBuf, numOfRows, &pCxt->msg); - } - if (TSDB_CODE_SUCCESS == code) { - pDataBuf->numOfTables = 1; pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_INSERT); @@ -1394,11 +1342,9 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* p return code; } -static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf, - int maxRows, int32_t* pNumOfRows) { - int32_t code = insInitRowBuilder(&pDataBuf->rowBuilder, pDataBuf->pTableMeta->sversion, &pDataBuf->boundColumnInfo); - - int32_t extendedRowSize = insGetExtendedRowSize(pDataBuf); +static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, + int32_t* pNumOfRows) { + int32_t code = TSDB_CODE_SUCCESS; (*pNumOfRows) = 0; char* pLine = NULL; int64_t readLen = 0; @@ -1414,16 +1360,13 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, continue; } - if ((*pNumOfRows) >= maxRows || pDataBuf->size + extendedRowSize >= pDataBuf->nAllocSize) { - code = allocateMemIfNeed(pDataBuf, extendedRowSize, &maxRows); - } - bool gotRow = false; if (TSDB_CODE_SUCCESS == code) { SToken token; strtolower(pLine, pLine); const char* pRow = pLine; - code = parseOneRow(pCxt, (const char**)&pRow, pDataBuf, &gotRow, &token); + + code = parseOneRow(pCxt, (const char**)&pRow, pTableCxt, &gotRow, &token); if (code && firstLine) { firstLine = false; code = 0; @@ -1432,11 +1375,10 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, } if (TSDB_CODE_SUCCESS == code && gotRow) { - pDataBuf->size += extendedRowSize; (*pNumOfRows)++; } - if (TSDB_CODE_SUCCESS == code && pDataBuf->nAllocSize > tsMaxMemUsedByInsert * 1024 * 1024) { + if (TSDB_CODE_SUCCESS == code && (*pNumOfRows) > tsMaxMemUsedByInsert * 1024 * 1024) { pStmt->fileProcessing = true; break; } @@ -1452,18 +1394,10 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, return code; } -static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf) { - int32_t maxNumOfRows = 0; +static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { int32_t numOfRows = 0; - int32_t code = allocateMemIfNeed(pDataBuf, insGetExtendedRowSize(pDataBuf), &maxNumOfRows); + int32_t code = parseCsvFile(pCxt, pStmt, pTableCxt, &numOfRows); if (TSDB_CODE_SUCCESS == code) { - code = parseCsvFile(pCxt, pStmt, pDataBuf, maxNumOfRows, &numOfRows); - } - if (TSDB_CODE_SUCCESS == code) { - code = insSetBlockInfo((SSubmitBlk*)(pDataBuf->pData), pDataBuf, numOfRows, &pCxt->msg); - } - if (TSDB_CODE_SUCCESS == code) { - pDataBuf->numOfTables = 1; pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_FILE_INSERT); @@ -1476,8 +1410,8 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifOpStm return code; } -static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pFilePath, - STableDataBlocks* pDataBuf) { +static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pFilePath, + STableDataCxt* pTableCxt) { char filePathStr[TSDB_FILENAME_LEN] = {0}; if (TK_NK_STRING == pFilePath->type) { trimString(pFilePath->z, pFilePath->n, filePathStr, sizeof(filePathStr)); @@ -1489,10 +1423,10 @@ static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* p return TAOS_SYSTEM_ERROR(errno); } - return parseDataFromFileImpl(pCxt, pStmt, pDataBuf); + return parseDataFromFileImpl(pCxt, pStmt, pTableCxt); } -static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf, +static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, SToken* pToken) { if (tsUseAdapter) { return buildInvalidOperationMsg(&pCxt->msg, "proxy mode does not support csv loading"); @@ -1502,18 +1436,18 @@ static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt if (0 == pToken->n || (TK_NK_STRING != pToken->type && TK_NK_ID != pToken->type)) { return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", pToken->z); } - return parseDataFromFile(pCxt, pStmt, pToken, pDataBuf); + return parseDataFromFile(pCxt, pStmt, pToken, pTableCxt); } // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path -static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf) { +static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { SToken token; NEXT_TOKEN(pStmt->pSql, token); switch (token.type) { case TK_VALUES: - return parseValuesClause(pCxt, pStmt, pDataBuf, &token); + return parseValuesClause(pCxt, pStmt, pTableCxt, &token); case TK_FILE: - return parseFileClause(pCxt, pStmt, pDataBuf, &token); + return parseFileClause(pCxt, pStmt, pTableCxt, &token); default: break; } @@ -1523,19 +1457,20 @@ static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt // input pStmt->pSql: // 1. [(tag1_name, ...)] ... // 2. VALUES ... | FILE ... -static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { - STableDataBlocks* pDataBuf = NULL; - int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pDataBuf); +static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + STableDataCxt* pTableCxt = NULL; + int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pTableCxt); if (TSDB_CODE_SUCCESS == code) { - code = parseDataClause(pCxt, pStmt, pDataBuf); + code = parseDataClause(pCxt, pStmt, pTableCxt); } return code; } -static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { - destroyBoundColumnInfo(&pCxt->tags); +static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + insDestroyBoundColInfo(&pCxt->tags); taosMemoryFreeClear(pStmt->pTableMeta); - tdDestroySVCreateTbReq(&pStmt->createTblReq); + tdDestroySVCreateTbReq(pStmt->pCreateTblReq); + taosMemoryFreeClear(pStmt->pCreateTblReq); pCxt->missCache = false; pCxt->usingDuplicateTable = false; pStmt->pBoundCols = NULL; @@ -1544,7 +1479,7 @@ static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt } // input pStmt->pSql: [(field1_name, ...)] [ USING ... ] VALUES ... | FILE ... -static int32_t parseInsertTableClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) { +static int32_t parseInsertTableClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pTbName) { resetEnvPreTable(pCxt, pStmt); int32_t code = parseSchemaClauseTop(pCxt, pStmt, pTbName); if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { @@ -1553,7 +1488,7 @@ static int32_t parseInsertTableClause(SInsertParseContext* pCxt, SVnodeModifOpSt return code; } -static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName, +static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pTbName, bool* pHasData) { // no data in the sql string anymore. if (0 == pTbName->n) { @@ -1592,8 +1527,8 @@ static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModif return TSDB_CODE_SUCCESS; } -static int32_t setStmtInfo(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { - SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); +static int32_t setStmtInfo(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + SBoundColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); if (NULL == tags) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1610,19 +1545,17 @@ static int32_t setStmtInfo(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) return code; } -static int32_t parseInsertBodyBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseInsertBodyBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { if (TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { return setStmtInfo(pCxt, pStmt); } // merge according to vgId - int32_t code = TSDB_CODE_SUCCESS; - if (taosHashGetSize(pStmt->pTableBlockHashObj) > 0) { - code = insMergeTableDataBlocks(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks); - } + int32_t code = insMergeTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks); if (TSDB_CODE_SUCCESS == code) { - code = insBuildOutput(pStmt->pVgroupsHashObj, pStmt->pVgDataBlocks, &pStmt->pDataBlocks); + code = insBuildVgDataBlocks(pStmt->pVgroupsHashObj, pStmt->pVgDataBlocks, &pStmt->pDataBlocks); } + return code; } @@ -1631,7 +1564,7 @@ static int32_t parseInsertBodyBottom(SInsertParseContext* pCxt, SVnodeModifOpStm // [(field1_name, ...)] // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path // [...]; -static int32_t parseInsertBody(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseInsertBody(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { SToken token; int32_t code = TSDB_CODE_SUCCESS; bool hasData = true; @@ -1654,7 +1587,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt static void destroySubTableHashElem(void* p) { taosMemoryFree(*(STableMeta**)p); } static int32_t createVnodeModifOpStmt(SInsertParseContext* pCxt, bool reentry, SNode** pOutput) { - SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT); if (NULL == pStmt) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1663,8 +1596,8 @@ static int32_t createVnodeModifOpStmt(SInsertParseContext* pCxt, bool reentry, S TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT); } pStmt->pSql = pCxt->pComCxt->pSql; - pStmt->freeHashFunc = insDestroyBlockHashmap; - pStmt->freeArrayFunc = insDestroyBlockArrayList; + pStmt->freeHashFunc = insDestroyTableDataCxtHashMap; + pStmt->freeArrayFunc = insDestroyVgroupDataCxtList; if (!reentry) { pStmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); @@ -1733,7 +1666,7 @@ static int32_t getTableMetaFromMetaData(const SArray* pTables, STableMeta** pMet return pRes->code; } -static int32_t getTableVgroupFromMetaData(const SArray* pTables, SVnodeModifOpStmt* pStmt, bool isStb) { +static int32_t getTableVgroupFromMetaData(const SArray* pTables, SVnodeModifyOpStmt* pStmt, bool isStb) { if (1 != taosArrayGetSize(pTables)) { return TSDB_CODE_FAILED; } @@ -1752,7 +1685,7 @@ static int32_t getTableVgroupFromMetaData(const SArray* pTables, SVnodeModifOpSt } static int32_t getTableSchemaFromMetaData(SInsertParseContext* pCxt, const SMetaData* pMetaData, - SVnodeModifOpStmt* pStmt, bool isStb) { + SVnodeModifyOpStmt* pStmt, bool isStb) { int32_t code = checkAuthFromMetaData(pMetaData->pUser); if (TSDB_CODE_SUCCESS == code) { code = getTableMetaFromMetaData(pMetaData->pTableMeta, &pStmt->pTableMeta); @@ -1785,7 +1718,7 @@ static void clearCatalogReq(SCatalogReq* pCatalogReq) { } static int32_t setVnodeModifOpStmt(SInsertParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData, - SVnodeModifOpStmt* pStmt) { + SVnodeModifyOpStmt* pStmt) { clearCatalogReq(pCatalogReq); if (pStmt->usingTableProcessing) { @@ -1799,7 +1732,7 @@ static int32_t resetVnodeModifOpStmt(SInsertParseContext* pCxt, SQuery* pQuery) int32_t code = createVnodeModifOpStmt(pCxt, true, &pQuery->pRoot); if (TSDB_CODE_SUCCESS == code) { - SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot; + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; (*pCxt->pComCxt->pStmtCb->getExecInfoFn)(pCxt->pComCxt->pStmtCb->pStmt, &pStmt->pVgroupsHashObj, &pStmt->pTableBlockHashObj); @@ -1828,7 +1761,7 @@ static int32_t initInsertQuery(SInsertParseContext* pCxt, SCatalogReq* pCatalogR return resetVnodeModifOpStmt(pCxt, *pQuery); } - SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)(*pQuery)->pRoot; + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)(*pQuery)->pRoot; if (!pStmt->fileProcessing) { return setVnodeModifOpStmt(pCxt, pCatalogReq, pMetaData, pStmt); @@ -1838,7 +1771,7 @@ static int32_t initInsertQuery(SInsertParseContext* pCxt, SCatalogReq* pCatalogR } static int32_t setRefreshMate(SQuery* pQuery) { - SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot; + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; if (taosHashGetSize(pStmt->pTableNameHashObj) > 0) { taosArrayDestroy(pQuery->pTableList); @@ -1869,7 +1802,7 @@ static int32_t setRefreshMate(SQuery* pQuery) { // [(field1_name, ...)] // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path // [...]; -static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { int32_t code = skipInsertInto(&pStmt->pSql, &pCxt->msg); if (TSDB_CODE_SUCCESS == code) { code = parseInsertBody(pCxt, pStmt); @@ -1877,11 +1810,11 @@ static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifOpS return code; } -static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { - STableDataBlocks* pDataBuf = NULL; - int32_t code = getTableDataBlocks(pCxt, pStmt, &pDataBuf); +static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + STableDataCxt* pTableCxt = NULL; + int32_t code = getTableDataCxt(pCxt, pStmt, &pTableCxt); if (TSDB_CODE_SUCCESS == code) { - code = parseDataFromFileImpl(pCxt, pStmt, pDataBuf); + code = parseDataFromFileImpl(pCxt, pStmt, pTableCxt); } if (TSDB_CODE_SUCCESS == code) { @@ -1895,7 +1828,7 @@ static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifOpStm return code; } -static int32_t parseInsertSqlFromTable(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseInsertSqlFromTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { int32_t code = parseInsertTableClauseBottom(pCxt, pStmt); if (TSDB_CODE_SUCCESS == code) { code = parseInsertBody(pCxt, pStmt); @@ -1903,7 +1836,7 @@ static int32_t parseInsertSqlFromTable(SInsertParseContext* pCxt, SVnodeModifOpS return code; } -static int32_t parseInsertSqlImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseInsertSqlImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { if (pStmt->pSql == pCxt->pComCxt->pSql || NULL != pCxt->pComCxt->pStmtCb) { return parseInsertSqlFromStart(pCxt, pStmt); } @@ -1955,7 +1888,7 @@ static int32_t buildInsertUserAuthReq(const char* pUser, SName* pName, SArray** return TSDB_CODE_SUCCESS; } -static int32_t buildInsertCatalogReq(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SCatalogReq* pCatalogReq) { +static int32_t buildInsertCatalogReq(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SCatalogReq* pCatalogReq) { int32_t code = buildInsertUserAuthReq(pCxt->pComCxt->pUser, &pStmt->targetTableName, &pCatalogReq->pUser); if (TSDB_CODE_SUCCESS == code) { if (0 == pStmt->usingTableName.type) { @@ -1971,7 +1904,7 @@ static int32_t buildInsertCatalogReq(SInsertParseContext* pCxt, SVnodeModifOpStm } static int32_t setNextStageInfo(SInsertParseContext* pCxt, SQuery* pQuery, SCatalogReq* pCatalogReq) { - SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot; + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; if (pCxt->missCache) { parserDebug("0x%" PRIx64 " %d rows of %d tables have been inserted before cache miss", pCxt->pComCxt->requestId, pStmt->totalRowsNum, pStmt->totalTbNum); @@ -1996,7 +1929,7 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal int32_t code = initInsertQuery(&context, pCatalogReq, pMetaData, pQuery); if (TSDB_CODE_SUCCESS == code) { - code = parseInsertSqlImpl(&context, (SVnodeModifOpStmt*)(*pQuery)->pRoot); + code = parseInsertSqlImpl(&context, (SVnodeModifyOpStmt*)(*pQuery)->pRoot); } if (TSDB_CODE_SUCCESS == code) { code = setNextStageInfo(&context, *pQuery, pCatalogReq); @@ -2005,6 +1938,6 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal QUERY_EXEC_STAGE_SCHEDULE == (*pQuery)->execStage) { code = setRefreshMate(*pQuery); } - destroyBoundColumnInfo(&context.tags); + insDestroyBoundColInfo(&context.tags); return code; } diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 4ed72e6c14622872d2535f17ec0f407135594392..01a635e4b2d34bbb7982d45400429ac323e9547e 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -29,26 +29,53 @@ typedef struct SKvParam { char buf[TSDB_MAX_TAGS_LEN]; } SKvParam; +int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData** pData) { + *pData = taosMemoryCalloc(1, sizeof(SSubmitTbData)); + if (NULL == *pData) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SSubmitTbData* pNew = *pData; + + *pNew = *pDataBlock->pData; + + cloneSVreateTbReq(pDataBlock->pData->pCreateTbReq, &pNew->pCreateTbReq); + pNew->aCol = taosArrayDup(pDataBlock->pData->aCol, NULL); + + int32_t colNum = taosArrayGetSize(pNew->aCol); + for (int32_t i = 0; i < colNum; ++i) { + SColData* pCol = (SColData*)taosArrayGet(pNew->aCol, i); + tColDataDeepClear(pCol); + } + + return TSDB_CODE_SUCCESS; +} + int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash) { - int32_t code = TSDB_CODE_SUCCESS; - SArray* pVgDataBlocks = NULL; + int32_t code = TSDB_CODE_SUCCESS; + SArray* pVgDataBlocks = NULL; + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; + // merge according to vgId if (taosHashGetSize(pBlockHash) > 0) { - code = insMergeTableDataBlocks(pBlockHash, &pVgDataBlocks); + code = insMergeTableDataCxt(pBlockHash, &pVgDataBlocks); } if (TSDB_CODE_SUCCESS == code) { - code = insBuildOutput(pVgHash, pVgDataBlocks, &((SVnodeModifOpStmt*)pQuery->pRoot)->pDataBlocks); + code = insBuildVgDataBlocks(pVgHash, pVgDataBlocks, &pStmt->pDataBlocks); + } + + if (pStmt->freeArrayFunc) { + pStmt->freeArrayFunc(pVgDataBlocks); } - insDestroyBlockArrayList(pVgDataBlocks); return code; } int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; - int32_t code = TSDB_CODE_SUCCESS; - SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + int32_t code = TSDB_CODE_SUCCESS; + SBoundColInfo* tags = (SBoundColInfo*)boundTags; if (NULL == tags) { return TSDB_CODE_APP_ERROR; } @@ -64,7 +91,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch goto end; } - SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); + SSchema* pSchema = getTableTagSchema(pDataBlock->pMeta); bool isJson = false; STag* pTag = NULL; @@ -74,7 +101,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch continue; } - SSchema* pTagSchema = &pSchema[tags->boundColumns[c]]; + SSchema* pTagSchema = &pSchema[tags->pColIndex[c]]; int32_t colLen = pTagSchema->bytes; if (IS_VAR_DATA_TYPE(pTagSchema->type)) { colLen = bind[c].length[0]; @@ -136,10 +163,16 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch goto end; } - SVCreateTbReq tbReq = {0}; - insBuildCreateTbReq(&tbReq, tName, pTag, suid, sTableName, tagName, pDataBlock->pTableMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); - code = insBuildCreateTbMsg(pDataBlock, &tbReq); - tdDestroySVCreateTbReq(&tbReq); + if (NULL == pDataBlock->pData->pCreateTbReq) { + pDataBlock->pData->pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == pDataBlock->pData->pCreateTbReq) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + } + + insBuildCreateTbReq(pDataBlock->pData->pCreateTbReq, tName, pTag, suid, sTableName, tagName, + pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); end: for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) { @@ -154,199 +187,178 @@ end: return code; } -int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - int32_t extendedRowSize = insGetExtendedRowSize(pDataBlock); - SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; - SRowBuilder* pBuilder = &pDataBlock->rowBuilder; - SMemParam param = {.rb = pBuilder}; - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; - int32_t rowNum = bind->num; - - CHECK_CODE( - insInitRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo)); +int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND* src, TAOS_MULTI_BIND* dst) { + int32_t output = 0; + int32_t newBuflen = (pSchema->bytes - VARSTR_HEADER_SIZE) * src->num; + if (dst->buffer_length < newBuflen) { + dst->buffer = taosMemoryRealloc(dst->buffer, newBuflen); + if (NULL == dst->buffer) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } - CHECK_CODE(insAllocateMemForSize(pDataBlock, extendedRowSize * bind->num)); + if (NULL == dst->length) { + dst->length = taosMemoryRealloc(dst->length, sizeof(int32_t) * src->num); + if (NULL == dst->buffer) { + taosMemoryFreeClear(dst->buffer); + return TSDB_CODE_OUT_OF_MEMORY; + } + } - for (int32_t r = 0; r < bind->num; ++r) { - STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header - tdSRowResetBuf(pBuilder, row); + dst->buffer_length = pSchema->bytes - VARSTR_HEADER_SIZE; - for (int c = 0; c < spd->numOfBound; ++c) { - SSchema* pColSchema = &pSchema[spd->boundColumns[c]]; + for (int32_t i = 0; i < src->num; ++i) { + if (src->is_null && src->is_null[i]) { + continue; + } - if (bind[c].num != rowNum) { - return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); + if (!taosMbsToUcs4(((char*)src->buffer) + src->buffer_length * i, src->length[i], + (TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output)) { + if (errno == E2BIG) { + return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } + char buf[512] = {0}; + snprintf(buf, tListLen(buf), "%s", strerror(errno)); + return buildSyntaxErrMsg(pMsgBuf, buf, NULL); + } - param.schema = pColSchema; - insGetSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); - - if (bind[c].is_null && bind[c].is_null[r]) { - if (pColSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - return buildInvalidOperationMsg(&pBuf, "primary timestamp should not be NULL"); - } - - CHECK_CODE(insMemRowAppend(&pBuf, NULL, 0, ¶m)); - } else { - if (bind[c].buffer_type != pColSchema->type) { - return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); - } + dst->length[i] = output; + } - int32_t colLen = pColSchema->bytes; - if (IS_VAR_DATA_TYPE(pColSchema->type)) { - colLen = bind[c].length[r]; - } + dst->buffer_type = src->buffer_type; + dst->is_null = src->is_null; + dst->num = src->num; - CHECK_CODE(insMemRowAppend(&pBuf, (char*)bind[c].buffer + bind[c].buffer_length * r, colLen, ¶m)); - } + return TSDB_CODE_SUCCESS; +} - if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { - TSKEY tsKey = TD_ROW_KEY(row); - insCheckTimestamp(pDataBlock, (const char*)&tsKey); - } - } - // set the null value for the columns that do not assign values - if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - pBuilder->hasNone = true; +int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); + SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + int32_t rowNum = bind->num; + TAOS_MULTI_BIND ncharBind = {0}; + TAOS_MULTI_BIND* pBind = NULL; + int32_t code = 0; + + for (int c = 0; c < boundInfo->numOfBound; ++c) { + SSchema* pColSchema = &pSchema[boundInfo->pColIndex[c]]; + SColData* pCol = taosArrayGet(pDataBlock->pData->aCol, c); + + if (bind[c].num != rowNum) { + code = buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); + goto _return; } - tdSRowEnd(pBuilder); -#ifdef TD_DEBUG_PRINT_ROW - STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1); - tdSRowPrint(row, pSTSchema, __func__); - taosMemoryFree(pSTSchema); -#endif - pDataBlock->size += extendedRowSize; - } - SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); - return insSetBlockInfo(pBlocks, pDataBlock, bind->num, &pBuf); -} + if (bind[c].buffer_type != pColSchema->type) { + code = buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); + goto _return; + } -int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, - int32_t rowNum) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - int32_t extendedRowSize = insGetExtendedRowSize(pDataBlock); - SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; - SRowBuilder* pBuilder = &pDataBlock->rowBuilder; - SMemParam param = {.rb = pBuilder}; - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; - bool rowStart = (0 == colIdx); - bool rowEnd = ((colIdx + 1) == spd->numOfBound); - - if (rowStart) { - CHECK_CODE( - insInitRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo)); - CHECK_CODE(insAllocateMemForSize(pDataBlock, extendedRowSize * bind->num)); - } - - for (int32_t r = 0; r < bind->num; ++r) { - STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size + extendedRowSize * r); // skip the SSubmitBlk header - if (rowStart) { - tdSRowResetBuf(pBuilder, row); + if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { + code = convertStmtNcharCol(&pBuf, pColSchema, bind + c, &ncharBind); + if (code) { + goto _return; + } + pBind = &ncharBind; } else { - tdSRowGetBuf(pBuilder, row); + pBind = bind + c; } - SSchema* pColSchema = &pSchema[spd->boundColumns[colIdx]]; + tColDataAddValueByBind(pCol, pBind); + } - if (bind->num != rowNum) { - return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); - } + qDebug("stmt all %d columns bind %d rows data", boundInfo->numOfBound, rowNum); - param.schema = pColSchema; - insGetSTSRowAppendInfo(pBuilder->rowType, spd, colIdx, ¶m.toffset, ¶m.colIdx); +_return: - if (bind->is_null && bind->is_null[r]) { - if (pColSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - return buildInvalidOperationMsg(&pBuf, "primary timestamp should not be NULL"); - } + taosMemoryFree(ncharBind.buffer); + taosMemoryFree(ncharBind.length); - CHECK_CODE(insMemRowAppend(&pBuf, NULL, 0, ¶m)); - } else { - if (bind->buffer_type != pColSchema->type) { - return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); - } - - int32_t colLen = pColSchema->bytes; - if (IS_VAR_DATA_TYPE(pColSchema->type)) { - colLen = bind->length[r]; - } + return code; +} - CHECK_CODE(insMemRowAppend(&pBuf, (char*)bind->buffer + bind->buffer_length * r, colLen, ¶m)); - } +int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, + int32_t rowNum) { + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); + SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + SSchema* pColSchema = &pSchema[boundInfo->pColIndex[colIdx]]; + SColData* pCol = taosArrayGet(pDataBlock->pData->aCol, colIdx); + TAOS_MULTI_BIND ncharBind = {0}; + TAOS_MULTI_BIND* pBind = NULL; + int32_t code = 0; + + if (bind->num != rowNum) { + return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); + } - if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { - TSKEY tsKey = TD_ROW_KEY(row); - insCheckTimestamp(pDataBlock, (const char*)&tsKey); - } + if (bind->buffer_type != pColSchema->type) { + return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); + } - // set the null value for the columns that do not assign values - if (rowEnd && (spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - pBuilder->hasNone = true; + if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { + code = convertStmtNcharCol(&pBuf, pColSchema, bind, &ncharBind); + if (code) { + goto _return; } - if (rowEnd) { - tdSRowEnd(pBuilder); - } -#ifdef TD_DEBUG_PRINT_ROW - if (rowEnd) { - STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1); - tdSRowPrint(row, pSTSchema, __func__); - taosMemoryFree(pSTSchema); - } -#endif + pBind = &ncharBind; + } else { + pBind = bind; } - if (rowEnd) { - pDataBlock->size += extendedRowSize * bind->num; + tColDataAddValueByBind(pCol, pBind); - SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); - CHECK_CODE(insSetBlockInfo(pBlocks, pDataBlock, bind->num, &pBuf)); - } + qDebug("stmt col %d bind %d rows data", colIdx, rowNum); - return TSDB_CODE_SUCCESS; +_return: + + taosMemoryFree(ncharBind.buffer); + taosMemoryFree(ncharBind.length); + + return code; } -int32_t buildBoundFields(SParsedDataColInfo* boundInfo, SSchema* pSchema, int32_t* fieldNum, TAOS_FIELD_E** fields, - uint8_t timePrec) { +int32_t buildBoundFields(int32_t numOfBound, int16_t* boundColumns, SSchema* pSchema, int32_t* fieldNum, + TAOS_FIELD_E** fields, uint8_t timePrec) { if (fields) { - *fields = taosMemoryCalloc(boundInfo->numOfBound, sizeof(TAOS_FIELD)); + *fields = taosMemoryCalloc(numOfBound, sizeof(TAOS_FIELD_E)); if (NULL == *fields) { return TSDB_CODE_OUT_OF_MEMORY; } - SSchema* schema = &pSchema[boundInfo->boundColumns[0]]; + SSchema* schema = &pSchema[boundColumns[0]]; if (TSDB_DATA_TYPE_TIMESTAMP == schema->type) { (*fields)[0].precision = timePrec; } - for (int32_t i = 0; i < boundInfo->numOfBound; ++i) { - schema = &pSchema[boundInfo->boundColumns[i]]; + for (int32_t i = 0; i < numOfBound; ++i) { + schema = &pSchema[boundColumns[i]]; strcpy((*fields)[i].name, schema->name); (*fields)[i].type = schema->type; (*fields)[i].bytes = schema->bytes; } } - *fieldNum = boundInfo->numOfBound; + *fieldNum = numOfBound; return TSDB_CODE_SUCCESS; } int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + SBoundColInfo* tags = (SBoundColInfo*)boundTags; if (NULL == tags) { return TSDB_CODE_APP_ERROR; } - if (pDataBlock->pTableMeta->tableType != TSDB_SUPER_TABLE && pDataBlock->pTableMeta->tableType != TSDB_CHILD_TABLE) { + if (pDataBlock->pMeta->tableType != TSDB_SUPER_TABLE && pDataBlock->pMeta->tableType != TSDB_CHILD_TABLE) { return TSDB_CODE_TSC_STMT_API_ERROR; } - SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); + SSchema* pSchema = getTableTagSchema(pDataBlock->pMeta); if (tags->numOfBound <= 0) { *fieldNum = 0; *fields = NULL; @@ -354,15 +366,15 @@ int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TA return TSDB_CODE_SUCCESS; } - CHECK_CODE(buildBoundFields(tags, pSchema, fieldNum, fields, 0)); + CHECK_CODE(buildBoundFields(tags->numOfBound, tags->pColIndex, pSchema, fieldNum, fields, 0)); return TSDB_CODE_SUCCESS; } int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fields) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - if (pDataBlock->boundColumnInfo.numOfBound <= 0) { + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); + if (pDataBlock->boundColsInfo.numOfBound <= 0) { *fieldNum = 0; if (fields) { *fields = NULL; @@ -371,107 +383,125 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fiel return TSDB_CODE_SUCCESS; } - CHECK_CODE(buildBoundFields(&pDataBlock->boundColumnInfo, pSchema, fieldNum, fields, - pDataBlock->pTableMeta->tableInfo.precision)); + CHECK_CODE(buildBoundFields(pDataBlock->boundColsInfo.numOfBound, pDataBlock->boundColsInfo.pColIndex, pSchema, + fieldNum, fields, pDataBlock->pMeta->tableInfo.precision)); return TSDB_CODE_SUCCESS; } -int32_t qResetStmtDataBlock(void* block, bool keepBuf) { - STableDataBlocks* pBlock = (STableDataBlocks*)block; +int32_t qResetStmtDataBlock(STableDataCxt* block, bool deepClear) { + STableDataCxt* pBlock = (STableDataCxt*)block; + int32_t colNum = taosArrayGetSize(pBlock->pData->aCol); - if (keepBuf) { - taosMemoryFreeClear(pBlock->pData); - pBlock->pData = taosMemoryMalloc(TSDB_PAYLOAD_SIZE); - if (NULL == pBlock->pData) { - return TSDB_CODE_OUT_OF_MEMORY; + for (int32_t i = 0; i < colNum; ++i) { + SColData* pCol = (SColData*)taosArrayGet(pBlock->pData->aCol, i); + if (deepClear) { + tColDataDeepClear(pCol); + } else { + tColDataClear(pCol); } - memset(pBlock->pData, 0, sizeof(SSubmitBlk)); - } else { - pBlock->pData = NULL; } - pBlock->ordered = true; - pBlock->prevTS = INT64_MIN; - pBlock->size = sizeof(SSubmitBlk); - pBlock->tsSource = -1; - pBlock->numOfTables = 1; - pBlock->nAllocSize = TSDB_PAYLOAD_SIZE; - pBlock->headerSize = pBlock->size; - pBlock->createTbReqLen = 0; - - memset(&pBlock->rowBuilder, 0, sizeof(pBlock->rowBuilder)); - return TSDB_CODE_SUCCESS; } -int32_t qCloneStmtDataBlock(void** pDst, void* pSrc) { - *pDst = taosMemoryMalloc(sizeof(STableDataBlocks)); +int32_t qCloneStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, bool reset) { + int32_t code = 0; + + *pDst = taosMemoryCalloc(1, sizeof(STableDataCxt)); if (NULL == *pDst) { return TSDB_CODE_OUT_OF_MEMORY; } - memcpy(*pDst, pSrc, sizeof(STableDataBlocks)); - ((STableDataBlocks*)(*pDst))->cloned = true; + STableDataCxt* pNewCxt = (STableDataCxt*)*pDst; + STableDataCxt* pCxt = (STableDataCxt*)pSrc; + pNewCxt->pSchema = NULL; + pNewCxt->pValues = NULL; - STableDataBlocks* pBlock = (STableDataBlocks*)(*pDst); - if (pBlock->pTableMeta) { - void* pNewMeta = taosMemoryMalloc(TABLE_META_SIZE(pBlock->pTableMeta)); + if (pCxt->pMeta) { + void* pNewMeta = taosMemoryMalloc(TABLE_META_SIZE(pCxt->pMeta)); if (NULL == pNewMeta) { - taosMemoryFreeClear(*pDst); + insDestroyTableDataCxt(*pDst); return TSDB_CODE_OUT_OF_MEMORY; } - memcpy(pNewMeta, pBlock->pTableMeta, TABLE_META_SIZE(pBlock->pTableMeta)); - pBlock->pTableMeta = pNewMeta; + memcpy(pNewMeta, pCxt->pMeta, TABLE_META_SIZE(pCxt->pMeta)); + pNewCxt->pMeta = pNewMeta; } - return qResetStmtDataBlock(*pDst, false); + memcpy(&pNewCxt->boundColsInfo, &pCxt->boundColsInfo, sizeof(pCxt->boundColsInfo)); + pNewCxt->boundColsInfo.pColIndex = NULL; + + if (pCxt->boundColsInfo.pColIndex) { + void* pNewColIdx = taosMemoryMalloc(pCxt->boundColsInfo.numOfBound * sizeof(*pCxt->boundColsInfo.pColIndex)); + if (NULL == pNewColIdx) { + insDestroyTableDataCxt(*pDst); + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(pNewColIdx, pCxt->boundColsInfo.pColIndex, + pCxt->boundColsInfo.numOfBound * sizeof(*pCxt->boundColsInfo.pColIndex)); + pNewCxt->boundColsInfo.pColIndex = pNewColIdx; + } + + if (pCxt->pData) { + SSubmitTbData* pNewTb = (SSubmitTbData*)taosMemoryMalloc(sizeof(SSubmitTbData)); + if (NULL == pNewTb) { + insDestroyTableDataCxt(*pDst); + return TSDB_CODE_OUT_OF_MEMORY; + } + + memcpy(pNewTb, pCxt->pData, sizeof(*pCxt->pData)); + pNewTb->pCreateTbReq = NULL; + + pNewTb->aCol = taosArrayDup(pCxt->pData->aCol, NULL); + if (NULL == pNewTb) { + insDestroyTableDataCxt(*pDst); + return TSDB_CODE_OUT_OF_MEMORY; + } + + pNewCxt->pData = pNewTb; + + if (reset) { + code = qResetStmtDataBlock(*pDst, true); + } + } + + return code; } -int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgId) { - int32_t code = qCloneStmtDataBlock(pDst, pSrc); +int32_t qRebuildStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_t uid, uint64_t suid, int32_t vgId, + bool rebuildCreateTb) { + int32_t code = qCloneStmtDataBlock(pDst, pSrc, false); if (code) { return code; } - STableDataBlocks* pBlock = (STableDataBlocks*)*pDst; - pBlock->pData = taosMemoryMalloc(pBlock->nAllocSize); - if (NULL == pBlock->pData) { - qFreeStmtDataBlock(pBlock); - return TSDB_CODE_OUT_OF_MEMORY; + STableDataCxt* pBlock = (STableDataCxt*)*pDst; + if (pBlock->pMeta) { + pBlock->pMeta->uid = uid; + pBlock->pMeta->vgId = vgId; + pBlock->pMeta->suid = suid; } - pBlock->vgId = vgId; + pBlock->pData->suid = suid; + pBlock->pData->uid = uid; - if (pBlock->pTableMeta) { - pBlock->pTableMeta->uid = uid; - pBlock->pTableMeta->vgId = vgId; + if (rebuildCreateTb && NULL == pBlock->pData->pCreateTbReq) { + pBlock->pData->pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == pBlock->pData->pCreateTbReq) { + return TSDB_CODE_OUT_OF_MEMORY; + } } - memset(pBlock->pData, 0, sizeof(SSubmitBlk)); - return TSDB_CODE_SUCCESS; } -STableMeta* qGetTableMetaInDataBlock(void* pDataBlock) { return ((STableDataBlocks*)pDataBlock)->pTableMeta; } +STableMeta* qGetTableMetaInDataBlock(STableDataCxt* pDataBlock) { return ((STableDataCxt*)pDataBlock)->pMeta; } -void qFreeStmtDataBlock(void* pDataBlock) { - if (pDataBlock == NULL) { - return; - } - - taosMemoryFreeClear(((STableDataBlocks*)pDataBlock)->pTableMeta); - taosMemoryFreeClear(((STableDataBlocks*)pDataBlock)->pData); - taosMemoryFreeClear(pDataBlock); -} - -void qDestroyStmtDataBlock(void* pBlock) { +void qDestroyStmtDataBlock(STableDataCxt* pBlock) { if (pBlock == NULL) { return; } - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - - pDataBlock->cloned = false; - insDestroyDataBlock(pDataBlock); + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + insDestroyTableDataCxt(pDataBlock); } diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 73cedfeb3d4109622062477214cd148bdd441391..b8ef7d970df5dd74176f6dcb0b7a8ab478e34129 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -20,938 +20,671 @@ #include "parUtil.h" #include "querynodes.h" #include "tRealloc.h" +#include "tdatablock.h" -typedef struct SBlockKeyTuple { - TSKEY skey; - void* payloadAddr; - int16_t index; -} SBlockKeyTuple; - -typedef struct SBlockKeyInfo { - int32_t maxBytesAlloc; - SBlockKeyTuple* pKeyTuple; -} SBlockKeyInfo; +void qDestroyBoundColInfo(void* pInfo) { + if (NULL == pInfo) { + return; + } -typedef struct { - int32_t index; - SArray* rowArray; // array of merged rows(mem allocated by tRealloc/free by tFree) - STSchema* pSchema; - int64_t tbUid; // suid for child table, uid for normal table -} SBlockRowMerger; + SBoundColInfo* pBoundInfo = (SBoundColInfo*)pInfo; -static FORCE_INLINE void tdResetSBlockRowMerger(SBlockRowMerger* pMerger) { - if (pMerger) { - pMerger->index = -1; - } + taosMemoryFreeClear(pBoundInfo->pColIndex); } -static void tdFreeSBlockRowMerger(SBlockRowMerger* pMerger) { - if (pMerger) { - int32_t size = taosArrayGetSize(pMerger->rowArray); - for (int32_t i = 0; i < size; ++i) { - tFree(*(void**)taosArrayGet(pMerger->rowArray, i)); +static char* tableNameGetPosition(SToken* pToken, char target) { + bool inEscape = false; + bool inQuote = false; + char quotaStr = 0; + + for (uint32_t i = 0; i < pToken->n; ++i) { + if (*(pToken->z + i) == target && (!inEscape) && (!inQuote)) { + return pToken->z + i; } - taosArrayDestroy(pMerger->rowArray); - taosMemoryFreeClear(pMerger->pSchema); - taosMemoryFree(pMerger); - } -} + if (*(pToken->z + i) == TS_ESCAPE_CHAR) { + if (!inQuote) { + inEscape = !inEscape; + } + } -static int32_t rowDataCompar(const void* lhs, const void* rhs) { - TSKEY left = *(TSKEY*)lhs; - TSKEY right = *(TSKEY*)rhs; - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; + if (*(pToken->z + i) == '\'' || *(pToken->z + i) == '"') { + if (!inEscape) { + if (!inQuote) { + quotaStr = *(pToken->z + i); + inQuote = !inQuote; + } else if (quotaStr == *(pToken->z + i)) { + inQuote = !inQuote; + } + } + } } -} -static int32_t rowDataComparStable(const void* lhs, const void* rhs) { - TSKEY left = *(TSKEY*)lhs; - TSKEY right = *(TSKEY*)rhs; - if (left == right) { - return ((SBlockKeyTuple*)lhs)->index - ((SBlockKeyTuple*)rhs)->index; - } else { - return left > right ? 1 : -1; - } + return NULL; } -int32_t insGetExtendedRowSize(STableDataBlocks* pBlock) { - STableComInfo* pTableInfo = &pBlock->pTableMeta->tableInfo; - ASSERT(pBlock->rowSize == pTableInfo->rowSize); - return pBlock->rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + pBlock->boundColumnInfo.extendedVarLen + - (int32_t)TD_BITMAP_BYTES(pTableInfo->numOfColumns - 1); -} +int32_t insCreateSName(SName* pName, SToken* pTableName, int32_t acctId, const char* dbName, SMsgBuf* pMsgBuf) { + const char* msg1 = "name too long"; + const char* msg2 = "invalid database name"; + const char* msg3 = "db is not specified"; + const char* msg4 = "invalid table name"; -void insGetSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo* spd, col_id_t idx, int32_t* toffset, - col_id_t* colIdx) { - col_id_t schemaIdx = 0; - if (IS_DATA_COL_ORDERED(spd)) { - schemaIdx = spd->boundColumns[idx]; - if (TD_IS_TP_ROW_T(rowType)) { - *toffset = (spd->cols + schemaIdx)->toffset; // the offset of firstPart - *colIdx = schemaIdx; - } else { - *toffset = idx * sizeof(SKvRowIdx); // the offset of SKvRowIdx - *colIdx = idx; + int32_t code = TSDB_CODE_SUCCESS; + char* p = tableNameGetPosition(pTableName, TS_PATH_DELIMITER[0]); + + if (p != NULL) { // db has been specified in sql string so we ignore current db path + int32_t dbLen = p - pTableName->z; + if (dbLen <= 0) { + return buildInvalidOperationMsg(pMsgBuf, msg2); } - } else { - ASSERT(idx == (spd->colIdxInfo + idx)->boundIdx); - schemaIdx = (spd->colIdxInfo + idx)->schemaColIdx; - if (TD_IS_TP_ROW_T(rowType)) { - *toffset = (spd->cols + schemaIdx)->toffset; - *colIdx = schemaIdx; - } else { - *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SKvRowIdx); - *colIdx = (spd->colIdxInfo + idx)->finalIdx; + char name[TSDB_DB_FNAME_LEN] = {0}; + strncpy(name, pTableName->z, dbLen); + int32_t actualDbLen = strdequote(name); + + code = tNameSetDbName(pName, acctId, name, actualDbLen); + if (code != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg1); } - } -} -int32_t insSetBlockInfo(SSubmitBlk* pBlocks, STableDataBlocks* dataBuf, int32_t numOfRows, SMsgBuf* pMsg) { - pBlocks->suid = (TSDB_NORMAL_TABLE == dataBuf->pTableMeta->tableType ? 0 : dataBuf->pTableMeta->suid); - pBlocks->uid = dataBuf->pTableMeta->uid; - pBlocks->sversion = dataBuf->pTableMeta->sversion; - pBlocks->schemaLen = dataBuf->createTbReqLen; + int32_t tbLen = pTableName->n - dbLen - 1; + if (tbLen <= 0) { + return buildInvalidOperationMsg(pMsgBuf, msg4); + } - if (pBlocks->numOfRows + numOfRows >= INT32_MAX) { - return buildInvalidOperationMsg(pMsg, "too many rows in sql, total number of rows should be less than INT32_MAX"); - } - pBlocks->numOfRows += numOfRows; - return TSDB_CODE_SUCCESS; -} + char tbname[TSDB_TABLE_FNAME_LEN] = {0}; + strncpy(tbname, p + 1, tbLen); + /*tbLen = */ strdequote(tbname); -void insSetBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, col_id_t numOfCols) { - pColList->numOfCols = numOfCols; - pColList->numOfBound = numOfCols; - pColList->orderStatus = ORDER_STATUS_ORDERED; // default is ORDERED for non-bound mode - pColList->boundColumns = taosMemoryCalloc(pColList->numOfCols, sizeof(col_id_t)); - pColList->cols = taosMemoryCalloc(pColList->numOfCols, sizeof(SBoundColumn)); - pColList->colIdxInfo = NULL; - pColList->flen = 0; - pColList->allNullLen = 0; - - int32_t nVar = 0; - for (int32_t i = 0; i < pColList->numOfCols; ++i) { - uint8_t type = pSchema[i].type; - if (i > 0) { - pColList->cols[i].offset = pColList->cols[i - 1].offset + pSchema[i - 1].bytes; - pColList->cols[i].toffset = pColList->flen; - pColList->flen += TYPE_BYTES[type]; + code = tNameFromString(pName, tbname, T_NAME_TABLE); + if (code != 0) { + return buildInvalidOperationMsg(pMsgBuf, msg1); } - switch (type) { - case TSDB_DATA_TYPE_BINARY: - pColList->allNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES); - ++nVar; - break; - case TSDB_DATA_TYPE_NCHAR: - pColList->allNullLen += (VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); - ++nVar; - break; - default: - break; + } else { // get current DB name first, and then set it into path + if (pTableName->n >= TSDB_TABLE_NAME_LEN) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + strncpy(name, pTableName->z, pTableName->n); + strdequote(name); + + if (dbName == NULL) { + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + + code = tNameSetDbName(pName, acctId, dbName, strlen(dbName)); + if (code != TSDB_CODE_SUCCESS) { + code = buildInvalidOperationMsg(pMsgBuf, msg2); + return code; + } + + code = tNameFromString(pName, name, T_NAME_TABLE); + if (code != 0) { + code = buildInvalidOperationMsg(pMsgBuf, msg1); } - pColList->boundColumns[i] = i; } - pColList->allNullLen += pColList->flen; - pColList->boundNullLen = pColList->allNullLen; // default set allNullLen - pColList->extendedVarLen = (uint16_t)(nVar * sizeof(VarDataOffsetT)); -} -int32_t insSchemaIdxCompar(const void* lhs, const void* rhs) { - uint16_t left = *(uint16_t*)lhs; - uint16_t right = *(uint16_t*)rhs; + if (NULL != strchr(pName->tname, '.')) { + code = generateSyntaxErrMsgExt(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, "The table name cannot contain '.'"); + } + + return code; +} - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; +int16_t insFindCol(SToken* pColname, int16_t start, int16_t end, SSchema* pSchema) { + while (start < end) { + if (strlen(pSchema[start].name) == pColname->n && strncmp(pColname->z, pSchema[start].name, pColname->n) == 0) { + return start; + } + ++start; } + return -1; } -int32_t insBoundIdxCompar(const void* lhs, const void* rhs) { - uint16_t left = *(uint16_t*)POINTER_SHIFT(lhs, sizeof(uint16_t)); - uint16_t right = *(uint16_t*)POINTER_SHIFT(rhs, sizeof(uint16_t)); +void insBuildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, int64_t suid, const char* sname, + SArray* tagName, uint8_t tagNum, int32_t ttl) { + pTbReq->type = TD_CHILD_TABLE; + pTbReq->name = strdup(tname); + pTbReq->ctb.suid = suid; + pTbReq->ctb.tagNum = tagNum; + if (sname) pTbReq->ctb.stbName = strdup(sname); + pTbReq->ctb.pTag = (uint8_t*)pTag; + pTbReq->ctb.tagName = taosArrayDup(tagName, NULL); + pTbReq->ttl = ttl; + pTbReq->commentLen = -1; + + return; +} - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; +static void initBoundCols(int32_t ncols, int16_t* pBoundCols) { + for (int32_t i = 0; i < ncols; ++i) { + pBoundCols[i] = i; } } -void destroyBoundColumnInfo(void* pBoundInfo) { - if (NULL == pBoundInfo) { - return; +static void initColValues(STableMeta* pTableMeta, SArray* pValues) { + SSchema* pSchemas = getTableColumnSchema(pTableMeta); + for (int32_t i = 0; i < pTableMeta->tableInfo.numOfColumns; ++i) { + SColVal val = COL_VAL_NONE(pSchemas[i].colId, pSchemas[i].type); + taosArrayPush(pValues, &val); } - - SParsedDataColInfo* pColList = (SParsedDataColInfo*)pBoundInfo; - - taosMemoryFreeClear(pColList->boundColumns); - taosMemoryFreeClear(pColList->cols); - taosMemoryFreeClear(pColList->colIdxInfo); } -static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, STableMeta* pTableMeta, - STableDataBlocks** dataBlocks) { - STableDataBlocks* dataBuf = (STableDataBlocks*)taosMemoryCalloc(1, sizeof(STableDataBlocks)); - if (dataBuf == NULL) { +int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) { + pInfo->numOfCols = numOfBound; + pInfo->numOfBound = numOfBound; + pInfo->pColIndex = taosMemoryCalloc(numOfBound, sizeof(int16_t)); + if (NULL == pInfo->pColIndex) { return TSDB_CODE_OUT_OF_MEMORY; } + initBoundCols(numOfBound, pInfo->pColIndex); + return TSDB_CODE_SUCCESS; +} - dataBuf->nAllocSize = (uint32_t)defaultSize; - dataBuf->headerSize = startOffset; +void insCheckTableDataOrder(STableDataCxt* pTableCxt, TSKEY tsKey) { + // once the data block is disordered, we do NOT keep last timestamp any more + if (!pTableCxt->ordered) { + return; + } - // the header size will always be the startOffset value, reserved for the subumit block header - if (dataBuf->nAllocSize <= dataBuf->headerSize) { - dataBuf->nAllocSize = dataBuf->headerSize * 2; + if (tsKey < pTableCxt->lastTs) { + pTableCxt->ordered = false; } - dataBuf->pData = taosMemoryMalloc(dataBuf->nAllocSize); - if (dataBuf->pData == NULL) { - taosMemoryFreeClear(dataBuf); - return TSDB_CODE_OUT_OF_MEMORY; + if (tsKey == pTableCxt->lastTs) { + pTableCxt->duplicateTs = true; } - memset(dataBuf->pData, 0, sizeof(SSubmitBlk)); - dataBuf->pTableMeta = tableMetaDup(pTableMeta); + pTableCxt->lastTs = tsKey; + return; +} - SParsedDataColInfo* pColInfo = &dataBuf->boundColumnInfo; - SSchema* pSchema = getTableColumnSchema(dataBuf->pTableMeta); - insSetBoundColumnInfo(pColInfo, pSchema, dataBuf->pTableMeta->tableInfo.numOfColumns); +void insDestroyBoundColInfo(SBoundColInfo* pInfo) { taosMemoryFreeClear(pInfo->pColIndex); } - dataBuf->ordered = true; - dataBuf->prevTS = INT64_MIN; - dataBuf->rowSize = rowSize; - dataBuf->size = startOffset; - dataBuf->vgId = dataBuf->pTableMeta->vgId; +static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreateTbReq, STableDataCxt** pOutput, + bool colMode) { + STableDataCxt* pTableCxt = taosMemoryCalloc(1, sizeof(STableDataCxt)); + if (NULL == pTableCxt) { + return TSDB_CODE_OUT_OF_MEMORY; + } - assert(defaultSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); + int32_t code = TSDB_CODE_SUCCESS; - *dataBlocks = dataBuf; - return TSDB_CODE_SUCCESS; -} + pTableCxt->lastTs = 0; + pTableCxt->ordered = true; + pTableCxt->duplicateTs = false; -int32_t insBuildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) { - SEncoder coder = {0}; - char* pBuf; - int32_t len; - - int32_t ret = 0; - tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len, ret); - if (pBlocks->nAllocSize - pBlocks->size < len) { - pBlocks->nAllocSize += len + pBlocks->rowSize; - char* pTmp = taosMemoryRealloc(pBlocks->pData, pBlocks->nAllocSize); - if (pTmp != NULL) { - pBlocks->pData = pTmp; - memset(pBlocks->pData + pBlocks->size, 0, pBlocks->nAllocSize - pBlocks->size); + pTableCxt->pMeta = tableMetaDup(pTableMeta); + if (NULL == pTableCxt->pMeta) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + if (TSDB_CODE_SUCCESS == code) { + pTableCxt->pSchema = + tBuildTSchema(getTableColumnSchema(pTableMeta), pTableMeta->tableInfo.numOfColumns, pTableMeta->sversion); + if (NULL == pTableCxt->pSchema) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + if (TSDB_CODE_SUCCESS == code) { + code = insInitBoundColsInfo(pTableMeta->tableInfo.numOfColumns, &pTableCxt->boundColsInfo); + } + if (TSDB_CODE_SUCCESS == code) { + pTableCxt->pValues = taosArrayInit(pTableMeta->tableInfo.numOfColumns, sizeof(SColVal)); + if (NULL == pTableCxt->pValues) { + code = TSDB_CODE_OUT_OF_MEMORY; } else { - pBlocks->nAllocSize -= len + pBlocks->rowSize; - return TSDB_CODE_OUT_OF_MEMORY; + initColValues(pTableMeta, pTableCxt->pValues); + } + } + if (TSDB_CODE_SUCCESS == code) { + pTableCxt->pData = taosMemoryCalloc(1, sizeof(SSubmitTbData)); + if (NULL == pTableCxt->pData) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pTableCxt->pData->flags = NULL != *pCreateTbReq ? SUBMIT_REQ_AUTO_CREATE_TABLE : 0; + pTableCxt->pData->flags |= colMode ? SUBMIT_REQ_COLUMN_DATA_FORMAT : 0; + pTableCxt->pData->suid = pTableMeta->suid; + pTableCxt->pData->uid = pTableMeta->uid; + pTableCxt->pData->sver = pTableMeta->sversion; + pTableCxt->pData->pCreateTbReq = *pCreateTbReq; + *pCreateTbReq = NULL; + if (pTableCxt->pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + pTableCxt->pData->aCol = taosArrayInit(128, sizeof(SColData)); + if (NULL == pTableCxt->pData->aCol) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } else { + pTableCxt->pData->aRowP = taosArrayInit(128, POINTER_BYTES); + if (NULL == pTableCxt->pData->aRowP) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } } } - pBuf = pBlocks->pData + pBlocks->size; - - tEncoderInit(&coder, pBuf, len); - int32_t code = tEncodeSVCreateTbReq(&coder, pCreateTbReq); - tEncoderClear(&coder); - pBlocks->size += len; - pBlocks->createTbReqLen = len; + if (TSDB_CODE_SUCCESS == code) { + *pOutput = pTableCxt; + qDebug("tableDataCxt created, uid:%" PRId64 ", vgId:%d", pTableMeta->uid, pTableMeta->vgId); + } else { + taosMemoryFree(pTableCxt); + } return code; } -void insDestroyDataBlock(STableDataBlocks* pDataBlock) { - if (pDataBlock == NULL) { - return; +static void resetColValues(SArray* pValues) { + int32_t num = taosArrayGetSize(pValues); + for (int32_t i = 0; i < num; ++i) { + SColVal* pVal = taosArrayGet(pValues, i); + pVal->flag = CV_FLAG_NONE; } +} - taosMemoryFreeClear(pDataBlock->pData); - taosMemoryFreeClear(pDataBlock->pTableMeta); - destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); - taosMemoryFreeClear(pDataBlock); +int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta* pTableMeta, + SVCreateTbReq** pCreateTbReq, STableDataCxt** pTableCxt, bool colMode) { + STableDataCxt** tmp = (STableDataCxt**)taosHashGet(pHash, id, idLen); + if (NULL != tmp) { + *pTableCxt = *tmp; + resetColValues((*pTableCxt)->pValues); + return TSDB_CODE_SUCCESS; + } + int32_t code = createTableDataCxt(pTableMeta, pCreateTbReq, pTableCxt, colMode); + if (TSDB_CODE_SUCCESS == code) { + code = taosHashPut(pHash, id, idLen, pTableCxt, POINTER_BYTES); + } + return code; } -int32_t insGetDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, int32_t size, int32_t startOffset, - int32_t rowSize, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, - SArray* pBlockList, SVCreateTbReq* pCreateTbReq) { - *dataBlocks = NULL; - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)id, idLen); - if (t1 != NULL) { - *dataBlocks = *t1; +static void destroyColVal(void* p) { + SColVal* pVal = p; + if (TSDB_DATA_TYPE_NCHAR == pVal->type) { + taosMemoryFree(pVal->value.pData); } +} - if (*dataBlocks == NULL) { - int32_t ret = createDataBlock((size_t)size, rowSize, startOffset, pTableMeta, dataBlocks); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } +void insDestroyTableDataCxt(STableDataCxt* pTableCxt) { + if (NULL == pTableCxt) { + return; + } - if (NULL != pCreateTbReq && NULL != pCreateTbReq->ctb.pTag) { - ret = insBuildCreateTbMsg(*dataBlocks, pCreateTbReq); - if (ret != TSDB_CODE_SUCCESS) { - insDestroyDataBlock(*dataBlocks); - return ret; - } - } + taosMemoryFreeClear(pTableCxt->pMeta); + tDestroyTSchema(pTableCxt->pSchema); + insDestroyBoundColInfo(&pTableCxt->boundColsInfo); + taosArrayDestroyEx(pTableCxt->pValues, destroyColVal); + if (pTableCxt->pData) { + tDestroySSubmitTbData(pTableCxt->pData, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pTableCxt->pData); + } + taosMemoryFree(pTableCxt); +} - // converting to 'const char*' is to handle coverity scan errors - taosHashPut(pHashList, (const char*)id, idLen, (const char*)dataBlocks, POINTER_BYTES); - if (pBlockList) { - taosArrayPush(pBlockList, dataBlocks); - } +void insDestroyVgroupDataCxt(SVgroupDataCxt* pVgCxt) { + if (NULL == pVgCxt) { + return; } - return TSDB_CODE_SUCCESS; + tDestroySSubmitReq2(pVgCxt->pData, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pVgCxt->pData); + taosMemoryFree(pVgCxt); } -void insDestroyBlockArrayList(SArray* pDataBlockList) { - if (pDataBlockList == NULL) { +void insDestroyVgroupDataCxtList(SArray* pVgCxtList) { + if (NULL == pVgCxtList) { return; } - size_t size = taosArrayGetSize(pDataBlockList); + size_t size = taosArrayGetSize(pVgCxtList); for (int32_t i = 0; i < size; i++) { - void* p = taosArrayGetP(pDataBlockList, i); - insDestroyDataBlock(p); + void* p = taosArrayGetP(pVgCxtList, i); + insDestroyVgroupDataCxt(p); } - taosArrayDestroy(pDataBlockList); + taosArrayDestroy(pVgCxtList); } -void insDestroyBlockHashmap(SHashObj* pDataBlockHash) { - if (pDataBlockHash == NULL) { +void insDestroyVgroupDataCxtHashMap(SHashObj* pVgCxtHash) { + if (NULL == pVgCxtHash) { return; } - void** p1 = taosHashIterate(pDataBlockHash, NULL); - while (p1) { - STableDataBlocks* pBlocks = *p1; - insDestroyDataBlock(pBlocks); + void** p = taosHashIterate(pVgCxtHash, NULL); + while (p) { + insDestroyVgroupDataCxt(*(SVgroupDataCxt**)p); - p1 = taosHashIterate(pDataBlockHash, p1); + p = taosHashIterate(pVgCxtHash, p); } - taosHashCleanup(pDataBlockHash); + taosHashCleanup(pVgCxtHash); } -// data block is disordered, sort it in ascending order -static int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo) { - SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData; - int16_t nRows = pBlocks->numOfRows; - - // size is less than the total size, since duplicated rows may be removed yet. - - // allocate memory - size_t nAlloc = nRows * sizeof(SBlockKeyTuple); - if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { - char* tmp = taosMemoryRealloc(pBlkKeyInfo->pKeyTuple, nAlloc); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple*)tmp; - pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc; +void insDestroyTableDataCxtHashMap(SHashObj* pTableCxtHash) { + if (NULL == pTableCxtHash) { + return; } - memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc); - int32_t extendedRowSize = insGetExtendedRowSize(dataBuf); - SBlockKeyTuple* pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - char* pBlockData = pBlocks->data + pBlocks->schemaLen; - int n = 0; - while (n < nRows) { - pBlkKeyTuple->skey = TD_ROW_KEY((STSRow*)pBlockData); - pBlkKeyTuple->payloadAddr = pBlockData; - pBlkKeyTuple->index = n; + void** p = taosHashIterate(pTableCxtHash, NULL); + while (p) { + insDestroyTableDataCxt(*(STableDataCxt**)p); - // next loop - pBlockData += extendedRowSize; - ++pBlkKeyTuple; - ++n; + p = taosHashIterate(pTableCxtHash, p); } - if (!dataBuf->ordered) { - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - - // todo. qsort is unstable, if timestamp is same, should get the last one - taosSort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataComparStable); - - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - int32_t i = 0; - int32_t j = 1; - while (j < nRows) { - TSKEY ti = (pBlkKeyTuple + i)->skey; - TSKEY tj = (pBlkKeyTuple + j)->skey; - - if (ti == tj) { - ++j; - continue; - } + taosHashCleanup(pTableCxtHash); +} - int32_t nextPos = (++i); - if (nextPos != j) { - memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + j, sizeof(SBlockKeyTuple)); - } - ++j; +static int32_t fillVgroupDataCxt(STableDataCxt* pTableCxt, SVgroupDataCxt* pVgCxt) { + if (NULL == pVgCxt->pData->aSubmitTbData) { + pVgCxt->pData->aSubmitTbData = taosArrayInit(128, sizeof(SSubmitTbData)); + if (NULL == pVgCxt->pData->aSubmitTbData) { + return TSDB_CODE_OUT_OF_MEMORY; } - - dataBuf->ordered = true; - pBlocks->numOfRows = i + 1; } + taosArrayPush(pVgCxt->pData->aSubmitTbData, pTableCxt->pData); + taosMemoryFreeClear(pTableCxt->pData); - dataBuf->size = sizeof(SSubmitBlk) + pBlocks->numOfRows * extendedRowSize; - dataBuf->prevTS = INT64_MIN; + qDebug("add tableDataCxt uid:%" PRId64 " to vgId:%d", pTableCxt->pMeta->uid, pVgCxt->vgId); - return 0; -} - -static void* tdGetCurRowFromBlockMerger(SBlockRowMerger* pBlkRowMerger) { - if (pBlkRowMerger && (pBlkRowMerger->index >= 0)) { - ASSERT(pBlkRowMerger->index < taosArrayGetSize(pBlkRowMerger->rowArray)); - return *(void**)taosArrayGet(pBlkRowMerger->rowArray, pBlkRowMerger->index); - } - return NULL; + return TSDB_CODE_SUCCESS; } -static int32_t tdBlockRowMerge(STableMeta* pTableMeta, SBlockKeyTuple* pEndKeyTp, int32_t nDupRows, - SBlockRowMerger** pBlkRowMerger, int32_t rowSize) { - ASSERT(nDupRows > 1); - SBlockKeyTuple* pStartKeyTp = pEndKeyTp - (nDupRows - 1); - ASSERT(pStartKeyTp->skey == pEndKeyTp->skey); - - // TODO: optimization if end row is all normal -#if 0 - STSRow* pEndRow = (STSRow*)pEndKeyTp->payloadAddr; - if(isNormal(pEndRow)) { // set the end row if it is normal and return directly - pStartKeyTp->payloadAddr = pEndKeyTp->payloadAddr; - return TSDB_CODE_SUCCESS; - } -#endif - - if (!(*pBlkRowMerger)) { - (*pBlkRowMerger) = taosMemoryCalloc(1, sizeof(**pBlkRowMerger)); - if (!(*pBlkRowMerger)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - (*pBlkRowMerger)->index = -1; - if (!(*pBlkRowMerger)->rowArray) { - (*pBlkRowMerger)->rowArray = taosArrayInit(1, sizeof(void*)); - if (!(*pBlkRowMerger)->rowArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - } - } - - if ((*pBlkRowMerger)->pSchema) { - if ((*pBlkRowMerger)->pSchema->version != pTableMeta->sversion) { - taosMemoryFreeClear((*pBlkRowMerger)->pSchema); - } else { - if ((*pBlkRowMerger)->tbUid != (pTableMeta->suid > 0 ? pTableMeta->suid : pTableMeta->uid)) { - taosMemoryFreeClear((*pBlkRowMerger)->pSchema); - } - } +static int32_t createVgroupDataCxt(STableDataCxt* pTableCxt, SHashObj* pVgroupHash, SArray* pVgroupList, + SVgroupDataCxt** pOutput) { + SVgroupDataCxt* pVgCxt = taosMemoryCalloc(1, sizeof(SVgroupDataCxt)); + if (NULL == pVgCxt) { + return TSDB_CODE_OUT_OF_MEMORY; } - - if (!(*pBlkRowMerger)->pSchema) { - (*pBlkRowMerger)->pSchema = - tdGetSTSChemaFromSSChema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->sversion); - - if (!(*pBlkRowMerger)->pSchema) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - (*pBlkRowMerger)->tbUid = pTableMeta->suid > 0 ? pTableMeta->suid : pTableMeta->uid; + pVgCxt->pData = taosMemoryCalloc(1, sizeof(SSubmitReq2)); + if (NULL == pVgCxt->pData) { + insDestroyVgroupDataCxt(pVgCxt); + return TSDB_CODE_OUT_OF_MEMORY; } - void* pDestRow = NULL; - ++((*pBlkRowMerger)->index); - if ((*pBlkRowMerger)->index < taosArrayGetSize((*pBlkRowMerger)->rowArray)) { - void** pAlloc = (void**)taosArrayGet((*pBlkRowMerger)->rowArray, (*pBlkRowMerger)->index); - if (tRealloc((uint8_t**)pAlloc, rowSize) != 0) { - return TSDB_CODE_FAILED; - } - pDestRow = *pAlloc; + pVgCxt->vgId = pTableCxt->pMeta->vgId; + int32_t code = taosHashPut(pVgroupHash, &pVgCxt->vgId, sizeof(pVgCxt->vgId), &pVgCxt, POINTER_BYTES); + if (TSDB_CODE_SUCCESS == code) { + taosArrayPush(pVgroupList, &pVgCxt); + *pOutput = pVgCxt; } else { - if (tRealloc((uint8_t**)&pDestRow, rowSize) != 0) { - return TSDB_CODE_FAILED; - } - taosArrayPush((*pBlkRowMerger)->rowArray, &pDestRow); - } - - // merge rows to pDestRow - STSchema* pSchema = (*pBlkRowMerger)->pSchema; - SArray* pArray = taosArrayInit(pSchema->numOfCols, sizeof(SColVal)); - for (int32_t i = 0; i < pSchema->numOfCols; ++i) { - SColVal colVal = {0}; - for (int32_t j = 0; j < nDupRows; ++j) { - tTSRowGetVal((pEndKeyTp - j)->payloadAddr, pSchema, i, &colVal); - if (!COL_VAL_IS_NONE(&colVal)) { - break; - } - } - taosArrayPush(pArray, &colVal); + insDestroyVgroupDataCxt(pVgCxt); } - if (tdSTSRowNew(pArray, pSchema, (STSRow**)&pDestRow) < 0) { - taosArrayDestroy(pArray); - return TSDB_CODE_FAILED; - } - - taosArrayDestroy(pArray); - return TSDB_CODE_SUCCESS; + return code; } -// data block is disordered, sort it in ascending order, and merge dup rows if exists -static int sortMergeDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo, - SBlockRowMerger** ppBlkRowMerger) { - SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData; - STableMeta* pTableMeta = dataBuf->pTableMeta; - int32_t nRows = pBlocks->numOfRows; - - // size is less than the total size, since duplicated rows may be removed. - - // allocate memory - size_t nAlloc = nRows * sizeof(SBlockKeyTuple); - if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { - char* tmp = taosMemoryRealloc(pBlkKeyInfo->pKeyTuple, nAlloc); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple*)tmp; - pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc; +int insColDataComp(const void* lp, const void* rp) { + SColData* pLeft = (SColData*)lp; + SColData* pRight = (SColData*)rp; + if (pLeft->cid < pRight->cid) { + return -1; + } else if (pLeft->cid > pRight->cid) { + return 1; } - memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc); - tdResetSBlockRowMerger(*ppBlkRowMerger); - - int32_t extendedRowSize = insGetExtendedRowSize(dataBuf); - SBlockKeyTuple* pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - char* pBlockData = pBlocks->data + pBlocks->schemaLen; - int32_t n = 0; - while (n < nRows) { - pBlkKeyTuple->skey = TD_ROW_KEY((STSRow*)pBlockData); - pBlkKeyTuple->payloadAddr = pBlockData; - pBlkKeyTuple->index = n; + return 0; +} - // next loop - pBlockData += extendedRowSize; - ++pBlkKeyTuple; - ++n; +int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks) { + SHashObj* pVgroupHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); + SArray* pVgroupList = taosArrayInit(8, POINTER_BYTES); + if (NULL == pVgroupHash || NULL == pVgroupList) { + taosHashCleanup(pVgroupHash); + taosArrayDestroy(pVgroupList); + return TSDB_CODE_OUT_OF_MEMORY; } - if (!dataBuf->ordered) { - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - - taosSort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataComparStable); - - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - bool hasDup = false; - int32_t nextPos = 0; - int32_t i = 0; - int32_t j = 1; + int32_t code = TSDB_CODE_SUCCESS; + bool colFormat = false; - while (j < nRows) { - TSKEY ti = (pBlkKeyTuple + i)->skey; - TSKEY tj = (pBlkKeyTuple + j)->skey; + void* p = taosHashIterate(pTableHash, NULL); + if (p) { + STableDataCxt* pTableCxt = *(STableDataCxt**)p; + colFormat = (0 != (pTableCxt->pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT)); + } - if (ti == tj) { - ++j; + while (TSDB_CODE_SUCCESS == code && NULL != p) { + STableDataCxt* pTableCxt = *(STableDataCxt**)p; + if (colFormat) { + SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, 0); + if (pCol->nVal <= 0) { + p = taosHashIterate(pTableHash, p); continue; } - if ((j - i) > 1) { - if (tdBlockRowMerge(pTableMeta, (pBlkKeyTuple + j - 1), j - i, ppBlkRowMerger, extendedRowSize) < 0) { - return TSDB_CODE_FAILED; - } - (pBlkKeyTuple + nextPos)->payloadAddr = tdGetCurRowFromBlockMerger(*ppBlkRowMerger); - if (!hasDup) { - hasDup = true; - } - i = j; - } else { - if (hasDup) { - memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + i, sizeof(SBlockKeyTuple)); - } - ++i; + if (pTableCxt->pData->pCreateTbReq) { + pTableCxt->pData->flags |= SUBMIT_REQ_AUTO_CREATE_TABLE; } - ++nextPos; - ++j; - } + taosArraySort(pTableCxt->pData->aCol, insColDataComp); - if ((j - i) > 1) { - ASSERT((pBlkKeyTuple + i)->skey == (pBlkKeyTuple + j - 1)->skey); - if (tdBlockRowMerge(pTableMeta, (pBlkKeyTuple + j - 1), j - i, ppBlkRowMerger, extendedRowSize) < 0) { - return TSDB_CODE_FAILED; + tColDataSortMerge(pTableCxt->pData->aCol); + } else { + if (!pTableCxt->ordered) { + tRowSort(pTableCxt->pData->aRowP); + } + if (!pTableCxt->ordered || pTableCxt->duplicateTs) { + code = tRowMerge(pTableCxt->pData->aRowP, pTableCxt->pSchema, 0); } - (pBlkKeyTuple + nextPos)->payloadAddr = tdGetCurRowFromBlockMerger(*ppBlkRowMerger); - } else if (hasDup) { - memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + i, sizeof(SBlockKeyTuple)); } - dataBuf->ordered = true; - pBlocks->numOfRows = nextPos + 1; - } - - dataBuf->size = sizeof(SSubmitBlk) + pBlocks->numOfRows * extendedRowSize; - dataBuf->prevTS = INT64_MIN; - - return TSDB_CODE_SUCCESS; -} - -// Erase the empty space reserved for binary data -static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SBlockKeyTuple* blkKeyTuple) { - // TODO: optimize this function, handle the case while binary is not presented - int32_t nonDataLen = sizeof(SSubmitBlk) + pTableDataBlock->createTbReqLen; - SSubmitBlk* pBlock = pDataBlock; - memcpy(pDataBlock, pTableDataBlock->pData, nonDataLen); - pDataBlock = (char*)pDataBlock + nonDataLen; - - pBlock->schemaLen = pTableDataBlock->createTbReqLen; - pBlock->dataLen = 0; - - int32_t numOfRows = pBlock->numOfRows; - for (int32_t i = 0; i < numOfRows; ++i) { - void* payload = (blkKeyTuple + i)->payloadAddr; - TDRowLenT rowTLen = TD_ROW_LEN((STSRow*)payload); - memcpy(pDataBlock, payload, rowTLen); - pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen); - pBlock->dataLen += rowTLen; - } - - return pBlock->dataLen + pBlock->schemaLen; -} - -int32_t insMergeTableDataBlocks(SHashObj* pHashObj, SArray** pVgDataBlocks) { - const int INSERT_HEAD_SIZE = sizeof(SSubmitReq); - int code = 0; - SHashObj* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); - SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); - - STableDataBlocks** p = taosHashIterate(pHashObj, NULL); - STableDataBlocks* pOneTableBlock = *p; - SBlockKeyInfo blkKeyInfo = {0}; // share by pOneTableBlock - SBlockRowMerger* pBlkRowMerger = NULL; - - while (pOneTableBlock) { - SSubmitBlk* pBlocks = (SSubmitBlk*)pOneTableBlock->pData; - if (pBlocks->numOfRows > 0) { - STableDataBlocks* dataBuf = NULL; - pOneTableBlock->pTableMeta->vgId = pOneTableBlock->vgId; // for schemaless, restore origin vgId - int32_t ret = insGetDataBlockFromList(pVnodeDataBlockHashList, &pOneTableBlock->vgId, - sizeof(pOneTableBlock->vgId), TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, - pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList, NULL); - if (ret != TSDB_CODE_SUCCESS) { - tdFreeSBlockRowMerger(pBlkRowMerger); - taosHashCleanup(pVnodeDataBlockHashList); - insDestroyBlockArrayList(pVnodeDataBlockList); - taosMemoryFreeClear(blkKeyInfo.pKeyTuple); - return ret; - } - ASSERT(pOneTableBlock->pTableMeta->tableInfo.rowSize > 0); - // the maximum expanded size in byte when a row-wise data is converted to SDataRow format - int64_t destSize = dataBuf->size + pOneTableBlock->size + - sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta) + - pOneTableBlock->createTbReqLen; - - if (dataBuf->nAllocSize < destSize) { - dataBuf->nAllocSize = (uint32_t)(destSize * 1.5); - char* tmp = taosMemoryRealloc(dataBuf->pData, dataBuf->nAllocSize); - if (tmp != NULL) { - dataBuf->pData = tmp; - } else { // failed to allocate memory, free already allocated memory and return error code - tdFreeSBlockRowMerger(pBlkRowMerger); - taosHashCleanup(pVnodeDataBlockHashList); - insDestroyBlockArrayList(pVnodeDataBlockList); - taosMemoryFreeClear(dataBuf->pData); - taosMemoryFreeClear(blkKeyInfo.pKeyTuple); - return TSDB_CODE_OUT_OF_MEMORY; - } + if (TSDB_CODE_SUCCESS == code) { + SVgroupDataCxt* pVgCxt = NULL; + int32_t vgId = pTableCxt->pMeta->vgId; + void** p = taosHashGet(pVgroupHash, &vgId, sizeof(vgId)); + if (NULL == p) { + code = createVgroupDataCxt(pTableCxt, pVgroupHash, pVgroupList, &pVgCxt); + } else { + pVgCxt = *(SVgroupDataCxt**)p; } - - if ((code = sortMergeDataBlockDupRows(pOneTableBlock, &blkKeyInfo, &pBlkRowMerger)) != 0) { - tdFreeSBlockRowMerger(pBlkRowMerger); - taosHashCleanup(pVnodeDataBlockHashList); - insDestroyBlockArrayList(pVnodeDataBlockList); - taosMemoryFreeClear(dataBuf->pData); - taosMemoryFreeClear(blkKeyInfo.pKeyTuple); - return code; + if (TSDB_CODE_SUCCESS == code) { + code = fillVgroupDataCxt(pTableCxt, pVgCxt); } - ASSERT(blkKeyInfo.pKeyTuple != NULL && pBlocks->numOfRows > 0); - - // erase the empty space reserved for binary data - int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, blkKeyInfo.pKeyTuple); - - dataBuf->size += (finalLen + sizeof(SSubmitBlk)); - assert(dataBuf->size <= dataBuf->nAllocSize); - dataBuf->numOfTables += 1; } - - p = taosHashIterate(pHashObj, p); - if (p == NULL) { - break; + if (TSDB_CODE_SUCCESS == code) { + p = taosHashIterate(pTableHash, p); } + } - pOneTableBlock = *p; + taosHashCleanup(pVgroupHash); + if (TSDB_CODE_SUCCESS == code) { + *pVgDataBlocks = pVgroupList; + } else { + insDestroyVgroupDataCxtList(pVgroupList); } - // free the table data blocks; - tdFreeSBlockRowMerger(pBlkRowMerger); - taosHashCleanup(pVnodeDataBlockHashList); - taosMemoryFreeClear(blkKeyInfo.pKeyTuple); - *pVgDataBlocks = pVnodeDataBlockList; - return TSDB_CODE_SUCCESS; + return code; } -int32_t insAllocateMemForSize(STableDataBlocks* pDataBlock, int32_t allSize) { - size_t remain = pDataBlock->nAllocSize - pDataBlock->size; - uint32_t nAllocSizeOld = pDataBlock->nAllocSize; - - // expand the allocated size - if (remain < allSize) { - pDataBlock->nAllocSize = (pDataBlock->size + allSize) * 1.5; - - char* tmp = taosMemoryRealloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize); - if (tmp != NULL) { - pDataBlock->pData = tmp; - memset(pDataBlock->pData + pDataBlock->size, 0, pDataBlock->nAllocSize - pDataBlock->size); - } else { - // do nothing, if allocate more memory failed - pDataBlock->nAllocSize = nAllocSizeOld; +static int32_t buildSubmitReq(int32_t vgId, SSubmitReq2* pReq, void** pData, uint32_t* pLen) { + int32_t code = TSDB_CODE_SUCCESS; + uint32_t len = 0; + void* pBuf = NULL; + tEncodeSize(tEncodeSSubmitReq2, pReq, len, code); + if (TSDB_CODE_SUCCESS == code) { + SEncoder encoder; + len += sizeof(SMsgHead); + pBuf = taosMemoryMalloc(len); + if (NULL == pBuf) { return TSDB_CODE_OUT_OF_MEMORY; } + ((SMsgHead*)pBuf)->vgId = htonl(vgId); + ((SMsgHead*)pBuf)->contLen = htonl(len); + tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SMsgHead)), len - sizeof(SMsgHead)); + code = tEncodeSSubmitReq2(&encoder, pReq); + tEncoderClear(&encoder); } - return TSDB_CODE_SUCCESS; + if (TSDB_CODE_SUCCESS == code) { + *pData = pBuf; + *pLen = len; + } else { + taosMemoryFree(pBuf); + } + return code; } -int32_t insInitRowBuilder(SRowBuilder* pBuilder, int16_t schemaVer, SParsedDataColInfo* pColInfo) { - ASSERT(pColInfo->numOfCols > 0 && (pColInfo->numOfBound <= pColInfo->numOfCols)); - tdSRowInit(pBuilder, schemaVer); - tdSRowSetExtendedInfo(pBuilder, pColInfo->numOfCols, pColInfo->numOfBound, pColInfo->flen, pColInfo->allNullLen, - pColInfo->boundNullLen); - return TSDB_CODE_SUCCESS; +static void destroyVgDataBlocks(void* p) { + SVgDataBlocks* pVg = p; + taosMemoryFree(pVg->pData); + taosMemoryFree(pVg); } -static char* tableNameGetPosition(SToken* pToken, char target) { - bool inEscape = false; - bool inQuote = false; - char quotaStr = 0; - - for (uint32_t i = 0; i < pToken->n; ++i) { - if (*(pToken->z + i) == target && (!inEscape) && (!inQuote)) { - return pToken->z + i; - } - - if (*(pToken->z + i) == TS_ESCAPE_CHAR) { - if (!inQuote) { - inEscape = !inEscape; - } - } - - if (*(pToken->z + i) == '\'' || *(pToken->z + i) == '"') { - if (!inEscape) { - if (!inQuote) { - quotaStr = *(pToken->z + i); - inQuote = !inQuote; - } else if (quotaStr == *(pToken->z + i)) { - inQuote = !inQuote; - } - } - } +int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList, SArray** pVgDataBlocks) { + size_t numOfVg = taosArrayGetSize(pVgDataCxtList); + SArray* pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); + if (NULL == pDataBlocks) { + return TSDB_CODE_OUT_OF_MEMORY; } - return NULL; -} - -int32_t insCreateSName(SName* pName, SToken* pTableName, int32_t acctId, const char* dbName, SMsgBuf* pMsgBuf) { - const char* msg1 = "name too long"; - const char* msg2 = "invalid database name"; - const char* msg3 = "db is not specified"; - const char* msg4 = "invalid table name"; - int32_t code = TSDB_CODE_SUCCESS; - char* p = tableNameGetPosition(pTableName, TS_PATH_DELIMITER[0]); - - if (p != NULL) { // db has been specified in sql string so we ignore current db path - assert(*p == TS_PATH_DELIMITER[0]); - - int32_t dbLen = p - pTableName->z; - if (dbLen <= 0) { - return buildInvalidOperationMsg(pMsgBuf, msg2); + for (size_t i = 0; TSDB_CODE_SUCCESS == code && i < numOfVg; ++i) { + SVgroupDataCxt* src = taosArrayGetP(pVgDataCxtList, i); + SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); + if (NULL == dst) { + code = TSDB_CODE_OUT_OF_MEMORY; } - char name[TSDB_DB_FNAME_LEN] = {0}; - strncpy(name, pTableName->z, dbLen); - int32_t actualDbLen = strdequote(name); - - code = tNameSetDbName(pName, acctId, name, actualDbLen); - if (code != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg1); + if (TSDB_CODE_SUCCESS == code) { + dst->numOfTables = taosArrayGetSize(src->pData->aSubmitTbData); + code = taosHashGetDup(pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); } - - int32_t tbLen = pTableName->n - dbLen - 1; - if (tbLen <= 0) { - return buildInvalidOperationMsg(pMsgBuf, msg4); + if (TSDB_CODE_SUCCESS == code) { + code = buildSubmitReq(src->vgId, src->pData, &dst->pData, &dst->size); } - - char tbname[TSDB_TABLE_FNAME_LEN] = {0}; - strncpy(tbname, p + 1, tbLen); - /*tbLen = */ strdequote(tbname); - - code = tNameFromString(pName, tbname, T_NAME_TABLE); - if (code != 0) { - return buildInvalidOperationMsg(pMsgBuf, msg1); - } - } else { // get current DB name first, and then set it into path - if (pTableName->n >= TSDB_TABLE_NAME_LEN) { - return buildInvalidOperationMsg(pMsgBuf, msg1); + if (TSDB_CODE_SUCCESS == code) { + code = (NULL == taosArrayPush(pDataBlocks, &dst) ? TSDB_CODE_OUT_OF_MEMORY : TSDB_CODE_SUCCESS); } + } - assert(pTableName->n < TSDB_TABLE_FNAME_LEN); + if (TSDB_CODE_SUCCESS == code) { + *pVgDataBlocks = pDataBlocks; + } else { + taosArrayDestroyP(pDataBlocks, destroyVgDataBlocks); + } - char name[TSDB_TABLE_FNAME_LEN] = {0}; - strncpy(name, pTableName->z, pTableName->n); - strdequote(name); + return code; +} - if (dbName == NULL) { - return buildInvalidOperationMsg(pMsgBuf, msg3); - } +static int bindFileds(SBoundColInfo* pBoundInfo, SSchema* pSchema, TAOS_FIELD* fields, int numFields) { + bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool)); + if (NULL == pUseCols) { + return TSDB_CODE_OUT_OF_MEMORY; + } - code = tNameSetDbName(pName, acctId, dbName, strlen(dbName)); - if (code != TSDB_CODE_SUCCESS) { - code = buildInvalidOperationMsg(pMsgBuf, msg2); - return code; - } + pBoundInfo->numOfBound = 0; - code = tNameFromString(pName, name, T_NAME_TABLE); - if (code != 0) { - code = buildInvalidOperationMsg(pMsgBuf, msg1); + int16_t lastColIdx = -1; // last column found + int32_t code = TSDB_CODE_SUCCESS; + for (int i = 0; i < numFields; i++) { + SToken token; + token.z = fields[i].name; + token.n = strlen(fields[i].name); + + int16_t t = lastColIdx + 1; + int16_t index = insFindCol(&token, t, pBoundInfo->numOfCols, pSchema); + if (index < 0 && t > 0) { + index = insFindCol(&token, 0, t, pSchema); + } + if (index < 0) { + uError("can not find column name:%s", token.z); + code = TSDB_CODE_PAR_INVALID_COLUMN; + break; + } else if (pUseCols[index]) { + code = TSDB_CODE_PAR_INVALID_COLUMN; + uError("duplicated column name:%s", token.z); + break; + } else { + lastColIdx = index; + pUseCols[index] = true; + pBoundInfo->pColIndex[pBoundInfo->numOfBound] = index; + ++pBoundInfo->numOfBound; } } - if (NULL != strchr(pName->tname, '.')) { - code = generateSyntaxErrMsgExt(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, "The table name cannot contain '.'"); + if (TSDB_CODE_SUCCESS == code && !pUseCols[0]) { + uError("primary timestamp column can not be null:"); + code = TSDB_CODE_PAR_INVALID_COLUMN; } + taosMemoryFree(pUseCols); return code; } -int32_t insFindCol(SToken* pColname, int32_t start, int32_t end, SSchema* pSchema) { - while (start < end) { - if (strlen(pSchema[start].name) == pColname->n && strncmp(pColname->z, pSchema[start].name, pColname->n) == 0) { - return start; +int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD* tFields, + int numFields) { + STableDataCxt* pTableCxt = NULL; + int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, + sizeof(pTableMeta->uid), pTableMeta, &pCreateTb, &pTableCxt, true); + if (ret != TSDB_CODE_SUCCESS) { + uError("insGetTableDataCxt error"); + goto end; + } + if (tFields != NULL) { + ret = bindFileds(&pTableCxt->boundColsInfo, getTableColumnSchema(pTableMeta), tFields, numFields); + if (ret != TSDB_CODE_SUCCESS) { + uError("bindFileds error"); + goto end; } - ++start; } - return -1; -} + // no need to bind, because select * get all fields + ret = initTableColSubmitData(pTableCxt); + if (ret != TSDB_CODE_SUCCESS) { + uError("initTableColSubmitData error"); + goto end; + } -void insBuildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, int64_t suid, const char* sname, - SArray* tagName, uint8_t tagNum, int32_t ttl) { - pTbReq->type = TD_CHILD_TABLE; - pTbReq->name = strdup(tname); - pTbReq->ctb.suid = suid; - pTbReq->ctb.tagNum = tagNum; - if (sname) pTbReq->ctb.stbName = strdup(sname); - pTbReq->ctb.pTag = (uint8_t*)pTag; - pTbReq->ctb.tagName = taosArrayDup(tagName, NULL); - pTbReq->ttl = ttl; - pTbReq->commentLen = -1; + char* p = (char*)data; + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column + // length | + p += sizeof(int32_t); + p += sizeof(int32_t); - return; -} + int32_t numOfRows = *(int32_t*)p; + p += sizeof(int32_t); -int32_t insMemRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param) { - SMemParam* pa = (SMemParam*)param; - SRowBuilder* rb = pa->rb; + int32_t numOfCols = *(int32_t*)p; + p += sizeof(int32_t); - if (value == NULL) { // it is a null data - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, pa->colIdx); - return TSDB_CODE_SUCCESS; - } + p += sizeof(int32_t); + p += sizeof(uint64_t); - if (TSDB_DATA_TYPE_BINARY == pa->schema->type) { - const char* rowEnd = tdRowEnd(rb->pBuf); - STR_WITH_SIZE_TO_VARSTR(rowEnd, value, len); - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, false, pa->toffset, pa->colIdx); - } else if (TSDB_DATA_TYPE_NCHAR == pa->schema->type) { - // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' - int32_t output = 0; - const char* rowEnd = tdRowEnd(rb->pBuf); - if (!taosMbsToUcs4(value, len, (TdUcs4*)varDataVal(rowEnd), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { - if (errno == E2BIG) { - return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pa->schema->name); - } - char buf[512] = {0}; - snprintf(buf, tListLen(buf), "%s", strerror(errno)); - return buildSyntaxErrMsg(pMsgBuf, buf, value); - } - varDataSetLen(rowEnd, output); - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, false, pa->toffset, pa->colIdx); - } else { - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, value, false, pa->toffset, pa->colIdx); - } + int8_t* fields = p; + p += numOfCols * (sizeof(int8_t) + sizeof(int32_t)); - return TSDB_CODE_SUCCESS; -} + int32_t* colLength = (int32_t*)p; + p += sizeof(int32_t) * numOfCols; -int32_t insCheckTimestamp(STableDataBlocks* pDataBlocks, const char* start) { - // once the data block is disordered, we do NOT keep previous timestamp any more - if (!pDataBlocks->ordered) { - return TSDB_CODE_SUCCESS; - } - - TSKEY k = *(TSKEY*)start; - if (k <= pDataBlocks->prevTS) { - pDataBlocks->ordered = false; - } + char* pStart = p; - pDataBlocks->prevTS = k; - return TSDB_CODE_SUCCESS; -} + SSchema* pSchema = getTableColumnSchema(pTableCxt->pMeta); + SBoundColInfo* boundInfo = &pTableCxt->boundColsInfo; -static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { - SSubmitReq* submit = (SSubmitReq*)blocks->pData; - submit->header.vgId = htonl(blocks->vg.vgId); - submit->header.contLen = htonl(blocks->size); - submit->length = submit->header.contLen; - submit->numOfBlocks = htonl(blocks->numOfTables); - SSubmitBlk* blk = (SSubmitBlk*)(submit + 1); - int32_t numOfBlocks = blocks->numOfTables; - while (numOfBlocks--) { - int32_t dataLen = blk->dataLen; - int32_t schemaLen = blk->schemaLen; - blk->uid = htobe64(blk->uid); - blk->suid = htobe64(blk->suid); - blk->sversion = htonl(blk->sversion); - blk->dataLen = htonl(blk->dataLen); - blk->schemaLen = htonl(blk->schemaLen); - blk->numOfRows = htonl(blk->numOfRows); - blk = (SSubmitBlk*)(blk->data + schemaLen + dataLen); + if (boundInfo->numOfBound != numOfCols) { + uError("boundInfo->numOfBound:%d != numOfCols:%d", boundInfo->numOfBound, numOfCols); + ret = TSDB_CODE_INVALID_PARA; + goto end; } -} + for (int c = 0; c < boundInfo->numOfBound; ++c) { + SSchema* pColSchema = &pSchema[boundInfo->pColIndex[c]]; + SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, c); -int32_t insBuildOutput(SHashObj* pVgroupsHashObj, SArray* pVgDataBlocks, SArray** pDataBlocks) { - size_t numOfVg = taosArrayGetSize(pVgDataBlocks); - *pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); - if (NULL == *pDataBlocks) { - return TSDB_CODE_OUT_OF_MEMORY; - } - for (size_t i = 0; i < numOfVg; ++i) { - STableDataBlocks* src = taosArrayGetP(pVgDataBlocks, i); - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - return TSDB_CODE_OUT_OF_MEMORY; + if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { + uError("type or bytes not equal"); + ret = TSDB_CODE_INVALID_PARA; + goto end; + } + + colLength[c] = htonl(colLength[c]); + int8_t* offset = pStart; + if (IS_VAR_DATA_TYPE(pColSchema->type)) { + pStart += numOfRows * sizeof(int32_t); + } else { + pStart += BitmapLen(numOfRows); } - taosHashGetDup(pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); - dst->numOfTables = src->numOfTables; - dst->size = src->size; - TSWAP(dst->pData, src->pData); - buildMsgHeader(src, dst); - taosArrayPush(*pDataBlocks, &dst); + char* pData = pStart; + + tColDataAddValueByDataBlock(pCol, pColSchema->type, pColSchema->bytes, numOfRows, offset, pData); + fields += sizeof(int8_t) + sizeof(int32_t); + pStart += colLength[c]; } - return TSDB_CODE_SUCCESS; -} + +end: + return ret; +} \ No newline at end of file diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index e62b2f0f5ab2db843d41009bb21e68a60f19cd67..93bfe179f486775111d0f5aa9781b585cb583057 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -90,6 +90,7 @@ static SKeyword keywordTable[] = { {"EXISTS", TK_EXISTS}, {"EXPIRED", TK_EXPIRED}, {"EXPLAIN", TK_EXPLAIN}, + {"EVENT_WINDOW", TK_EVENT_WINDOW}, {"EVERY", TK_EVERY}, {"FILE", TK_FILE}, {"FILL", TK_FILL}, @@ -195,15 +196,16 @@ static SKeyword keywordTable[] = { {"SNODES", TK_SNODES}, {"SOFFSET", TK_SOFFSET}, {"SPLIT", TK_SPLIT}, - {"STT_TRIGGER", TK_STT_TRIGGER}, {"STABLE", TK_STABLE}, {"STABLES", TK_STABLES}, + {"START", TK_START}, {"STATE", TK_STATE}, {"STATE_WINDOW", TK_STATE_WINDOW}, {"STORAGE", TK_STORAGE}, {"STREAM", TK_STREAM}, {"STREAMS", TK_STREAMS}, {"STRICT", TK_STRICT}, + {"STT_TRIGGER", TK_STT_TRIGGER}, {"SUBSCRIBE", TK_SUBSCRIBE}, {"SUBSCRIPTIONS", TK_SUBSCRIPTIONS}, {"SUBTABLE", TK_SUBTABLE}, @@ -258,6 +260,7 @@ static SKeyword keywordTable[] = { {"WRITE", TK_WRITE}, {"_C0", TK_ROWTS}, {"_IROWTS", TK_IROWTS}, + {"_ISFILLED", TK_ISFILLED}, {"_QDURATION", TK_QDURATION}, {"_QEND", TK_QEND}, {"_QSTART", TK_QSTART}, @@ -266,6 +269,7 @@ static SKeyword keywordTable[] = { {"_WDURATION", TK_WDURATION}, {"_WEND", TK_WEND}, {"_WSTART", TK_WSTART}, + {"ALIVE", TK_ALIVE}, }; // clang-format on @@ -711,14 +715,3 @@ void taosCleanupKeywordsTable() { taosHashCleanup(m); } } - -SToken taosTokenDup(SToken* pToken, char* buf, int32_t len) { - assert(pToken != NULL && buf != NULL && len > pToken->n); - - strncpy(buf, pToken->z, pToken->n); - buf[pToken->n] = 0; - - SToken token = *pToken; - token.z = buf; - return token; -} diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 05d49bb0271190a6c1769a6d27abf06495efbdc8..df04d92599d3e3c743372f4ba2f68ca4c62b8fad 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -352,7 +352,7 @@ static int32_t getTableMetaImpl(STranslateContext* pCxt, const SName* pName, STa code = catalogGetTableMeta(pParCxt->pCatalog, &conn, pName, pMeta); } } - if (TSDB_CODE_SUCCESS != code) { + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_PAR_TABLE_NOT_EXIST != code) { parserError("0x%" PRIx64 " catalogGetTableMeta error, code:%s, dbName:%s, tbName:%s", pCxt->pParseCxt->requestId, tstrerror(code), pName->dbname, pName->tname); } @@ -3147,6 +3147,15 @@ static int32_t translateSessionWindow(STranslateContext* pCxt, SSelectStmt* pSel return TSDB_CODE_SUCCESS; } +static int32_t translateEventWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && + !isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TIMELINE_QUERY, + "EVENT_WINDOW requires valid time series input"); + } + return TSDB_CODE_SUCCESS; +} + static int32_t translateSpecificWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { switch (nodeType(pSelect->pWindow)) { case QUERY_NODE_STATE_WINDOW: @@ -3155,6 +3164,8 @@ static int32_t translateSpecificWindow(STranslateContext* pCxt, SSelectStmt* pSe return translateSessionWindow(pCxt, pSelect); case QUERY_NODE_INTERVAL_WINDOW: return translateIntervalWindow(pCxt, pSelect); + case QUERY_NODE_EVENT_WINDOW: + return translateEventWindow(pCxt, pSelect); default: break; } @@ -3891,7 +3902,8 @@ static int32_t checkDbKeepOption(STranslateContext* pCxt, SDatabaseOptions* pOpt if (pOptions->keep[0] < TSDB_MIN_KEEP || pOptions->keep[1] < TSDB_MIN_KEEP || pOptions->keep[2] < TSDB_MIN_KEEP || pOptions->keep[0] > tsdbMaxKeep || pOptions->keep[1] > tsdbMaxKeep || pOptions->keep[2] > tsdbMaxKeep) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, - "Invalid option keep: %" PRId64 ", %" PRId64 ", %" PRId64 " valid range: [%dm, %dm]", + "Invalid option keep: %" PRId64 ", %" PRId64 ", %" PRId64 + " valid range: [%dm, %" PRId64 "m]", pOptions->keep[0], pOptions->keep[1], pOptions->keep[2], TSDB_MIN_KEEP, tsdbMaxKeep); } @@ -4558,58 +4570,109 @@ typedef struct SSampleAstInfo { SNode* pSliding; SNodeList* pPartitionByList; STableMeta* pRollupTableMeta; + bool createSmaIndex; } SSampleAstInfo; -static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, char** pAst, int32_t* pLen, char** pExpr, - int32_t* pExprLen) { - SSelectStmt* pSelect = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); - if (NULL == pSelect) { - return TSDB_CODE_OUT_OF_MEMORY; - } - sprintf(pSelect->stmtName, "%p", pSelect); - +static int32_t buildTableForSampleAst(SSampleAstInfo* pInfo, SNode** pOutput) { SRealTableNode* pTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); if (NULL == pTable) { - nodesDestroyNode((SNode*)pSelect); return TSDB_CODE_OUT_OF_MEMORY; } snprintf(pTable->table.dbName, sizeof(pTable->table.dbName), "%s", pInfo->pDbName); snprintf(pTable->table.tableName, sizeof(pTable->table.tableName), "%s", pInfo->pTableName); TSWAP(pTable->pMeta, pInfo->pRollupTableMeta); - pSelect->pFromTable = (SNode*)pTable; + *pOutput = (SNode*)pTable; + return TSDB_CODE_SUCCESS; +} - TSWAP(pSelect->pProjectionList, pInfo->pFuncs); +static int32_t addWstartToSampleProjects(SNodeList* pProjectionList) { SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); - if (NULL == pSelect->pProjectionList || NULL == pFunc) { - nodesDestroyNode((SNode*)pSelect); + if (NULL == pFunc) { return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pFunc->functionName, "_wstart"); - nodesListPushFront(pSelect->pProjectionList, (SNode*)pFunc); - SNode* pProject = NULL; - FOREACH(pProject, pSelect->pProjectionList) { sprintf(((SExprNode*)pProject)->aliasName, "#%p", pProject); } + return nodesListPushFront(pProjectionList, (SNode*)pFunc); +} + +static int32_t addWendToSampleProjects(SNodeList* pProjectionList) { + SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(pFunc->functionName, "_wend"); + return nodesListAppend(pProjectionList, (SNode*)pFunc); +} - TSWAP(pSelect->pPartitionByList, pInfo->pPartitionByList); +static int32_t addWdurationToSampleProjects(SNodeList* pProjectionList) { + SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(pFunc->functionName, "_wduration"); + return nodesListAppend(pProjectionList, (SNode*)pFunc); +} + +static int32_t buildProjectsForSampleAst(SSampleAstInfo* pInfo, SNodeList** pList) { + SNodeList* pProjectionList = pInfo->pFuncs; + pInfo->pFuncs = NULL; + int32_t code = addWstartToSampleProjects(pProjectionList); + if (TSDB_CODE_SUCCESS == code && pInfo->createSmaIndex) { + code = addWendToSampleProjects(pProjectionList); + if (TSDB_CODE_SUCCESS == code) { + code = addWdurationToSampleProjects(pProjectionList); + } + } + + if (TSDB_CODE_SUCCESS == code) { + SNode* pProject = NULL; + FOREACH(pProject, pProjectionList) { sprintf(((SExprNode*)pProject)->aliasName, "#%p", pProject); } + *pList = pProjectionList; + } else { + nodesDestroyList(pProjectionList); + } + return code; +} + +static int32_t buildIntervalForSampleAst(SSampleAstInfo* pInfo, SNode** pOutput) { SIntervalWindowNode* pInterval = (SIntervalWindowNode*)nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW); if (NULL == pInterval) { - nodesDestroyNode((SNode*)pSelect); return TSDB_CODE_OUT_OF_MEMORY; } - pSelect->pWindow = (SNode*)pInterval; TSWAP(pInterval->pInterval, pInfo->pInterval); TSWAP(pInterval->pOffset, pInfo->pOffset); TSWAP(pInterval->pSliding, pInfo->pSliding); pInterval->pCol = nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pInterval->pCol) { - nodesDestroyNode((SNode*)pSelect); + nodesDestroyNode((SNode*)pInterval); return TSDB_CODE_OUT_OF_MEMORY; } ((SColumnNode*)pInterval->pCol)->colId = PRIMARYKEY_TIMESTAMP_COL_ID; strcpy(((SColumnNode*)pInterval->pCol)->colName, ROWTS_PSEUDO_COLUMN_NAME); + *pOutput = (SNode*)pInterval; + return TSDB_CODE_SUCCESS; +} - pCxt->createStream = true; - int32_t code = translateQuery(pCxt, (SNode*)pSelect); +static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, char** pAst, int32_t* pLen, char** pExpr, + int32_t* pExprLen) { + SSelectStmt* pSelect = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); + if (NULL == pSelect) { + return TSDB_CODE_OUT_OF_MEMORY; + } + sprintf(pSelect->stmtName, "%p", pSelect); + + int32_t code = buildTableForSampleAst(pInfo, &pSelect->pFromTable); + if (TSDB_CODE_SUCCESS == code) { + code = buildProjectsForSampleAst(pInfo, &pSelect->pProjectionList); + } + if (TSDB_CODE_SUCCESS == code) { + TSWAP(pSelect->pPartitionByList, pInfo->pPartitionByList); + code = buildIntervalForSampleAst(pInfo, &pSelect->pWindow); + } + if (TSDB_CODE_SUCCESS == code) { + pCxt->createStream = true; + code = translateQuery(pCxt, (SNode*)pSelect); + } if (TSDB_CODE_SUCCESS == code) { code = nodesNodeToString((SNode*)pSelect, false, pAst, pLen); } @@ -4917,10 +4980,10 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* return TSDB_CODE_SUCCESS; } -static SSchema* getColSchema(STableMeta* pTableMeta, const char* pColName) { +static const SSchema* getColSchema(const STableMeta* pTableMeta, const char* pColName) { int32_t numOfFields = getNumOfTags(pTableMeta) + getNumOfColumns(pTableMeta); for (int32_t i = 0; i < numOfFields; ++i) { - SSchema* pSchema = pTableMeta->schema + i; + const SSchema* pSchema = pTableMeta->schema + i; if (0 == strcmp(pColName, pSchema->name)) { return pSchema; } @@ -4940,7 +5003,8 @@ static SSchema* getTagSchema(STableMeta* pTableMeta, const char* pTagName) { return NULL; } -static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta) { +static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTableStmt* pStmt, + const STableMeta* pTableMeta) { SSchema* pTagsSchema = getTableTagSchema(pTableMeta); if (getNumOfTags(pTableMeta) == 1 && pTagsSchema->type == TSDB_DATA_TYPE_JSON && (pStmt->alterType == TSDB_ALTER_TABLE_ADD_TAG || pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG || @@ -4963,7 +5027,7 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "Table is not super table"); } - SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); + const SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); if (NULL == pSchema) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); } else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type || @@ -5076,7 +5140,7 @@ static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* p static int32_t translateCreateUser(STranslateContext* pCxt, SCreateUserStmt* pStmt) { SCreateUserReq createReq = {0}; - strcpy(createReq.user, pStmt->useName); + strcpy(createReq.user, pStmt->userName); createReq.createType = 0; createReq.superUser = 0; createReq.sysInfo = pStmt->sysinfo; @@ -5088,7 +5152,7 @@ static int32_t translateCreateUser(STranslateContext* pCxt, SCreateUserStmt* pSt static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt) { SAlterUserReq alterReq = {0}; - strcpy(alterReq.user, pStmt->useName); + strcpy(alterReq.user, pStmt->userName); alterReq.alterType = pStmt->alterType; alterReq.superUser = 0; alterReq.enable = pStmt->enable; @@ -5103,7 +5167,7 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt static int32_t translateDropUser(STranslateContext* pCxt, SDropUserStmt* pStmt) { SDropUserReq dropReq = {0}; - strcpy(dropReq.user, pStmt->useName); + strcpy(dropReq.user, pStmt->userName); return buildCmdMsg(pCxt, TDMT_MND_DROP_USER, (FSerializeFunc)tSerializeSDropUserReq, &dropReq); } @@ -5155,6 +5219,7 @@ static int32_t getSmaIndexSql(STranslateContext* pCxt, char** pSql, int32_t* pLe } static int32_t buildSampleAstInfoByIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SSampleAstInfo* pInfo) { + pInfo->createSmaIndex = true; pInfo->pDbName = pStmt->dbName; pInfo->pTableName = pStmt->tableName; pInfo->pFuncs = nodesCloneList(pStmt->pOptions->pFuncs); @@ -5666,6 +5731,144 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm return TSDB_CODE_SUCCESS; } +static int32_t adjustDataTypeOfProjections(STranslateContext* pCxt, const STableMeta* pMeta, SNodeList* pProjections) { + if (getNumOfColumns(pMeta) != LIST_LENGTH(pProjections)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM, "Illegal number of columns"); + } + + SSchema* pSchemas = getTableColumnSchema(pMeta); + int32_t index = 0; + SNode* pProj = NULL; + FOREACH(pProj, pProjections) { + SSchema* pSchema = pSchemas + index++; + SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes}; + if (!dataTypeEqual(&dt, &((SExprNode*)pProj)->resType)) { + SNode* pFunc = NULL; + int32_t code = createCastFunc(pCxt, pProj, dt, &pFunc); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + REPLACE_NODE(pFunc); + } + } + + return TSDB_CODE_SUCCESS; +} + +typedef struct SProjColPos { + int32_t colId; + SNode* pProj; +} SProjColPos; + +static int32_t projColPosCompar(const void* l, const void* r) { + return ((SProjColPos*)l)->colId > ((SProjColPos*)r)->colId; +} + +static void projColPosDelete(void* p) { taosMemoryFree(((SProjColPos*)p)->pProj); } + +static int32_t addProjToProjColPos(STranslateContext* pCxt, const SSchema* pSchema, SNode* pProj, SArray* pProjColPos) { + SNode* pNewProj = nodesCloneNode(pProj); + if (NULL == pNewProj) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes}; + if (!dataTypeEqual(&dt, &((SExprNode*)pNewProj)->resType)) { + SNode* pFunc = NULL; + code = createCastFunc(pCxt, pNewProj, dt, &pFunc); + pNewProj = pFunc; + } + if (TSDB_CODE_SUCCESS == code) { + SProjColPos pos = {.colId = pSchema->colId, .pProj = pNewProj}; + code = (NULL == taosArrayPush(pProjColPos, &pos) ? TSDB_CODE_OUT_OF_MEMORY : TSDB_CODE_SUCCESS); + } + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pNewProj); + } + return code; +} + +static int32_t adjustOrderOfProjection(STranslateContext* pCxt, SNodeList* pCols, const STableMeta* pMeta, + SNodeList** pProjections) { + if (LIST_LENGTH(pCols) != LIST_LENGTH(*pProjections)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM, "Illegal number of columns"); + } + + SArray* pProjColPos = taosArrayInit(LIST_LENGTH(pCols), sizeof(SProjColPos)); + if (NULL == pProjColPos) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + SNode* pCol = NULL; + SNode* pProj = NULL; + FORBOTH(pCol, pCols, pProj, *pProjections) { + const SSchema* pSchema = getColSchema(pMeta, ((SColumnNode*)pCol)->colName); + code = addProjToProjColPos(pCxt, pSchema, pProj, pProjColPos); + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + + SNodeList* pNewProjections = NULL; + if (TSDB_CODE_SUCCESS == code) { + taosArraySort(pProjColPos, projColPosCompar); + int32_t num = taosArrayGetSize(pProjColPos); + pNewProjections = nodesMakeList(); + if (NULL == pNewProjections) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + for (int32_t i = 0; TSDB_CODE_SUCCESS == code && i < num; ++i) { + SProjColPos* pPos = taosArrayGet(pProjColPos, i); + code = nodesListStrictAppend(pNewProjections, pPos->pProj); + pPos->pProj = NULL; + } + } + + if (TSDB_CODE_SUCCESS == code) { + taosArrayDestroy(pProjColPos); + nodesDestroyList(*pProjections); + *pProjections = pNewProjections; + } else { + taosArrayDestroyEx(pProjColPos, projColPosDelete); + nodesDestroyList(pNewProjections); + } + + return code; +} + +static int32_t adjustStreamQueryForExistTableImpl(STranslateContext* pCxt, SCreateStreamStmt* pStmt, + const STableMeta* pMeta) { + SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; + if (NULL == pStmt->pCols) { + return adjustDataTypeOfProjections(pCxt, pMeta, pSelect->pProjectionList); + } + return adjustOrderOfProjection(pCxt, pStmt->pCols, pMeta, &pSelect->pProjectionList); +} + +static int32_t adjustStreamQueryForExistTable(STranslateContext* pCxt, SCreateStreamStmt* pStmt, + SCMCreateStreamReq* pReq) { + STableMeta* pMeta = NULL; + int32_t code = getTableMeta(pCxt, pStmt->targetDbName, pStmt->targetTabName, &pMeta); + if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { + if (NULL != pStmt->pCols) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TABLE_NOT_EXIST, pStmt->targetTabName); + } + pReq->createStb = STREAM_CREATE_STABLE_TRUE; + pReq->targetStbUid = 0; + return TSDB_CODE_SUCCESS; + } else { + pReq->createStb = STREAM_CREATE_STABLE_FALSE; + pReq->targetStbUid = pMeta->suid; + } + if (TSDB_CODE_SUCCESS == code) { + code = adjustStreamQueryForExistTableImpl(pCxt, pStmt, pMeta); + } + taosMemoryFree(pMeta); + return code; +} + static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SCMCreateStreamReq* pReq) { pCxt->createStream = true; int32_t code = addSubtableInfoToCreateStreamQuery(pCxt, pStmt); @@ -5678,6 +5881,9 @@ static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt if (TSDB_CODE_SUCCESS == code) { code = checkStreamQuery(pCxt, pStmt); } + if (TSDB_CODE_SUCCESS == code) { + code = adjustStreamQueryForExistTable(pCxt, pStmt, pReq); + } if (TSDB_CODE_SUCCESS == code) { getSourceDatabase(pStmt->pQuery, pCxt->pParseCxt->acctId, pReq->sourceDB); code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL); @@ -6172,6 +6378,20 @@ static int32_t extractShowCreateDatabaseResultSchema(int32_t* numOfCols, SSchema return TSDB_CODE_SUCCESS; } +static int32_t extractShowAliveResultSchema(int32_t* numOfCols, SSchema** pSchema) { + *numOfCols = 1; + *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); + if (NULL == (*pSchema)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + (*pSchema)[0].type = TSDB_DATA_TYPE_INT; + (*pSchema)[0].bytes = sizeof(int32_t); + strcpy((*pSchema)[0].name, "status"); + + return TSDB_CODE_SUCCESS; +} + static int32_t extractShowCreateTableResultSchema(int32_t* numOfCols, SSchema** pSchema) { *numOfCols = 2; *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); @@ -6223,6 +6443,9 @@ int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pS return extractDescribeResultSchema(numOfCols, pSchema); case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: return extractShowCreateDatabaseResultSchema(numOfCols, pSchema); + case QUERY_NODE_SHOW_DB_ALIVE_STMT: + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: + return extractShowAliveResultSchema(numOfCols, pSchema); case QUERY_NODE_SHOW_CREATE_TABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT: return extractShowCreateTableResultSchema(numOfCols, pSchema); @@ -6327,7 +6550,7 @@ static int32_t createOperatorNode(EOperatorType opType, const char* pColName, SN nodesDestroyNode((SNode*)pOper); return TSDB_CODE_OUT_OF_MEMORY; } - strcpy(((SColumnNode*)pOper->pLeft)->colName, pColName); + snprintf(((SColumnNode*)pOper->pLeft)->colName, sizeof(((SColumnNode*)pOper->pLeft)->colName), "%s", pColName); *pOp = (SNode*)pOper; return TSDB_CODE_SUCCESS; @@ -6647,7 +6870,7 @@ static void destroyCreateTbReqBatch(void* data) { } int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray) { - SVnodeModifOpStmt* pNewStmt = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); + SVnodeModifyOpStmt* pNewStmt = (SVnodeModifyOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT); if (pNewStmt == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -7031,7 +7254,7 @@ SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap) { } static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) { - SCreateMultiTableStmt* pStmt = (SCreateMultiTableStmt*)pQuery->pRoot; + SCreateMultiTablesStmt* pStmt = (SCreateMultiTablesStmt*)pQuery->pRoot; SHashObj* pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); if (NULL == pVgroupHashmap) { @@ -7301,12 +7524,12 @@ static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, S return TSDB_CODE_SUCCESS; } -static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, +static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, const STableMeta* pTableMeta, SVAlterTbReq* pReq) { if (2 == getNumOfColumns(pTableMeta)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DROP_COL); } - SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); + const SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); if (NULL == pSchema) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); } else if (PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { @@ -7322,11 +7545,11 @@ static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, return TSDB_CODE_SUCCESS; } -static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, +static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, const STableMeta* pTableMeta, SVAlterTbReq* pReq) { pReq->colModBytes = calcTypeBytes(pStmt->dataType); pReq->colModType = pStmt->dataType.type; - SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); + const SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); if (NULL == pSchema) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); } else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type || @@ -7639,7 +7862,7 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { code = rewriteCreateTable(pCxt, pQuery); } break; - case QUERY_NODE_CREATE_MULTI_TABLE_STMT: + case QUERY_NODE_CREATE_MULTI_TABLES_STMT: code = rewriteCreateMultiTable(pCxt, pQuery); break; case QUERY_NODE_DROP_TABLE_STMT: @@ -7736,12 +7959,14 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->msgType = TDMT_VND_SUBMIT; break; - case QUERY_NODE_VNODE_MODIF_STMT: + case QUERY_NODE_VNODE_MODIFY_STMT: pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->msgType = toMsgType(((SVnodeModifOpStmt*)pQuery->pRoot)->sqlNodeType); + pQuery->msgType = toMsgType(((SVnodeModifyOpStmt*)pQuery->pRoot)->sqlNodeType); break; case QUERY_NODE_DESCRIBE_STMT: case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: + case QUERY_NODE_SHOW_DB_ALIVE_STMT: + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: case QUERY_NODE_SHOW_CREATE_TABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT: case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index d3b9bf069b08f65e4cada7b22530326347f772e7..7460664de2c955ff60f08faf6b739942ac914785 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -104,26 +104,26 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 459 +#define YYNOCODE 465 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SToken typedef union { int yyinit; ParseTOKENTYPE yy0; - EOperatorType yy20; - SNode* yy74; - ENullOrder yy109; - SToken yy317; - EOrder yy326; - bool yy335; - int8_t yy449; - int64_t yy531; - EJoinType yy630; - SAlterOption yy767; - EFillMode yy828; - int32_t yy856; + EJoinType yy42; + int8_t yy113; + int64_t yy159; + SToken yy179; + EOperatorType yy290; + EFillMode yy324; + SDataType yy394; + ENullOrder yy487; + SNode* yy602; + bool yy767; + int32_t yy820; + SAlterOption yy845; SNodeList* yy874; - SDataType yy898; + EOrder yy878; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -139,17 +139,17 @@ typedef union { #define ParseCTX_FETCH #define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 711 -#define YYNRULE 541 -#define YYNTOKEN 322 -#define YY_MAX_SHIFT 710 -#define YY_MIN_SHIFTREDUCE 1054 -#define YY_MAX_SHIFTREDUCE 1594 -#define YY_ERROR_ACTION 1595 -#define YY_ACCEPT_ACTION 1596 -#define YY_NO_ACTION 1597 -#define YY_MIN_REDUCE 1598 -#define YY_MAX_REDUCE 2138 +#define YYNSTATE 730 +#define YYNRULE 553 +#define YYNTOKEN 326 +#define YY_MAX_SHIFT 729 +#define YY_MIN_SHIFTREDUCE 1081 +#define YY_MAX_SHIFTREDUCE 1633 +#define YY_ERROR_ACTION 1634 +#define YY_ACCEPT_ACTION 1635 +#define YY_NO_ACTION 1636 +#define YY_MIN_REDUCE 1637 +#define YY_MAX_REDUCE 2189 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -216,742 +216,740 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2723) +#define YY_ACTTAB_COUNT (2706) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 459, 353, 460, 1634, 566, 600, 578, 2114, 2109, 155, - /* 10 */ 1940, 2109, 43, 41, 1525, 39, 38, 37, 1753, 123, - /* 20 */ 361, 1936, 1375, 565, 173, 599, 498, 2113, 2110, 567, - /* 30 */ 1954, 2110, 2112, 1455, 1400, 1373, 1751, 131, 468, 406, - /* 40 */ 460, 1634, 36, 35, 1940, 586, 42, 40, 39, 38, - /* 50 */ 37, 1932, 1938, 344, 541, 1936, 167, 578, 1450, 465, - /* 60 */ 585, 1972, 610, 16, 366, 461, 1804, 1797, 1799, 581, - /* 70 */ 1381, 599, 1401, 330, 1922, 158, 616, 333, 1851, 319, - /* 80 */ 1705, 1972, 1802, 43, 41, 1932, 1938, 356, 131, 560, - /* 90 */ 477, 361, 475, 1375, 1860, 12, 610, 326, 172, 2049, - /* 100 */ 2050, 1953, 129, 2054, 1455, 1988, 1373, 2113, 100, 1955, - /* 110 */ 620, 1957, 1958, 615, 600, 610, 1106, 707, 1105, 654, - /* 120 */ 170, 458, 2041, 599, 463, 1640, 355, 2037, 123, 1450, - /* 130 */ 559, 210, 1457, 1458, 16, 503, 33, 275, 1484, 566, - /* 140 */ 175, 1381, 157, 2109, 1610, 1751, 46, 1107, 2067, 255, - /* 150 */ 2049, 577, 578, 124, 576, 1621, 1954, 2109, 565, 173, - /* 160 */ 600, 1431, 1440, 2110, 567, 578, 12, 42, 40, 39, - /* 170 */ 38, 37, 565, 173, 52, 1309, 1310, 2110, 567, 97, - /* 180 */ 1376, 586, 1374, 131, 1256, 1257, 467, 1972, 707, 463, - /* 190 */ 1640, 1751, 1402, 132, 1485, 614, 131, 561, 46, 1922, - /* 200 */ 1922, 1743, 616, 1457, 1458, 1379, 1380, 62, 1430, 1433, - /* 210 */ 1434, 1435, 1436, 1437, 1438, 1439, 612, 608, 1448, 1449, - /* 220 */ 1451, 1452, 1453, 1454, 1456, 1459, 2, 1953, 595, 58, - /* 230 */ 1860, 1988, 1431, 1440, 311, 1955, 620, 1957, 1958, 615, - /* 240 */ 613, 610, 601, 2006, 174, 2049, 2050, 1400, 129, 2054, - /* 250 */ 338, 1376, 2056, 1374, 1798, 1799, 580, 171, 2049, 2050, - /* 260 */ 556, 129, 2054, 1620, 58, 32, 359, 1479, 1480, 1481, - /* 270 */ 1482, 1483, 1487, 1488, 1489, 1490, 1379, 1380, 2053, 1430, - /* 280 */ 1433, 1434, 1435, 1436, 1437, 1438, 1439, 612, 608, 1448, - /* 290 */ 1449, 1451, 1452, 1453, 1454, 1456, 1459, 2, 58, 9, - /* 300 */ 43, 41, 1667, 47, 1551, 1596, 1847, 1922, 361, 1529, - /* 310 */ 1375, 339, 1729, 337, 336, 1400, 500, 181, 1954, 1619, - /* 320 */ 502, 1455, 58, 1373, 1213, 642, 641, 640, 1217, 639, - /* 330 */ 1219, 1220, 638, 1222, 635, 257, 1228, 632, 1230, 1231, - /* 340 */ 629, 626, 501, 562, 557, 258, 1450, 176, 399, 1972, - /* 350 */ 398, 16, 553, 1549, 1550, 1552, 1553, 617, 1381, 1106, - /* 360 */ 1173, 1105, 1922, 1922, 616, 569, 395, 513, 512, 511, - /* 370 */ 376, 43, 41, 1460, 477, 128, 507, 400, 176, 361, - /* 380 */ 506, 1375, 58, 12, 85, 505, 510, 397, 393, 1953, - /* 390 */ 1107, 504, 1455, 1988, 1373, 1175, 100, 1955, 620, 1957, - /* 400 */ 1958, 615, 1954, 610, 80, 707, 134, 1618, 141, 2012, - /* 410 */ 2041, 600, 546, 176, 355, 2037, 2109, 1450, 127, 546, - /* 420 */ 1457, 1458, 169, 2109, 1401, 178, 226, 1746, 573, 1381, - /* 430 */ 1617, 2115, 173, 1972, 1599, 1791, 2110, 567, 2115, 173, - /* 440 */ 1375, 617, 1751, 2110, 567, 167, 1922, 176, 616, 1431, - /* 450 */ 1440, 1922, 176, 1373, 44, 113, 1432, 587, 112, 111, - /* 460 */ 110, 109, 108, 107, 106, 105, 104, 1852, 1376, 352, - /* 470 */ 1374, 176, 1864, 1953, 1922, 227, 707, 1988, 1781, 1804, - /* 480 */ 160, 1955, 620, 1957, 1958, 615, 354, 610, 1381, 1352, - /* 490 */ 1353, 1457, 1458, 1379, 1380, 1802, 1430, 1433, 1434, 1435, - /* 500 */ 1436, 1437, 1438, 1439, 612, 608, 1448, 1449, 1451, 1452, - /* 510 */ 1453, 1454, 1456, 1459, 2, 509, 508, 678, 676, 113, - /* 520 */ 1431, 1440, 112, 111, 110, 109, 108, 107, 106, 105, - /* 530 */ 104, 176, 568, 2130, 364, 707, 513, 512, 511, 1376, - /* 540 */ 367, 1374, 155, 1804, 128, 507, 1465, 1909, 155, 506, - /* 550 */ 365, 1753, 1400, 1403, 505, 510, 1728, 1753, 1954, 1802, - /* 560 */ 504, 266, 267, 1400, 1379, 1380, 265, 1430, 1433, 1434, - /* 570 */ 1435, 1436, 1437, 1438, 1439, 612, 608, 1448, 1449, 1451, - /* 580 */ 1452, 1453, 1454, 1456, 1459, 2, 43, 41, 1598, 1972, - /* 590 */ 1616, 600, 185, 527, 361, 383, 1375, 581, 1376, 1727, - /* 600 */ 1374, 1615, 1922, 1399, 616, 404, 525, 1455, 523, 1373, - /* 610 */ 1742, 600, 122, 121, 120, 119, 118, 117, 116, 115, - /* 620 */ 114, 1936, 1751, 1379, 1380, 405, 11, 10, 77, 1953, - /* 630 */ 1614, 76, 1450, 1988, 1922, 1613, 100, 1955, 620, 1957, - /* 640 */ 1958, 615, 1751, 610, 1381, 1922, 91, 80, 170, 574, - /* 650 */ 2041, 1932, 1938, 133, 355, 2037, 2012, 43, 41, 600, - /* 660 */ 518, 654, 610, 652, 2114, 361, 1954, 1375, 1744, 44, - /* 670 */ 1747, 442, 1612, 414, 1922, 528, 2068, 587, 1455, 1922, - /* 680 */ 1373, 1402, 146, 145, 649, 648, 647, 143, 29, 224, - /* 690 */ 1751, 707, 1865, 1432, 36, 35, 182, 1972, 42, 40, - /* 700 */ 39, 38, 37, 1450, 521, 617, 1457, 1458, 1609, 515, - /* 710 */ 1922, 1608, 616, 2056, 223, 1381, 1922, 1607, 2114, 36, - /* 720 */ 35, 1498, 2109, 42, 40, 39, 38, 37, 1522, 189, - /* 730 */ 188, 1403, 1641, 1486, 2056, 1431, 1440, 1953, 2113, 2052, - /* 740 */ 12, 1988, 2110, 2111, 101, 1955, 620, 1957, 1958, 615, - /* 750 */ 64, 610, 1922, 63, 1376, 1922, 1374, 1584, 2041, 31, - /* 760 */ 2051, 1922, 707, 2038, 1606, 36, 35, 1605, 225, 42, - /* 770 */ 40, 39, 38, 37, 9, 1604, 1087, 1457, 1458, 1379, - /* 780 */ 1380, 702, 1430, 1433, 1434, 1435, 1436, 1437, 1438, 1439, - /* 790 */ 612, 608, 1448, 1449, 1451, 1452, 1453, 1454, 1456, 1459, - /* 800 */ 2, 318, 1384, 1398, 30, 1603, 1431, 1440, 1922, 710, - /* 810 */ 436, 1922, 155, 449, 1491, 1089, 448, 1092, 1093, 1922, - /* 820 */ 602, 1754, 2013, 282, 257, 1376, 1664, 1374, 82, 321, - /* 830 */ 1847, 420, 531, 450, 529, 9, 422, 7, 166, 1834, - /* 840 */ 604, 183, 2013, 6, 700, 696, 692, 688, 280, 1922, - /* 850 */ 1379, 1380, 1726, 1430, 1433, 1434, 1435, 1436, 1437, 1438, - /* 860 */ 1439, 612, 608, 1448, 1449, 1451, 1452, 1453, 1454, 1456, - /* 870 */ 1459, 2, 1804, 1403, 235, 36, 35, 334, 1400, 42, - /* 880 */ 40, 39, 38, 37, 98, 1847, 607, 273, 1803, 410, - /* 890 */ 645, 684, 683, 682, 681, 371, 187, 680, 679, 135, - /* 900 */ 674, 673, 672, 671, 670, 669, 668, 148, 664, 663, - /* 910 */ 662, 370, 369, 659, 658, 657, 656, 655, 646, 446, - /* 920 */ 596, 1795, 441, 440, 439, 438, 435, 434, 433, 432, - /* 930 */ 431, 427, 426, 425, 424, 335, 417, 416, 415, 1381, - /* 940 */ 412, 411, 332, 156, 1092, 1093, 36, 35, 295, 1387, - /* 950 */ 42, 40, 39, 38, 37, 1602, 1601, 260, 1954, 652, - /* 960 */ 666, 650, 293, 66, 1795, 289, 65, 651, 1781, 1521, - /* 970 */ 1795, 667, 1740, 1721, 1346, 48, 229, 3, 146, 145, - /* 980 */ 649, 648, 647, 143, 193, 455, 453, 36, 35, 1972, - /* 990 */ 423, 42, 40, 39, 38, 37, 502, 617, 1943, 1922, - /* 1000 */ 1922, 600, 1922, 652, 616, 1736, 36, 35, 2061, 1518, - /* 1010 */ 42, 40, 39, 38, 37, 428, 234, 233, 501, 1432, - /* 1020 */ 58, 600, 146, 145, 649, 648, 647, 143, 1476, 1953, - /* 1030 */ 154, 600, 1751, 1988, 1561, 429, 100, 1955, 620, 1957, - /* 1040 */ 1958, 615, 25, 610, 144, 476, 1945, 1518, 2129, 137, - /* 1050 */ 2041, 125, 1751, 407, 355, 2037, 83, 1738, 67, 99, - /* 1060 */ 600, 1954, 1751, 36, 35, 2075, 408, 42, 40, 39, - /* 1070 */ 38, 37, 36, 35, 1748, 1734, 42, 40, 39, 38, - /* 1080 */ 37, 600, 216, 218, 570, 214, 217, 600, 1654, 1383, - /* 1090 */ 220, 1751, 1972, 219, 1647, 139, 51, 74, 73, 403, - /* 1100 */ 617, 542, 180, 600, 60, 1922, 1954, 616, 75, 1645, - /* 1110 */ 514, 222, 1751, 1941, 221, 239, 516, 582, 1751, 45, - /* 1120 */ 317, 1593, 1594, 391, 1936, 389, 385, 381, 378, 375, - /* 1130 */ 263, 519, 1953, 600, 1751, 600, 1988, 1972, 1611, 100, - /* 1140 */ 1955, 620, 1957, 1958, 615, 617, 610, 270, 140, 597, - /* 1150 */ 1922, 2129, 616, 2041, 1932, 1938, 1548, 355, 2037, 1706, - /* 1160 */ 1541, 600, 600, 230, 1751, 610, 1751, 241, 2103, 176, - /* 1170 */ 142, 1322, 611, 50, 144, 598, 276, 1953, 545, 1954, - /* 1180 */ 60, 1988, 268, 252, 100, 1955, 620, 1957, 1958, 615, - /* 1190 */ 660, 610, 1751, 1751, 358, 357, 2129, 600, 2041, 45, - /* 1200 */ 592, 45, 355, 2037, 1389, 11, 10, 661, 644, 96, - /* 1210 */ 1972, 368, 1154, 2060, 2081, 1455, 624, 1382, 617, 93, - /* 1220 */ 211, 584, 272, 1922, 374, 616, 1206, 1591, 1751, 1152, - /* 1230 */ 209, 554, 1492, 246, 1954, 162, 1386, 142, 1973, 372, - /* 1240 */ 1450, 494, 490, 486, 482, 208, 144, 1856, 1635, 126, - /* 1250 */ 1953, 1441, 1381, 288, 1988, 1792, 142, 100, 1955, 620, - /* 1260 */ 1957, 1958, 615, 546, 610, 1972, 546, 2109, 1234, 2016, - /* 1270 */ 2109, 2041, 1135, 617, 579, 355, 2037, 2071, 1922, 251, - /* 1280 */ 616, 81, 2115, 173, 206, 2115, 173, 2110, 567, 1238, - /* 1290 */ 2110, 567, 254, 1, 373, 4, 382, 331, 1245, 606, - /* 1300 */ 377, 1243, 1339, 571, 283, 1953, 186, 1136, 147, 1988, - /* 1310 */ 409, 1403, 100, 1955, 620, 1957, 1958, 615, 1857, 610, - /* 1320 */ 413, 1590, 444, 418, 2014, 1398, 2041, 430, 1849, 437, - /* 1330 */ 355, 2037, 443, 451, 452, 445, 546, 190, 454, 456, - /* 1340 */ 2109, 1404, 457, 466, 1406, 470, 1954, 534, 469, 1405, - /* 1350 */ 196, 205, 199, 198, 204, 2115, 173, 473, 471, 1407, - /* 1360 */ 2110, 567, 1390, 472, 1385, 201, 474, 203, 78, 79, - /* 1370 */ 478, 1109, 207, 197, 495, 496, 1954, 1972, 499, 497, - /* 1380 */ 103, 1741, 1899, 213, 320, 617, 1898, 1393, 1395, 546, - /* 1390 */ 1922, 1737, 616, 2109, 533, 284, 535, 215, 149, 608, - /* 1400 */ 1448, 1449, 1451, 1452, 1453, 1454, 150, 1972, 2115, 173, - /* 1410 */ 228, 536, 1739, 2110, 567, 617, 540, 1953, 1735, 151, - /* 1420 */ 1922, 1988, 616, 152, 100, 1955, 620, 1957, 1958, 615, - /* 1430 */ 231, 610, 1954, 543, 550, 537, 603, 2072, 2041, 590, - /* 1440 */ 2082, 555, 355, 2037, 5, 552, 345, 1953, 2087, 558, - /* 1450 */ 237, 1988, 1954, 240, 101, 1955, 620, 1957, 1958, 615, - /* 1460 */ 2063, 610, 2086, 1972, 564, 548, 551, 549, 2041, 572, - /* 1470 */ 250, 617, 2040, 2037, 247, 245, 1922, 346, 616, 575, - /* 1480 */ 130, 248, 1518, 1972, 249, 163, 1402, 2057, 349, 583, - /* 1490 */ 259, 617, 1408, 1861, 285, 593, 1922, 588, 616, 286, - /* 1500 */ 589, 1870, 1869, 1953, 1868, 2108, 351, 1988, 88, 1752, - /* 1510 */ 101, 1955, 620, 1957, 1958, 615, 287, 610, 90, 594, - /* 1520 */ 1954, 253, 57, 618, 2041, 2022, 2132, 1988, 605, 2037, - /* 1530 */ 101, 1955, 620, 1957, 1958, 615, 1954, 610, 92, 622, - /* 1540 */ 279, 703, 1796, 1722, 2041, 704, 290, 706, 325, 2037, - /* 1550 */ 49, 1972, 314, 322, 323, 294, 1916, 1915, 71, 617, - /* 1560 */ 72, 292, 1914, 1913, 1922, 1910, 616, 1972, 379, 299, - /* 1570 */ 1367, 313, 303, 380, 1368, 617, 179, 1908, 384, 386, - /* 1580 */ 1922, 387, 616, 388, 1907, 390, 1906, 392, 1905, 394, - /* 1590 */ 1904, 1953, 1342, 396, 1341, 1988, 1881, 1954, 159, 1955, - /* 1600 */ 620, 1957, 1958, 615, 1880, 610, 401, 1953, 402, 1879, - /* 1610 */ 1878, 1988, 1842, 1300, 159, 1955, 620, 1957, 1958, 615, - /* 1620 */ 1841, 610, 1839, 136, 1838, 1837, 1840, 1836, 1972, 184, - /* 1630 */ 419, 1830, 421, 1829, 1828, 1827, 617, 1835, 1833, 547, - /* 1640 */ 2078, 1922, 1954, 616, 1832, 1831, 1826, 1825, 1824, 1823, - /* 1650 */ 1822, 1821, 1820, 1819, 1818, 1817, 2079, 1816, 1815, 138, - /* 1660 */ 1814, 1813, 1812, 1811, 1810, 1302, 1809, 1808, 1953, 1807, - /* 1670 */ 1806, 447, 1988, 1972, 1181, 305, 1955, 620, 1957, 1958, - /* 1680 */ 615, 617, 610, 1805, 1669, 1668, 1922, 1666, 616, 1630, - /* 1690 */ 191, 1095, 1094, 1629, 192, 1894, 1888, 1877, 194, 202, - /* 1700 */ 1876, 1859, 1730, 1954, 69, 1128, 1942, 1665, 1663, 1661, - /* 1710 */ 1659, 70, 168, 1953, 479, 195, 1657, 1988, 462, 563, - /* 1720 */ 160, 1955, 620, 1957, 1958, 615, 1954, 610, 464, 200, - /* 1730 */ 480, 481, 1644, 483, 1972, 1643, 484, 1626, 485, 350, - /* 1740 */ 487, 491, 617, 489, 1732, 1249, 59, 1922, 1731, 616, - /* 1750 */ 1250, 488, 493, 1172, 1171, 1170, 1169, 1972, 675, 492, - /* 1760 */ 1166, 677, 1164, 1165, 1163, 614, 1655, 340, 1648, 341, - /* 1770 */ 1922, 1646, 616, 2131, 1953, 212, 517, 342, 1988, 1625, - /* 1780 */ 1624, 312, 1955, 620, 1957, 1958, 615, 1954, 610, 520, - /* 1790 */ 1623, 522, 524, 102, 526, 1357, 1356, 1953, 530, 1893, - /* 1800 */ 1348, 1988, 1887, 53, 311, 1955, 620, 1957, 1958, 615, - /* 1810 */ 1875, 610, 1873, 2007, 538, 1874, 2114, 1954, 1972, 17, - /* 1820 */ 1359, 14, 56, 360, 244, 24, 617, 1563, 26, 236, - /* 1830 */ 243, 1922, 238, 616, 1547, 1540, 161, 242, 1943, 28, - /* 1840 */ 1954, 27, 18, 539, 84, 61, 1583, 1584, 1972, 19, - /* 1850 */ 15, 232, 1578, 362, 343, 1577, 617, 347, 1953, 153, - /* 1860 */ 1582, 1922, 1988, 616, 1581, 312, 1955, 620, 1957, 1958, - /* 1870 */ 615, 1972, 610, 348, 256, 55, 1872, 1515, 1514, 617, - /* 1880 */ 164, 544, 1871, 261, 1922, 1954, 616, 1858, 1953, 20, - /* 1890 */ 87, 264, 1988, 262, 1545, 312, 1955, 620, 1957, 1958, - /* 1900 */ 615, 1954, 610, 269, 86, 93, 89, 274, 21, 10, - /* 1910 */ 1391, 532, 1445, 1991, 609, 1988, 1972, 271, 307, 1955, - /* 1920 */ 620, 1957, 1958, 615, 617, 610, 1443, 34, 591, 1922, - /* 1930 */ 1442, 616, 1972, 165, 13, 22, 54, 177, 1423, 1415, - /* 1940 */ 617, 23, 625, 621, 1235, 1922, 623, 616, 363, 1467, - /* 1950 */ 1232, 628, 627, 630, 1954, 1229, 1953, 1477, 631, 633, - /* 1960 */ 1988, 1223, 634, 296, 1955, 620, 1957, 1958, 615, 1221, - /* 1970 */ 610, 619, 1953, 636, 637, 1212, 1988, 1227, 643, 297, - /* 1980 */ 1955, 620, 1957, 1958, 615, 1972, 610, 8, 1226, 1225, - /* 1990 */ 1466, 1224, 94, 617, 277, 95, 1244, 1240, 1922, 1954, - /* 2000 */ 616, 68, 1160, 1126, 653, 1159, 1158, 1157, 1156, 1155, - /* 2010 */ 1179, 1153, 1151, 1150, 1149, 665, 1147, 1146, 1145, 1144, - /* 2020 */ 1143, 1142, 1141, 1174, 278, 1953, 1176, 1138, 1137, 1988, - /* 2030 */ 1972, 1134, 298, 1955, 620, 1957, 1958, 615, 617, 610, - /* 2040 */ 1133, 1132, 1131, 1922, 1662, 616, 685, 686, 687, 1660, - /* 2050 */ 689, 690, 1954, 691, 1658, 693, 694, 695, 1656, 697, - /* 2060 */ 698, 699, 1642, 701, 1084, 1622, 709, 281, 1954, 705, - /* 2070 */ 1953, 1597, 1377, 291, 1988, 1597, 708, 304, 1955, 620, - /* 2080 */ 1957, 1958, 615, 1972, 610, 1597, 1597, 1597, 1597, 1597, - /* 2090 */ 1597, 617, 1597, 1597, 1597, 1597, 1922, 1597, 616, 1972, - /* 2100 */ 1597, 1597, 1597, 1597, 1597, 1597, 1597, 617, 1597, 1597, - /* 2110 */ 1597, 1597, 1922, 1954, 616, 1597, 1597, 1597, 1597, 1597, - /* 2120 */ 1597, 1597, 1597, 1953, 1597, 1597, 1597, 1988, 1597, 1597, - /* 2130 */ 308, 1955, 620, 1957, 1958, 615, 1597, 610, 1597, 1953, - /* 2140 */ 1597, 1597, 1597, 1988, 1972, 1597, 300, 1955, 620, 1957, - /* 2150 */ 1958, 615, 617, 610, 1597, 1597, 1597, 1922, 1597, 616, - /* 2160 */ 1597, 1597, 1597, 1597, 1597, 1597, 1954, 1597, 1597, 1597, - /* 2170 */ 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1597, - /* 2180 */ 1597, 1597, 1954, 1597, 1953, 1597, 1597, 1597, 1988, 1597, - /* 2190 */ 1597, 309, 1955, 620, 1957, 1958, 615, 1972, 610, 1597, - /* 2200 */ 1597, 1597, 1597, 1597, 1597, 617, 1597, 1597, 1597, 1597, - /* 2210 */ 1922, 1597, 616, 1972, 1597, 1597, 1597, 1597, 1597, 1597, - /* 2220 */ 1597, 617, 1597, 1597, 1597, 1597, 1922, 1954, 616, 1597, - /* 2230 */ 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1953, 1597, 1597, - /* 2240 */ 1597, 1988, 1597, 1597, 301, 1955, 620, 1957, 1958, 615, - /* 2250 */ 1954, 610, 1597, 1953, 1597, 1597, 1597, 1988, 1972, 1597, - /* 2260 */ 310, 1955, 620, 1957, 1958, 615, 617, 610, 1597, 1597, - /* 2270 */ 1597, 1922, 1597, 616, 1597, 1597, 1597, 1597, 1597, 1597, - /* 2280 */ 1954, 1972, 1597, 1597, 1597, 1597, 1597, 1597, 1597, 617, - /* 2290 */ 1597, 1597, 1597, 1597, 1922, 1597, 616, 1597, 1953, 1597, - /* 2300 */ 1597, 1597, 1988, 1597, 1597, 302, 1955, 620, 1957, 1958, - /* 2310 */ 615, 1972, 610, 1597, 1597, 1597, 1597, 1597, 1597, 617, - /* 2320 */ 1597, 1953, 1597, 1597, 1922, 1988, 616, 1597, 315, 1955, - /* 2330 */ 620, 1957, 1958, 615, 1597, 610, 1597, 1597, 1597, 1597, - /* 2340 */ 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1597, - /* 2350 */ 1597, 1953, 1597, 1597, 1597, 1988, 1954, 1597, 316, 1955, - /* 2360 */ 620, 1957, 1958, 615, 1597, 610, 1597, 1597, 1597, 1597, - /* 2370 */ 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1954, 1597, - /* 2380 */ 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1972, 1597, 1597, - /* 2390 */ 1597, 1597, 1597, 1597, 1597, 617, 1597, 1597, 1597, 1597, - /* 2400 */ 1922, 1597, 616, 1597, 1597, 1597, 1597, 1597, 1597, 1972, - /* 2410 */ 1597, 1597, 1597, 1597, 1597, 1597, 1597, 617, 1597, 1597, - /* 2420 */ 1597, 1597, 1922, 1597, 616, 1597, 1597, 1953, 1597, 1597, - /* 2430 */ 1597, 1988, 1597, 1954, 1966, 1955, 620, 1957, 1958, 615, - /* 2440 */ 1597, 610, 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1953, - /* 2450 */ 1597, 1597, 1597, 1988, 1597, 1597, 1965, 1955, 620, 1957, - /* 2460 */ 1958, 615, 1597, 610, 1972, 1597, 1597, 1597, 1597, 1597, - /* 2470 */ 1597, 1597, 617, 1597, 1597, 1597, 1597, 1922, 1954, 616, - /* 2480 */ 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1597, - /* 2490 */ 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1597, - /* 2500 */ 1597, 1597, 1597, 1597, 1953, 1597, 1597, 1597, 1988, 1972, - /* 2510 */ 1597, 1964, 1955, 620, 1957, 1958, 615, 617, 610, 1597, - /* 2520 */ 1597, 1597, 1922, 1954, 616, 1597, 1597, 1597, 1597, 1597, - /* 2530 */ 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1954, - /* 2540 */ 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1953, - /* 2550 */ 1597, 1597, 1597, 1988, 1972, 1597, 327, 1955, 620, 1957, - /* 2560 */ 1958, 615, 617, 610, 1597, 1597, 1597, 1922, 1597, 616, - /* 2570 */ 1972, 1597, 1597, 1597, 1597, 1597, 1597, 1597, 617, 1597, - /* 2580 */ 1597, 1597, 1597, 1922, 1954, 616, 1597, 1597, 1597, 1597, - /* 2590 */ 1597, 1597, 1597, 1597, 1953, 1597, 1597, 1597, 1988, 1597, - /* 2600 */ 1597, 328, 1955, 620, 1957, 1958, 615, 1954, 610, 1597, - /* 2610 */ 1953, 1597, 1597, 1597, 1988, 1972, 1597, 324, 1955, 620, - /* 2620 */ 1957, 1958, 615, 617, 610, 1597, 1597, 1597, 1922, 1597, - /* 2630 */ 616, 1597, 1597, 1597, 1597, 1597, 1597, 1954, 1972, 1597, - /* 2640 */ 1597, 1597, 1597, 1597, 1597, 1597, 617, 1597, 1597, 1597, - /* 2650 */ 1597, 1922, 1597, 616, 1597, 1953, 1597, 1597, 1597, 1988, - /* 2660 */ 1597, 1597, 329, 1955, 620, 1957, 1958, 615, 1972, 610, - /* 2670 */ 1597, 1597, 1597, 1597, 1597, 1597, 617, 1597, 618, 1597, - /* 2680 */ 1597, 1922, 1988, 616, 1597, 307, 1955, 620, 1957, 1958, - /* 2690 */ 615, 1597, 610, 1597, 1597, 1597, 1597, 1597, 1597, 1597, - /* 2700 */ 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1597, 1953, 1597, - /* 2710 */ 1597, 1597, 1988, 1597, 1597, 306, 1955, 620, 1957, 1958, - /* 2720 */ 615, 1597, 610, + /* 0 */ 1909, 471, 376, 472, 1673, 1836, 1838, 480, 1989, 472, + /* 10 */ 1673, 1909, 45, 43, 1563, 1907, 606, 173, 1650, 1985, + /* 20 */ 371, 1989, 1412, 362, 1843, 1781, 1906, 606, 2003, 35, + /* 30 */ 283, 339, 1985, 1493, 1660, 1410, 1985, 618, 342, 1892, + /* 40 */ 1841, 470, 38, 37, 475, 1679, 44, 42, 41, 40, + /* 50 */ 39, 1981, 1987, 353, 416, 594, 604, 582, 1488, 2021, + /* 60 */ 618, 2160, 629, 18, 1981, 1987, 366, 597, 1981, 1987, + /* 70 */ 1418, 619, 1971, 1635, 635, 629, 581, 179, 1971, 629, + /* 80 */ 1843, 2161, 583, 45, 43, 130, 138, 351, 1437, 1114, + /* 90 */ 618, 371, 510, 1412, 325, 14, 1841, 335, 1437, 582, + /* 100 */ 1843, 2002, 1790, 2160, 1493, 2038, 1410, 359, 107, 2004, + /* 110 */ 639, 2006, 2007, 634, 64, 629, 1841, 726, 581, 179, + /* 120 */ 176, 265, 2091, 2161, 583, 191, 365, 2087, 1116, 1488, + /* 130 */ 1119, 1120, 1495, 1496, 18, 48, 1990, 577, 1522, 386, + /* 140 */ 181, 1418, 44, 42, 41, 40, 39, 1985, 2117, 262, + /* 150 */ 2099, 593, 479, 131, 592, 475, 1679, 2160, 60, 2021, + /* 160 */ 90, 81, 1468, 1478, 80, 48, 14, 576, 1494, 1497, + /* 170 */ 274, 275, 581, 179, 163, 273, 1649, 2161, 583, 1981, + /* 180 */ 1987, 477, 558, 1413, 8, 1411, 2160, 473, 726, 346, + /* 190 */ 629, 233, 38, 37, 1820, 1523, 44, 42, 41, 40, + /* 200 */ 39, 2166, 179, 1495, 1496, 60, 2161, 583, 1416, 1417, + /* 210 */ 575, 1467, 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477, + /* 220 */ 631, 627, 1486, 1487, 1489, 1490, 1491, 1492, 2, 572, + /* 230 */ 1338, 1339, 1439, 1468, 1478, 60, 489, 84, 120, 1494, + /* 240 */ 1497, 119, 118, 117, 116, 115, 114, 113, 112, 111, + /* 250 */ 347, 134, 345, 344, 1413, 512, 1411, 38, 37, 514, + /* 260 */ 1785, 44, 42, 41, 40, 39, 673, 34, 369, 1517, + /* 270 */ 1518, 1519, 1520, 1521, 1525, 1526, 1527, 1528, 173, 1416, + /* 280 */ 1417, 513, 1467, 1470, 1471, 1472, 1473, 1474, 1475, 1476, + /* 290 */ 1477, 631, 627, 1486, 1487, 1489, 1490, 1491, 1492, 2, + /* 300 */ 1893, 11, 45, 43, 1706, 605, 1133, 1659, 1132, 182, + /* 310 */ 371, 1658, 1412, 578, 573, 567, 182, 11, 2003, 9, + /* 320 */ 60, 216, 409, 1493, 408, 1410, 1240, 661, 660, 659, + /* 330 */ 1244, 658, 1246, 1247, 657, 1249, 654, 1134, 1255, 651, + /* 340 */ 1257, 1258, 648, 645, 1567, 1600, 405, 2164, 1488, 2021, + /* 350 */ 1437, 1971, 487, 18, 1902, 1971, 182, 636, 619, 1630, + /* 360 */ 1418, 1133, 1971, 1132, 635, 1436, 1437, 407, 403, 525, + /* 370 */ 524, 523, 130, 45, 43, 1498, 264, 135, 519, 515, + /* 380 */ 2106, 371, 518, 1412, 1992, 14, 182, 517, 522, 1790, + /* 390 */ 1556, 2002, 1134, 516, 1493, 2038, 1410, 605, 107, 2004, + /* 400 */ 639, 2006, 2007, 634, 2003, 629, 2103, 726, 141, 2165, + /* 410 */ 147, 2062, 2091, 2160, 31, 217, 365, 2087, 1524, 1488, + /* 420 */ 38, 37, 1495, 1496, 44, 42, 41, 40, 39, 2164, + /* 430 */ 168, 1418, 1994, 2161, 2163, 2021, 506, 502, 498, 494, + /* 440 */ 214, 60, 1418, 636, 603, 1503, 1902, 2165, 1971, 619, + /* 450 */ 635, 1437, 1468, 1478, 1629, 1438, 46, 539, 1494, 1497, + /* 460 */ 104, 38, 37, 54, 1439, 44, 42, 41, 40, 39, + /* 470 */ 537, 182, 535, 1413, 139, 1411, 85, 2002, 726, 212, + /* 480 */ 1790, 2038, 1782, 454, 165, 2004, 639, 2006, 2007, 634, + /* 490 */ 32, 629, 1469, 1495, 1496, 1657, 664, 27, 1416, 1417, + /* 500 */ 1529, 1467, 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477, + /* 510 */ 631, 627, 1486, 1487, 1489, 1490, 1491, 1492, 2, 525, + /* 520 */ 524, 523, 84, 1468, 1478, 559, 2128, 135, 519, 1494, + /* 530 */ 1497, 164, 518, 11, 49, 619, 1744, 517, 522, 1971, + /* 540 */ 1623, 195, 194, 516, 1413, 1786, 1411, 211, 205, 184, + /* 550 */ 1656, 685, 210, 38, 37, 485, 1779, 44, 42, 41, + /* 560 */ 40, 39, 1438, 605, 453, 1440, 1790, 1355, 1356, 1416, + /* 570 */ 1417, 203, 1467, 1470, 1471, 1472, 1473, 1474, 1475, 1476, + /* 580 */ 1477, 631, 627, 1486, 1487, 1489, 1490, 1491, 1492, 2, + /* 590 */ 45, 43, 182, 1469, 1971, 671, 1837, 1838, 371, 1638, + /* 600 */ 1412, 521, 520, 1354, 1357, 1590, 2003, 1873, 264, 585, + /* 610 */ 614, 1493, 1902, 1410, 152, 151, 668, 667, 666, 149, + /* 620 */ 120, 1775, 594, 119, 118, 117, 116, 115, 114, 113, + /* 630 */ 112, 111, 13, 12, 38, 37, 1488, 2021, 44, 42, + /* 640 */ 41, 40, 39, 2165, 1768, 597, 619, 2160, 1418, 1777, + /* 650 */ 1971, 175, 635, 138, 569, 1588, 1589, 1591, 1592, 1888, + /* 660 */ 414, 45, 43, 2164, 1830, 1843, 231, 2161, 2162, 371, + /* 670 */ 187, 1412, 364, 46, 1283, 1284, 232, 1790, 621, 2002, + /* 680 */ 2063, 1841, 1493, 2038, 1410, 182, 107, 2004, 639, 2006, + /* 690 */ 2007, 634, 2003, 629, 619, 726, 1843, 1766, 176, 623, + /* 700 */ 2091, 2063, 1200, 375, 365, 2087, 489, 1488, 415, 242, + /* 710 */ 1495, 1496, 1841, 1655, 1773, 596, 177, 2099, 2100, 1418, + /* 720 */ 136, 2104, 1579, 2021, 1637, 1790, 2118, 1536, 237, 87, + /* 730 */ 330, 636, 1412, 543, 1654, 541, 1971, 1202, 635, 1653, + /* 740 */ 1468, 1478, 1389, 1390, 14, 1410, 1494, 1497, 129, 128, + /* 750 */ 127, 126, 125, 124, 123, 122, 121, 1971, 434, 673, + /* 760 */ 630, 1413, 686, 1411, 1760, 2002, 726, 433, 663, 2038, + /* 770 */ 697, 695, 165, 2004, 639, 2006, 2007, 634, 1971, 629, + /* 780 */ 1418, 1495, 1496, 1971, 2106, 1652, 1416, 1417, 410, 1467, + /* 790 */ 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477, 631, 627, + /* 800 */ 1486, 1487, 1489, 1490, 1491, 1492, 2, 619, 1888, 1888, + /* 810 */ 2102, 1468, 1478, 328, 2129, 1435, 619, 1494, 1497, 189, + /* 820 */ 193, 424, 447, 619, 98, 461, 2106, 726, 460, 1971, + /* 830 */ 439, 558, 1413, 1843, 1411, 2160, 363, 440, 1790, 553, + /* 840 */ 41, 40, 39, 430, 161, 462, 1783, 1790, 432, 1842, + /* 850 */ 2166, 179, 2101, 1792, 1790, 2161, 583, 1416, 1417, 1958, + /* 860 */ 1467, 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477, 631, + /* 870 */ 627, 1486, 1487, 1489, 1490, 1491, 1492, 2, 1767, 374, + /* 880 */ 1119, 1120, 161, 162, 1651, 619, 619, 161, 303, 343, + /* 890 */ 1703, 1793, 514, 1413, 1765, 1411, 1792, 1437, 33, 488, + /* 900 */ 1787, 420, 301, 70, 38, 37, 69, 393, 44, 42, + /* 910 */ 41, 40, 39, 377, 513, 1745, 1790, 1790, 1416, 1417, + /* 920 */ 140, 161, 259, 2062, 199, 467, 465, 2131, 1971, 665, + /* 930 */ 1792, 458, 1834, 1560, 452, 451, 450, 449, 446, 445, + /* 940 */ 444, 443, 442, 438, 437, 436, 435, 327, 427, 426, + /* 950 */ 425, 570, 422, 421, 341, 703, 702, 701, 700, 381, + /* 960 */ 60, 699, 698, 142, 693, 692, 691, 690, 689, 688, + /* 970 */ 687, 154, 683, 682, 681, 380, 379, 678, 677, 676, + /* 980 */ 675, 674, 1648, 530, 1440, 671, 38, 37, 594, 417, + /* 990 */ 44, 42, 41, 40, 39, 1632, 1633, 1440, 540, 106, + /* 1000 */ 619, 671, 418, 619, 152, 151, 668, 667, 666, 149, + /* 1010 */ 2111, 1556, 230, 1647, 234, 619, 2003, 284, 586, 138, + /* 1020 */ 152, 151, 668, 667, 666, 149, 1971, 533, 594, 554, + /* 1030 */ 619, 1790, 527, 1646, 1790, 1645, 619, 229, 150, 1469, + /* 1040 */ 78, 77, 413, 253, 598, 186, 1790, 2021, 1644, 669, + /* 1050 */ 602, 670, 1834, 297, 1834, 636, 1820, 1971, 1693, 138, + /* 1060 */ 1971, 1790, 635, 326, 619, 1643, 401, 1790, 399, 395, + /* 1070 */ 391, 388, 385, 67, 2003, 589, 66, 1971, 278, 1971, + /* 1080 */ 526, 1421, 178, 2099, 2100, 1642, 136, 2104, 619, 2002, + /* 1090 */ 53, 215, 1971, 2038, 71, 1790, 107, 2004, 639, 2006, + /* 1100 */ 2007, 634, 616, 629, 2003, 2021, 13, 12, 2180, 1971, + /* 1110 */ 2091, 182, 150, 636, 365, 2087, 382, 2022, 1971, 1790, + /* 1120 */ 635, 2003, 180, 2099, 2100, 2125, 136, 2104, 188, 1971, + /* 1130 */ 1641, 619, 38, 37, 1674, 2021, 44, 42, 41, 40, + /* 1140 */ 39, 241, 160, 636, 79, 617, 1640, 2002, 1971, 619, + /* 1150 */ 635, 2038, 2021, 1686, 107, 2004, 639, 2006, 2007, 634, + /* 1160 */ 636, 629, 1790, 378, 1387, 1971, 2180, 635, 2091, 50, + /* 1170 */ 1684, 3, 365, 2087, 1971, 528, 1559, 2002, 2003, 240, + /* 1180 */ 1790, 2038, 600, 2138, 313, 2004, 639, 2006, 2007, 634, + /* 1190 */ 1971, 629, 531, 144, 2002, 132, 222, 224, 2038, 220, + /* 1200 */ 223, 107, 2004, 639, 2006, 2007, 634, 1420, 629, 2021, + /* 1210 */ 1897, 226, 1831, 2180, 225, 2091, 52, 636, 88, 365, + /* 1220 */ 2087, 384, 1971, 557, 635, 558, 62, 626, 579, 2160, + /* 1230 */ 565, 1424, 228, 368, 367, 227, 2003, 2121, 246, 595, + /* 1240 */ 150, 587, 47, 1426, 2166, 179, 1, 271, 258, 2161, + /* 1250 */ 583, 2002, 47, 1680, 1493, 2038, 1419, 4, 107, 2004, + /* 1260 */ 639, 2006, 2007, 634, 558, 629, 2003, 2021, 2160, 261, + /* 1270 */ 2180, 1162, 2091, 68, 148, 636, 365, 2087, 1587, 1488, + /* 1280 */ 1971, 150, 635, 2166, 179, 103, 679, 2154, 2161, 583, + /* 1290 */ 248, 1418, 601, 387, 1352, 100, 392, 2021, 2003, 276, + /* 1300 */ 590, 340, 721, 62, 1479, 636, 1163, 47, 1181, 2002, + /* 1310 */ 1971, 643, 635, 2038, 291, 148, 107, 2004, 639, 2006, + /* 1320 */ 2007, 634, 150, 629, 133, 611, 280, 1374, 2180, 2021, + /* 1330 */ 2091, 192, 680, 1233, 365, 2087, 148, 636, 625, 2002, + /* 1340 */ 1440, 419, 1971, 2038, 635, 2110, 107, 2004, 639, 2006, + /* 1350 */ 2007, 634, 383, 629, 1179, 1530, 1898, 1423, 2066, 296, + /* 1360 */ 2091, 423, 456, 1261, 365, 2087, 1435, 1265, 428, 441, + /* 1370 */ 455, 2002, 1514, 1890, 1272, 2038, 1270, 2003, 107, 2004, + /* 1380 */ 639, 2006, 2007, 634, 448, 629, 457, 463, 153, 464, + /* 1390 */ 2064, 196, 2091, 466, 468, 558, 365, 2087, 1441, 2160, + /* 1400 */ 469, 478, 1443, 481, 1427, 202, 1422, 1438, 2021, 482, + /* 1410 */ 204, 1442, 483, 1444, 2166, 179, 636, 484, 207, 2161, + /* 1420 */ 583, 1971, 486, 635, 209, 546, 82, 490, 83, 1430, + /* 1430 */ 1432, 213, 1136, 507, 508, 2003, 511, 1780, 509, 219, + /* 1440 */ 1776, 110, 627, 1486, 1487, 1489, 1490, 1491, 1492, 221, + /* 1450 */ 2002, 155, 156, 329, 2038, 2003, 1778, 107, 2004, 639, + /* 1460 */ 2006, 2007, 634, 1774, 629, 545, 2021, 547, 558, 622, + /* 1470 */ 157, 2091, 2160, 158, 636, 365, 2087, 235, 86, 1971, + /* 1480 */ 292, 635, 1948, 548, 1947, 238, 2021, 2166, 179, 552, + /* 1490 */ 571, 555, 2161, 583, 636, 146, 549, 609, 2122, 1971, + /* 1500 */ 2132, 635, 562, 568, 2137, 2136, 244, 247, 2002, 7, + /* 1510 */ 354, 574, 2038, 2003, 580, 108, 2004, 639, 2006, 2007, + /* 1520 */ 634, 2113, 629, 560, 563, 252, 355, 561, 2002, 2091, + /* 1530 */ 255, 169, 2038, 2090, 2087, 108, 2004, 639, 2006, 2007, + /* 1540 */ 634, 588, 629, 2003, 2021, 254, 257, 591, 1556, 2091, + /* 1550 */ 256, 137, 636, 624, 2087, 1439, 2107, 1971, 358, 635, + /* 1560 */ 2003, 599, 1445, 266, 2183, 1903, 93, 612, 293, 294, + /* 1570 */ 607, 613, 608, 260, 2021, 1917, 95, 1916, 2159, 1915, + /* 1580 */ 361, 97, 633, 295, 1791, 59, 637, 1971, 2072, 635, + /* 1590 */ 2038, 2021, 99, 108, 2004, 639, 2006, 2007, 634, 636, + /* 1600 */ 629, 1835, 641, 298, 1971, 722, 635, 2091, 1761, 723, + /* 1610 */ 287, 334, 2087, 725, 51, 331, 2002, 2003, 300, 307, + /* 1620 */ 2038, 332, 322, 319, 2004, 639, 2006, 2007, 634, 632, + /* 1630 */ 629, 620, 2056, 2002, 302, 321, 1965, 2038, 311, 1964, + /* 1640 */ 166, 2004, 639, 2006, 2007, 634, 75, 629, 2021, 1963, + /* 1650 */ 1962, 76, 1959, 389, 390, 1404, 636, 1405, 185, 394, + /* 1660 */ 1957, 1971, 396, 635, 2003, 397, 398, 1956, 400, 1955, + /* 1670 */ 402, 1954, 1953, 404, 1377, 406, 1376, 1928, 1927, 411, + /* 1680 */ 412, 2003, 1926, 1925, 1329, 1881, 1880, 1878, 1877, 143, + /* 1690 */ 2002, 1876, 584, 2181, 2038, 2021, 1879, 108, 2004, 639, + /* 1700 */ 2006, 2007, 634, 636, 629, 1875, 1874, 1872, 1971, 190, + /* 1710 */ 635, 2091, 2021, 1871, 1870, 429, 2088, 360, 1869, 431, + /* 1720 */ 636, 1883, 1868, 1867, 1866, 1971, 1865, 635, 1864, 1863, + /* 1730 */ 1862, 1861, 1860, 1859, 1858, 1857, 2003, 2002, 1856, 1855, + /* 1740 */ 1854, 2038, 145, 1853, 166, 2004, 639, 2006, 2007, 634, + /* 1750 */ 1852, 629, 1851, 1882, 2002, 1850, 1849, 1848, 2038, 2003, + /* 1760 */ 1847, 320, 2004, 639, 2006, 2007, 634, 2021, 629, 1331, + /* 1770 */ 1846, 1845, 1844, 459, 1708, 633, 1208, 197, 1707, 1705, + /* 1780 */ 1971, 1669, 635, 198, 200, 73, 1668, 1122, 174, 1121, + /* 1790 */ 2021, 2003, 1991, 1941, 1935, 370, 474, 2182, 636, 201, + /* 1800 */ 74, 476, 1924, 1971, 206, 635, 208, 1923, 1901, 2002, + /* 1810 */ 1769, 1704, 1155, 2038, 1702, 491, 319, 2004, 639, 2006, + /* 1820 */ 2007, 634, 2021, 629, 492, 2057, 493, 372, 1700, 497, + /* 1830 */ 636, 495, 2002, 496, 1698, 1971, 2038, 635, 2003, 320, + /* 1840 */ 2004, 639, 2006, 2007, 634, 500, 629, 499, 501, 1696, + /* 1850 */ 503, 504, 729, 1683, 1682, 505, 1665, 1771, 1277, 1276, + /* 1860 */ 218, 1770, 2003, 694, 2002, 1199, 290, 61, 2038, 2021, + /* 1870 */ 1198, 320, 2004, 639, 2006, 2007, 634, 636, 629, 1197, + /* 1880 */ 1196, 172, 1971, 696, 635, 1193, 1192, 719, 715, 711, + /* 1890 */ 707, 288, 1191, 2021, 1190, 1694, 348, 1687, 349, 1685, + /* 1900 */ 350, 636, 529, 532, 1664, 1663, 1971, 534, 635, 1662, + /* 1910 */ 536, 544, 538, 109, 1394, 2038, 1393, 1396, 315, 2004, + /* 1920 */ 639, 2006, 2007, 634, 26, 629, 1940, 105, 2003, 1383, + /* 1930 */ 281, 542, 55, 1934, 550, 2002, 1922, 159, 1920, 2038, + /* 1940 */ 2165, 551, 304, 2004, 639, 2006, 2007, 634, 239, 629, + /* 1950 */ 2003, 19, 352, 16, 1602, 556, 564, 28, 566, 2021, + /* 1960 */ 243, 58, 5, 615, 63, 245, 6, 636, 250, 1586, + /* 1970 */ 251, 30, 1971, 1578, 635, 2003, 1992, 167, 249, 29, + /* 1980 */ 89, 2021, 1622, 1623, 21, 1617, 1616, 356, 1621, 636, + /* 1990 */ 1620, 357, 1553, 263, 1971, 170, 635, 1552, 268, 92, + /* 2000 */ 1899, 2002, 57, 267, 1921, 2038, 2021, 1919, 305, 2004, + /* 2010 */ 639, 2006, 2007, 634, 636, 629, 20, 1918, 1900, 1971, + /* 2020 */ 1381, 635, 236, 2002, 269, 91, 22, 2038, 56, 272, + /* 2030 */ 306, 2004, 639, 2006, 2007, 634, 277, 629, 2003, 17, + /* 2040 */ 270, 1584, 65, 282, 94, 23, 1505, 610, 2002, 96, + /* 2050 */ 12, 10, 2038, 1504, 279, 312, 2004, 639, 2006, 2007, + /* 2060 */ 634, 1428, 629, 1515, 100, 2041, 2003, 1483, 628, 2021, + /* 2070 */ 1481, 36, 171, 183, 1480, 15, 24, 636, 640, 1452, + /* 2080 */ 1460, 638, 1971, 25, 635, 1262, 642, 373, 644, 646, + /* 2090 */ 1259, 1256, 647, 650, 649, 652, 2003, 2021, 1250, 1248, + /* 2100 */ 655, 653, 1239, 656, 1254, 636, 1253, 101, 285, 1252, + /* 2110 */ 1971, 2002, 635, 1267, 102, 2038, 662, 1271, 316, 2004, + /* 2120 */ 639, 2006, 2007, 634, 72, 629, 1153, 2021, 2003, 1251, + /* 2130 */ 672, 1187, 1186, 1185, 1184, 636, 1183, 1182, 1180, 2002, + /* 2140 */ 1971, 1178, 635, 2038, 1177, 1176, 308, 2004, 639, 2006, + /* 2150 */ 2007, 634, 286, 629, 1206, 684, 1174, 1173, 1172, 2021, + /* 2160 */ 2003, 1171, 1170, 1169, 1168, 1203, 1201, 636, 1165, 2002, + /* 2170 */ 1164, 1161, 1971, 2038, 635, 1160, 317, 2004, 639, 2006, + /* 2180 */ 2007, 634, 1159, 629, 1158, 1701, 704, 706, 1699, 705, + /* 2190 */ 708, 2021, 2003, 709, 1697, 710, 712, 714, 713, 636, + /* 2200 */ 1695, 2002, 716, 718, 1971, 2038, 635, 717, 309, 2004, + /* 2210 */ 639, 2006, 2007, 634, 1681, 629, 720, 2003, 1111, 1661, + /* 2220 */ 289, 1414, 724, 2021, 299, 727, 728, 1636, 1636, 1636, + /* 2230 */ 1636, 636, 1636, 2002, 1636, 1636, 1971, 2038, 635, 1636, + /* 2240 */ 318, 2004, 639, 2006, 2007, 634, 1636, 629, 2021, 2003, + /* 2250 */ 1636, 1636, 1636, 1636, 1636, 1636, 636, 1636, 1636, 1636, + /* 2260 */ 1636, 1971, 1636, 635, 1636, 2002, 1636, 1636, 1636, 2038, + /* 2270 */ 1636, 1636, 310, 2004, 639, 2006, 2007, 634, 1636, 629, + /* 2280 */ 2021, 1636, 1636, 1636, 1636, 1636, 1636, 1636, 636, 1636, + /* 2290 */ 2002, 1636, 1636, 1971, 2038, 635, 1636, 323, 2004, 639, + /* 2300 */ 2006, 2007, 634, 1636, 629, 1636, 1636, 1636, 1636, 1636, + /* 2310 */ 1636, 2003, 1636, 1636, 1636, 1636, 1636, 1636, 1636, 1636, + /* 2320 */ 1636, 1636, 2002, 1636, 1636, 1636, 2038, 1636, 2003, 324, + /* 2330 */ 2004, 639, 2006, 2007, 634, 1636, 629, 1636, 1636, 1636, + /* 2340 */ 1636, 1636, 2021, 1636, 1636, 1636, 1636, 1636, 1636, 1636, + /* 2350 */ 636, 1636, 1636, 1636, 1636, 1971, 1636, 635, 1636, 2021, + /* 2360 */ 1636, 1636, 1636, 1636, 1636, 1636, 1636, 636, 1636, 1636, + /* 2370 */ 1636, 1636, 1971, 1636, 635, 1636, 1636, 1636, 1636, 1636, + /* 2380 */ 1636, 1636, 1636, 1636, 2002, 2003, 1636, 1636, 2038, 1636, + /* 2390 */ 1636, 2015, 2004, 639, 2006, 2007, 634, 1636, 629, 1636, + /* 2400 */ 1636, 2002, 1636, 1636, 1636, 2038, 1636, 2003, 2014, 2004, + /* 2410 */ 639, 2006, 2007, 634, 1636, 629, 2021, 1636, 1636, 1636, + /* 2420 */ 1636, 1636, 1636, 1636, 636, 1636, 1636, 1636, 1636, 1971, + /* 2430 */ 1636, 635, 1636, 1636, 1636, 1636, 1636, 1636, 2021, 1636, + /* 2440 */ 1636, 1636, 1636, 1636, 1636, 1636, 636, 1636, 1636, 1636, + /* 2450 */ 1636, 1971, 1636, 635, 1636, 1636, 1636, 1636, 2002, 1636, + /* 2460 */ 1636, 1636, 2038, 1636, 1636, 2013, 2004, 639, 2006, 2007, + /* 2470 */ 634, 1636, 629, 2003, 1636, 1636, 1636, 1636, 1636, 1636, + /* 2480 */ 2002, 1636, 1636, 1636, 2038, 1636, 1636, 336, 2004, 639, + /* 2490 */ 2006, 2007, 634, 1636, 629, 1636, 1636, 2003, 1636, 1636, + /* 2500 */ 1636, 1636, 1636, 1636, 2021, 1636, 1636, 1636, 1636, 1636, + /* 2510 */ 1636, 1636, 636, 1636, 1636, 1636, 1636, 1971, 1636, 635, + /* 2520 */ 1636, 1636, 1636, 1636, 1636, 1636, 1636, 1636, 2021, 1636, + /* 2530 */ 1636, 1636, 1636, 1636, 1636, 1636, 636, 1636, 1636, 1636, + /* 2540 */ 1636, 1971, 1636, 635, 1636, 1636, 2002, 1636, 1636, 1636, + /* 2550 */ 2038, 1636, 1636, 337, 2004, 639, 2006, 2007, 634, 1636, + /* 2560 */ 629, 2003, 1636, 1636, 1636, 1636, 1636, 1636, 1636, 1636, + /* 2570 */ 2002, 1636, 1636, 1636, 2038, 1636, 1636, 333, 2004, 639, + /* 2580 */ 2006, 2007, 634, 1636, 629, 1636, 2003, 1636, 1636, 1636, + /* 2590 */ 1636, 1636, 2021, 1636, 1636, 1636, 1636, 1636, 1636, 1636, + /* 2600 */ 636, 1636, 1636, 1636, 1636, 1971, 1636, 635, 1636, 1636, + /* 2610 */ 1636, 1636, 1636, 1636, 1636, 1636, 1636, 2021, 2003, 1636, + /* 2620 */ 1636, 1636, 1636, 1636, 1636, 636, 1636, 1636, 1636, 1636, + /* 2630 */ 1971, 1636, 635, 1636, 2002, 1636, 1636, 1636, 2038, 1636, + /* 2640 */ 1636, 338, 2004, 639, 2006, 2007, 634, 1636, 629, 2021, + /* 2650 */ 1636, 1636, 1636, 1636, 1636, 1636, 1636, 636, 1636, 637, + /* 2660 */ 1636, 1636, 1971, 2038, 635, 1636, 315, 2004, 639, 2006, + /* 2670 */ 2007, 634, 1636, 629, 1636, 1636, 1636, 1636, 1636, 1636, + /* 2680 */ 1636, 1636, 1636, 1636, 1636, 1636, 1636, 1636, 1636, 1636, + /* 2690 */ 1636, 2002, 1636, 1636, 1636, 2038, 1636, 1636, 314, 2004, + /* 2700 */ 639, 2006, 2007, 634, 1636, 629, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 329, 348, 331, 332, 429, 333, 333, 429, 433, 356, - /* 10 */ 358, 433, 12, 13, 14, 14, 15, 16, 365, 347, - /* 20 */ 20, 369, 22, 448, 449, 20, 354, 449, 453, 454, - /* 30 */ 325, 453, 454, 33, 20, 35, 364, 364, 329, 333, - /* 40 */ 331, 332, 8, 9, 358, 333, 12, 13, 14, 15, - /* 50 */ 16, 399, 400, 401, 391, 369, 356, 333, 58, 14, - /* 60 */ 20, 356, 410, 63, 367, 20, 356, 370, 371, 364, - /* 70 */ 70, 20, 20, 363, 369, 340, 371, 377, 378, 373, - /* 80 */ 345, 356, 372, 12, 13, 399, 400, 401, 364, 364, - /* 90 */ 62, 20, 380, 22, 382, 95, 410, 63, 425, 426, - /* 100 */ 427, 396, 429, 430, 33, 400, 35, 3, 403, 404, - /* 110 */ 405, 406, 407, 408, 333, 410, 20, 117, 22, 62, - /* 120 */ 415, 330, 417, 20, 333, 334, 421, 422, 347, 58, - /* 130 */ 405, 35, 132, 133, 63, 354, 418, 419, 104, 429, - /* 140 */ 435, 70, 324, 433, 326, 364, 95, 51, 443, 425, - /* 150 */ 426, 427, 333, 429, 430, 325, 325, 433, 448, 449, - /* 160 */ 333, 161, 162, 453, 454, 333, 95, 12, 13, 14, - /* 170 */ 15, 16, 448, 449, 347, 161, 162, 453, 454, 337, - /* 180 */ 180, 333, 182, 364, 132, 133, 330, 356, 117, 333, - /* 190 */ 334, 364, 20, 351, 160, 364, 364, 20, 95, 369, - /* 200 */ 369, 359, 371, 132, 133, 205, 206, 4, 208, 209, - /* 210 */ 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, - /* 220 */ 220, 221, 222, 223, 224, 225, 226, 396, 380, 95, - /* 230 */ 382, 400, 161, 162, 403, 404, 405, 406, 407, 408, - /* 240 */ 409, 410, 411, 412, 425, 426, 427, 20, 429, 430, - /* 250 */ 37, 180, 402, 182, 370, 371, 424, 425, 426, 427, - /* 260 */ 166, 429, 430, 325, 95, 231, 232, 233, 234, 235, - /* 270 */ 236, 237, 238, 239, 240, 241, 205, 206, 428, 208, - /* 280 */ 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, - /* 290 */ 219, 220, 221, 222, 223, 224, 225, 226, 95, 228, - /* 300 */ 12, 13, 0, 95, 205, 322, 364, 369, 20, 14, - /* 310 */ 22, 98, 0, 100, 101, 20, 103, 375, 325, 325, - /* 320 */ 107, 33, 95, 35, 108, 109, 110, 111, 112, 113, - /* 330 */ 114, 115, 116, 117, 118, 163, 120, 121, 122, 123, - /* 340 */ 124, 125, 129, 249, 250, 58, 58, 244, 179, 356, - /* 350 */ 181, 63, 253, 254, 255, 256, 257, 364, 70, 20, - /* 360 */ 35, 22, 369, 369, 371, 261, 175, 65, 66, 67, - /* 370 */ 387, 12, 13, 14, 62, 73, 74, 387, 244, 20, - /* 380 */ 78, 22, 95, 95, 97, 83, 84, 196, 197, 396, - /* 390 */ 51, 89, 33, 400, 35, 70, 403, 404, 405, 406, - /* 400 */ 407, 408, 325, 410, 339, 117, 413, 325, 415, 416, - /* 410 */ 417, 333, 429, 244, 421, 422, 433, 58, 353, 429, - /* 420 */ 132, 133, 355, 433, 20, 347, 126, 362, 44, 70, - /* 430 */ 325, 448, 449, 356, 0, 368, 453, 454, 448, 449, - /* 440 */ 22, 364, 364, 453, 454, 356, 369, 244, 371, 161, - /* 450 */ 162, 369, 244, 35, 95, 21, 161, 371, 24, 25, - /* 460 */ 26, 27, 28, 29, 30, 31, 32, 378, 180, 383, - /* 470 */ 182, 244, 386, 396, 369, 349, 117, 400, 352, 356, - /* 480 */ 403, 404, 405, 406, 407, 408, 363, 410, 70, 189, - /* 490 */ 190, 132, 133, 205, 206, 372, 208, 209, 210, 211, - /* 500 */ 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, - /* 510 */ 222, 223, 224, 225, 226, 342, 343, 342, 343, 21, - /* 520 */ 161, 162, 24, 25, 26, 27, 28, 29, 30, 31, - /* 530 */ 32, 244, 455, 456, 348, 117, 65, 66, 67, 180, - /* 540 */ 348, 182, 356, 356, 73, 74, 14, 0, 356, 78, - /* 550 */ 363, 365, 20, 20, 83, 84, 0, 365, 325, 372, - /* 560 */ 89, 126, 127, 20, 205, 206, 131, 208, 209, 210, - /* 570 */ 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, - /* 580 */ 221, 222, 223, 224, 225, 226, 12, 13, 0, 356, - /* 590 */ 325, 333, 58, 21, 20, 48, 22, 364, 180, 0, - /* 600 */ 182, 325, 369, 20, 371, 347, 34, 33, 36, 35, - /* 610 */ 358, 333, 24, 25, 26, 27, 28, 29, 30, 31, - /* 620 */ 32, 369, 364, 205, 206, 347, 1, 2, 94, 396, - /* 630 */ 325, 97, 58, 400, 369, 325, 403, 404, 405, 406, - /* 640 */ 407, 408, 364, 410, 70, 369, 337, 339, 415, 265, - /* 650 */ 417, 399, 400, 413, 421, 422, 416, 12, 13, 333, - /* 660 */ 4, 62, 410, 107, 3, 20, 325, 22, 359, 95, - /* 670 */ 362, 79, 325, 347, 369, 19, 443, 371, 33, 369, - /* 680 */ 35, 20, 126, 127, 128, 129, 130, 131, 2, 33, - /* 690 */ 364, 117, 386, 161, 8, 9, 163, 356, 12, 13, - /* 700 */ 14, 15, 16, 58, 48, 364, 132, 133, 325, 53, - /* 710 */ 369, 325, 371, 402, 58, 70, 369, 325, 429, 8, - /* 720 */ 9, 96, 433, 12, 13, 14, 15, 16, 4, 137, - /* 730 */ 138, 20, 0, 160, 402, 161, 162, 396, 449, 428, - /* 740 */ 95, 400, 453, 454, 403, 404, 405, 406, 407, 408, - /* 750 */ 94, 410, 369, 97, 180, 369, 182, 96, 417, 2, - /* 760 */ 428, 369, 117, 422, 325, 8, 9, 325, 127, 12, - /* 770 */ 13, 14, 15, 16, 228, 325, 4, 132, 133, 205, - /* 780 */ 206, 49, 208, 209, 210, 211, 212, 213, 214, 215, - /* 790 */ 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, - /* 800 */ 226, 18, 35, 20, 231, 325, 161, 162, 369, 19, - /* 810 */ 27, 369, 356, 30, 241, 43, 33, 45, 46, 369, - /* 820 */ 414, 365, 416, 33, 163, 180, 0, 182, 187, 188, - /* 830 */ 364, 48, 191, 50, 193, 228, 53, 230, 48, 0, - /* 840 */ 414, 375, 416, 39, 54, 55, 56, 57, 58, 369, - /* 850 */ 205, 206, 0, 208, 209, 210, 211, 212, 213, 214, - /* 860 */ 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, - /* 870 */ 225, 226, 356, 20, 163, 8, 9, 94, 20, 12, - /* 880 */ 13, 14, 15, 16, 94, 364, 63, 97, 372, 106, - /* 890 */ 106, 65, 66, 67, 68, 69, 375, 71, 72, 73, - /* 900 */ 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, - /* 910 */ 84, 85, 86, 87, 88, 89, 90, 91, 366, 136, - /* 920 */ 130, 369, 139, 140, 141, 142, 143, 144, 145, 146, - /* 930 */ 147, 148, 149, 150, 151, 152, 153, 154, 155, 70, - /* 940 */ 157, 158, 159, 18, 45, 46, 8, 9, 23, 182, - /* 950 */ 12, 13, 14, 15, 16, 325, 325, 167, 325, 107, - /* 960 */ 70, 366, 37, 38, 369, 349, 41, 366, 352, 245, - /* 970 */ 369, 344, 357, 346, 184, 42, 186, 44, 126, 127, - /* 980 */ 128, 129, 130, 131, 59, 60, 61, 8, 9, 356, - /* 990 */ 151, 12, 13, 14, 15, 16, 107, 364, 47, 369, - /* 1000 */ 369, 333, 369, 107, 371, 357, 8, 9, 242, 243, - /* 1010 */ 12, 13, 14, 15, 16, 347, 163, 58, 129, 161, - /* 1020 */ 95, 333, 126, 127, 128, 129, 130, 131, 205, 396, - /* 1030 */ 163, 333, 364, 400, 96, 347, 403, 404, 405, 406, - /* 1040 */ 407, 408, 44, 410, 44, 347, 95, 243, 415, 42, - /* 1050 */ 417, 44, 364, 22, 421, 422, 97, 357, 106, 134, - /* 1060 */ 333, 325, 364, 8, 9, 432, 35, 12, 13, 14, - /* 1070 */ 15, 16, 8, 9, 347, 357, 12, 13, 14, 15, - /* 1080 */ 16, 333, 99, 99, 44, 102, 102, 333, 0, 35, - /* 1090 */ 99, 364, 356, 102, 0, 347, 96, 172, 173, 174, - /* 1100 */ 364, 347, 177, 333, 44, 369, 325, 371, 156, 0, - /* 1110 */ 22, 99, 364, 358, 102, 44, 22, 347, 364, 44, - /* 1120 */ 195, 132, 133, 198, 369, 200, 201, 202, 203, 204, - /* 1130 */ 44, 22, 396, 333, 364, 333, 400, 356, 326, 403, - /* 1140 */ 404, 405, 406, 407, 408, 364, 410, 347, 44, 347, - /* 1150 */ 369, 415, 371, 417, 399, 400, 96, 421, 422, 345, - /* 1160 */ 96, 333, 333, 357, 364, 410, 364, 96, 432, 244, - /* 1170 */ 44, 96, 357, 163, 44, 347, 347, 396, 168, 325, - /* 1180 */ 44, 400, 96, 457, 403, 404, 405, 406, 407, 408, - /* 1190 */ 13, 410, 364, 364, 12, 13, 415, 333, 417, 44, - /* 1200 */ 96, 44, 421, 422, 22, 1, 2, 13, 357, 95, - /* 1210 */ 356, 347, 35, 432, 379, 33, 44, 35, 364, 105, - /* 1220 */ 33, 387, 96, 369, 387, 371, 96, 172, 364, 35, - /* 1230 */ 335, 446, 96, 440, 325, 48, 182, 44, 356, 335, - /* 1240 */ 58, 54, 55, 56, 57, 58, 44, 379, 332, 44, - /* 1250 */ 396, 96, 70, 96, 400, 368, 44, 403, 404, 405, - /* 1260 */ 406, 407, 408, 429, 410, 356, 429, 433, 96, 415, - /* 1270 */ 433, 417, 35, 364, 431, 421, 422, 379, 369, 423, - /* 1280 */ 371, 94, 448, 449, 97, 448, 449, 453, 454, 96, - /* 1290 */ 453, 454, 450, 434, 387, 246, 48, 397, 96, 117, - /* 1300 */ 398, 96, 178, 263, 389, 396, 42, 70, 96, 400, - /* 1310 */ 376, 20, 403, 404, 405, 406, 407, 408, 379, 410, - /* 1320 */ 376, 266, 160, 374, 415, 20, 417, 333, 333, 376, - /* 1330 */ 421, 422, 374, 93, 341, 374, 429, 333, 333, 333, - /* 1340 */ 433, 20, 327, 327, 20, 371, 325, 387, 393, 20, - /* 1350 */ 339, 164, 165, 339, 167, 448, 449, 170, 334, 20, - /* 1360 */ 453, 454, 180, 388, 182, 339, 334, 339, 339, 339, - /* 1370 */ 333, 52, 339, 186, 336, 336, 325, 356, 356, 327, - /* 1380 */ 333, 356, 369, 356, 327, 364, 369, 205, 206, 429, - /* 1390 */ 369, 356, 371, 433, 194, 393, 395, 356, 356, 217, - /* 1400 */ 218, 219, 220, 221, 222, 223, 356, 356, 448, 449, - /* 1410 */ 337, 185, 356, 453, 454, 364, 371, 396, 356, 356, - /* 1420 */ 369, 400, 371, 356, 403, 404, 405, 406, 407, 408, - /* 1430 */ 337, 410, 325, 333, 369, 392, 415, 379, 417, 251, - /* 1440 */ 379, 252, 421, 422, 258, 369, 369, 396, 439, 369, - /* 1450 */ 384, 400, 325, 384, 403, 404, 405, 406, 407, 408, - /* 1460 */ 442, 410, 439, 356, 171, 247, 260, 259, 417, 262, - /* 1470 */ 398, 364, 421, 422, 438, 441, 369, 267, 371, 264, - /* 1480 */ 364, 437, 243, 356, 436, 439, 20, 402, 334, 333, - /* 1490 */ 337, 364, 20, 382, 384, 165, 369, 369, 371, 384, - /* 1500 */ 369, 369, 369, 396, 369, 452, 369, 400, 337, 364, - /* 1510 */ 403, 404, 405, 406, 407, 408, 352, 410, 337, 381, - /* 1520 */ 325, 451, 95, 396, 417, 420, 458, 400, 421, 422, - /* 1530 */ 403, 404, 405, 406, 407, 408, 325, 410, 95, 360, - /* 1540 */ 337, 36, 369, 346, 417, 328, 333, 327, 421, 422, - /* 1550 */ 390, 356, 394, 385, 385, 323, 0, 0, 187, 364, - /* 1560 */ 42, 338, 0, 0, 369, 0, 371, 356, 35, 350, - /* 1570 */ 35, 350, 350, 199, 35, 364, 35, 0, 199, 35, - /* 1580 */ 369, 35, 371, 199, 0, 199, 0, 35, 0, 22, - /* 1590 */ 0, 396, 182, 35, 180, 400, 0, 325, 403, 404, - /* 1600 */ 405, 406, 407, 408, 0, 410, 176, 396, 175, 0, - /* 1610 */ 0, 400, 0, 47, 403, 404, 405, 406, 407, 408, - /* 1620 */ 0, 410, 0, 42, 0, 0, 0, 0, 356, 151, - /* 1630 */ 35, 0, 151, 0, 0, 0, 364, 0, 0, 444, - /* 1640 */ 445, 369, 325, 371, 0, 0, 0, 0, 0, 0, - /* 1650 */ 0, 0, 0, 0, 0, 0, 445, 0, 0, 42, - /* 1660 */ 0, 0, 0, 0, 0, 22, 0, 0, 396, 0, - /* 1670 */ 0, 135, 400, 356, 35, 403, 404, 405, 406, 407, - /* 1680 */ 408, 364, 410, 0, 0, 0, 369, 0, 371, 0, - /* 1690 */ 58, 14, 14, 0, 58, 0, 0, 0, 42, 171, - /* 1700 */ 0, 0, 0, 325, 39, 64, 47, 0, 0, 0, - /* 1710 */ 0, 39, 44, 396, 35, 40, 0, 400, 47, 447, - /* 1720 */ 403, 404, 405, 406, 407, 408, 325, 410, 47, 39, - /* 1730 */ 48, 39, 0, 35, 356, 0, 48, 0, 39, 361, - /* 1740 */ 35, 35, 364, 39, 0, 22, 104, 369, 0, 371, - /* 1750 */ 35, 48, 39, 35, 35, 35, 35, 356, 44, 48, - /* 1760 */ 35, 44, 22, 35, 35, 364, 0, 22, 0, 22, - /* 1770 */ 369, 0, 371, 456, 396, 102, 50, 22, 400, 0, - /* 1780 */ 0, 403, 404, 405, 406, 407, 408, 325, 410, 35, - /* 1790 */ 0, 35, 35, 20, 22, 35, 35, 396, 192, 0, - /* 1800 */ 35, 400, 0, 163, 403, 404, 405, 406, 407, 408, - /* 1810 */ 0, 410, 0, 412, 22, 0, 3, 325, 356, 44, - /* 1820 */ 96, 248, 44, 361, 47, 95, 364, 96, 95, 95, - /* 1830 */ 44, 369, 96, 371, 96, 96, 95, 95, 47, 44, - /* 1840 */ 325, 95, 248, 163, 95, 3, 96, 96, 356, 44, - /* 1850 */ 248, 165, 35, 361, 163, 35, 364, 35, 396, 183, - /* 1860 */ 35, 369, 400, 371, 35, 403, 404, 405, 406, 407, - /* 1870 */ 408, 356, 410, 35, 47, 44, 0, 96, 96, 364, - /* 1880 */ 47, 169, 0, 47, 369, 325, 371, 0, 396, 95, - /* 1890 */ 39, 95, 400, 96, 96, 403, 404, 405, 406, 407, - /* 1900 */ 408, 325, 410, 95, 95, 105, 95, 47, 44, 2, - /* 1910 */ 22, 396, 96, 95, 95, 400, 356, 164, 403, 404, - /* 1920 */ 405, 406, 407, 408, 364, 410, 96, 95, 166, 369, - /* 1930 */ 96, 371, 356, 47, 95, 95, 242, 47, 22, 96, - /* 1940 */ 364, 95, 95, 106, 96, 369, 35, 371, 35, 227, - /* 1950 */ 96, 95, 35, 35, 325, 96, 396, 205, 95, 35, - /* 1960 */ 400, 96, 95, 403, 404, 405, 406, 407, 408, 96, - /* 1970 */ 410, 207, 396, 35, 95, 22, 400, 119, 107, 403, - /* 1980 */ 404, 405, 406, 407, 408, 356, 410, 229, 119, 119, - /* 1990 */ 227, 119, 95, 364, 44, 95, 35, 22, 369, 325, - /* 2000 */ 371, 95, 35, 64, 63, 35, 35, 35, 35, 35, - /* 2010 */ 70, 35, 35, 35, 35, 92, 35, 35, 35, 22, - /* 2020 */ 35, 35, 35, 35, 44, 396, 70, 35, 35, 400, - /* 2030 */ 356, 35, 403, 404, 405, 406, 407, 408, 364, 410, - /* 2040 */ 35, 22, 35, 369, 0, 371, 35, 48, 39, 0, - /* 2050 */ 35, 48, 325, 39, 0, 35, 48, 39, 0, 35, - /* 2060 */ 48, 39, 0, 35, 35, 0, 20, 22, 325, 21, - /* 2070 */ 396, 459, 22, 22, 400, 459, 21, 403, 404, 405, - /* 2080 */ 406, 407, 408, 356, 410, 459, 459, 459, 459, 459, - /* 2090 */ 459, 364, 459, 459, 459, 459, 369, 459, 371, 356, - /* 2100 */ 459, 459, 459, 459, 459, 459, 459, 364, 459, 459, - /* 2110 */ 459, 459, 369, 325, 371, 459, 459, 459, 459, 459, - /* 2120 */ 459, 459, 459, 396, 459, 459, 459, 400, 459, 459, - /* 2130 */ 403, 404, 405, 406, 407, 408, 459, 410, 459, 396, - /* 2140 */ 459, 459, 459, 400, 356, 459, 403, 404, 405, 406, - /* 2150 */ 407, 408, 364, 410, 459, 459, 459, 369, 459, 371, - /* 2160 */ 459, 459, 459, 459, 459, 459, 325, 459, 459, 459, - /* 2170 */ 459, 459, 459, 459, 459, 459, 459, 459, 459, 459, - /* 2180 */ 459, 459, 325, 459, 396, 459, 459, 459, 400, 459, - /* 2190 */ 459, 403, 404, 405, 406, 407, 408, 356, 410, 459, - /* 2200 */ 459, 459, 459, 459, 459, 364, 459, 459, 459, 459, - /* 2210 */ 369, 459, 371, 356, 459, 459, 459, 459, 459, 459, - /* 2220 */ 459, 364, 459, 459, 459, 459, 369, 325, 371, 459, - /* 2230 */ 459, 459, 459, 459, 459, 459, 459, 396, 459, 459, - /* 2240 */ 459, 400, 459, 459, 403, 404, 405, 406, 407, 408, - /* 2250 */ 325, 410, 459, 396, 459, 459, 459, 400, 356, 459, - /* 2260 */ 403, 404, 405, 406, 407, 408, 364, 410, 459, 459, - /* 2270 */ 459, 369, 459, 371, 459, 459, 459, 459, 459, 459, - /* 2280 */ 325, 356, 459, 459, 459, 459, 459, 459, 459, 364, - /* 2290 */ 459, 459, 459, 459, 369, 459, 371, 459, 396, 459, - /* 2300 */ 459, 459, 400, 459, 459, 403, 404, 405, 406, 407, - /* 2310 */ 408, 356, 410, 459, 459, 459, 459, 459, 459, 364, - /* 2320 */ 459, 396, 459, 459, 369, 400, 371, 459, 403, 404, - /* 2330 */ 405, 406, 407, 408, 459, 410, 459, 459, 459, 459, - /* 2340 */ 459, 459, 459, 459, 459, 459, 459, 459, 459, 459, - /* 2350 */ 459, 396, 459, 459, 459, 400, 325, 459, 403, 404, - /* 2360 */ 405, 406, 407, 408, 459, 410, 459, 459, 459, 459, - /* 2370 */ 459, 459, 459, 459, 459, 459, 459, 459, 325, 459, - /* 2380 */ 459, 459, 459, 459, 459, 459, 459, 356, 459, 459, - /* 2390 */ 459, 459, 459, 459, 459, 364, 459, 459, 459, 459, - /* 2400 */ 369, 459, 371, 459, 459, 459, 459, 459, 459, 356, - /* 2410 */ 459, 459, 459, 459, 459, 459, 459, 364, 459, 459, - /* 2420 */ 459, 459, 369, 459, 371, 459, 459, 396, 459, 459, - /* 2430 */ 459, 400, 459, 325, 403, 404, 405, 406, 407, 408, - /* 2440 */ 459, 410, 459, 459, 459, 459, 459, 459, 459, 396, - /* 2450 */ 459, 459, 459, 400, 459, 459, 403, 404, 405, 406, - /* 2460 */ 407, 408, 459, 410, 356, 459, 459, 459, 459, 459, - /* 2470 */ 459, 459, 364, 459, 459, 459, 459, 369, 325, 371, - /* 2480 */ 459, 459, 459, 459, 459, 459, 459, 459, 459, 459, - /* 2490 */ 459, 459, 459, 459, 459, 459, 459, 459, 459, 459, - /* 2500 */ 459, 459, 459, 459, 396, 459, 459, 459, 400, 356, - /* 2510 */ 459, 403, 404, 405, 406, 407, 408, 364, 410, 459, - /* 2520 */ 459, 459, 369, 325, 371, 459, 459, 459, 459, 459, - /* 2530 */ 459, 459, 459, 459, 459, 459, 459, 459, 459, 325, - /* 2540 */ 459, 459, 459, 459, 459, 459, 459, 459, 459, 396, - /* 2550 */ 459, 459, 459, 400, 356, 459, 403, 404, 405, 406, - /* 2560 */ 407, 408, 364, 410, 459, 459, 459, 369, 459, 371, - /* 2570 */ 356, 459, 459, 459, 459, 459, 459, 459, 364, 459, - /* 2580 */ 459, 459, 459, 369, 325, 371, 459, 459, 459, 459, - /* 2590 */ 459, 459, 459, 459, 396, 459, 459, 459, 400, 459, - /* 2600 */ 459, 403, 404, 405, 406, 407, 408, 325, 410, 459, - /* 2610 */ 396, 459, 459, 459, 400, 356, 459, 403, 404, 405, - /* 2620 */ 406, 407, 408, 364, 410, 459, 459, 459, 369, 459, - /* 2630 */ 371, 459, 459, 459, 459, 459, 459, 325, 356, 459, - /* 2640 */ 459, 459, 459, 459, 459, 459, 364, 459, 459, 459, - /* 2650 */ 459, 369, 459, 371, 459, 396, 459, 459, 459, 400, - /* 2660 */ 459, 459, 403, 404, 405, 406, 407, 408, 356, 410, - /* 2670 */ 459, 459, 459, 459, 459, 459, 364, 459, 396, 459, - /* 2680 */ 459, 369, 400, 371, 459, 403, 404, 405, 406, 407, - /* 2690 */ 408, 459, 410, 459, 459, 459, 459, 459, 459, 459, - /* 2700 */ 459, 459, 459, 459, 459, 459, 459, 459, 396, 459, - /* 2710 */ 459, 459, 400, 459, 459, 403, 404, 405, 406, 407, - /* 2720 */ 408, 459, 410, + /* 0 */ 375, 333, 371, 335, 336, 374, 375, 333, 362, 335, + /* 10 */ 336, 375, 12, 13, 14, 390, 391, 360, 330, 373, + /* 20 */ 20, 362, 22, 387, 360, 362, 390, 391, 329, 424, + /* 30 */ 425, 367, 373, 33, 329, 35, 373, 20, 381, 382, + /* 40 */ 376, 334, 8, 9, 337, 338, 12, 13, 14, 15, + /* 50 */ 16, 405, 406, 407, 337, 337, 20, 435, 58, 360, + /* 60 */ 20, 439, 416, 63, 405, 406, 407, 368, 405, 406, + /* 70 */ 70, 337, 373, 326, 375, 416, 454, 455, 373, 416, + /* 80 */ 360, 459, 460, 12, 13, 351, 368, 367, 20, 4, + /* 90 */ 20, 20, 358, 22, 377, 95, 376, 63, 20, 435, + /* 100 */ 360, 402, 368, 439, 33, 406, 35, 367, 409, 410, + /* 110 */ 411, 412, 413, 414, 4, 416, 376, 117, 454, 455, + /* 120 */ 421, 58, 423, 459, 460, 58, 427, 428, 43, 58, + /* 130 */ 45, 46, 132, 133, 63, 95, 362, 20, 104, 392, + /* 140 */ 441, 70, 12, 13, 14, 15, 16, 373, 449, 431, + /* 150 */ 432, 433, 334, 435, 436, 337, 338, 439, 95, 360, + /* 160 */ 97, 94, 162, 163, 97, 95, 95, 368, 168, 169, + /* 170 */ 126, 127, 454, 455, 328, 131, 330, 459, 460, 405, + /* 180 */ 406, 14, 435, 183, 39, 185, 439, 20, 117, 37, + /* 190 */ 416, 353, 8, 9, 356, 161, 12, 13, 14, 15, + /* 200 */ 16, 454, 455, 132, 133, 95, 459, 460, 208, 209, + /* 210 */ 411, 211, 212, 213, 214, 215, 216, 217, 218, 219, + /* 220 */ 220, 221, 222, 223, 224, 225, 226, 227, 228, 167, + /* 230 */ 162, 163, 20, 162, 163, 95, 62, 343, 21, 168, + /* 240 */ 169, 24, 25, 26, 27, 28, 29, 30, 31, 32, + /* 250 */ 98, 357, 100, 101, 183, 103, 185, 8, 9, 107, + /* 260 */ 366, 12, 13, 14, 15, 16, 62, 233, 234, 235, + /* 270 */ 236, 237, 238, 239, 240, 241, 242, 243, 360, 208, + /* 280 */ 209, 129, 211, 212, 213, 214, 215, 216, 217, 218, + /* 290 */ 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, + /* 300 */ 382, 230, 12, 13, 0, 337, 20, 329, 22, 246, + /* 310 */ 20, 329, 22, 251, 252, 253, 246, 230, 329, 232, + /* 320 */ 95, 35, 182, 33, 184, 35, 108, 109, 110, 111, + /* 330 */ 112, 113, 114, 115, 116, 117, 118, 51, 120, 121, + /* 340 */ 122, 123, 124, 125, 14, 96, 178, 3, 58, 360, + /* 350 */ 20, 373, 384, 63, 386, 373, 246, 368, 337, 175, + /* 360 */ 70, 20, 373, 22, 375, 20, 20, 199, 200, 65, + /* 370 */ 66, 67, 351, 12, 13, 14, 164, 73, 74, 358, + /* 380 */ 408, 20, 78, 22, 47, 95, 246, 83, 84, 368, + /* 390 */ 245, 402, 51, 89, 33, 406, 35, 337, 409, 410, + /* 400 */ 411, 412, 413, 414, 329, 416, 434, 117, 419, 435, + /* 410 */ 421, 422, 423, 439, 2, 33, 427, 428, 161, 58, + /* 420 */ 8, 9, 132, 133, 12, 13, 14, 15, 16, 455, + /* 430 */ 48, 70, 95, 459, 460, 360, 54, 55, 56, 57, + /* 440 */ 58, 95, 70, 368, 384, 14, 386, 3, 373, 337, + /* 450 */ 375, 20, 162, 163, 270, 20, 95, 21, 168, 169, + /* 460 */ 341, 8, 9, 351, 20, 12, 13, 14, 15, 16, + /* 470 */ 34, 246, 36, 183, 355, 185, 94, 402, 117, 97, + /* 480 */ 368, 406, 363, 79, 409, 410, 411, 412, 413, 414, + /* 490 */ 233, 416, 162, 132, 133, 329, 106, 44, 208, 209, + /* 500 */ 243, 211, 212, 213, 214, 215, 216, 217, 218, 219, + /* 510 */ 220, 221, 222, 223, 224, 225, 226, 227, 228, 65, + /* 520 */ 66, 67, 343, 162, 163, 450, 451, 73, 74, 168, + /* 530 */ 169, 344, 78, 230, 95, 337, 349, 83, 84, 373, + /* 540 */ 96, 137, 138, 89, 183, 366, 185, 165, 166, 351, + /* 550 */ 329, 70, 170, 8, 9, 173, 361, 12, 13, 14, + /* 560 */ 15, 16, 20, 337, 160, 20, 368, 132, 133, 208, + /* 570 */ 209, 189, 211, 212, 213, 214, 215, 216, 217, 218, + /* 580 */ 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, + /* 590 */ 12, 13, 246, 162, 373, 107, 374, 375, 20, 0, + /* 600 */ 22, 346, 347, 168, 169, 208, 329, 0, 164, 265, + /* 610 */ 384, 33, 386, 35, 126, 127, 128, 129, 130, 131, + /* 620 */ 21, 361, 337, 24, 25, 26, 27, 28, 29, 30, + /* 630 */ 31, 32, 1, 2, 8, 9, 58, 360, 12, 13, + /* 640 */ 14, 15, 16, 435, 0, 368, 337, 439, 70, 361, + /* 650 */ 373, 359, 375, 368, 257, 258, 259, 260, 261, 368, + /* 660 */ 351, 12, 13, 455, 372, 360, 127, 459, 460, 20, + /* 670 */ 379, 22, 367, 95, 132, 133, 126, 368, 420, 402, + /* 680 */ 422, 376, 33, 406, 35, 246, 409, 410, 411, 412, + /* 690 */ 413, 414, 329, 416, 337, 117, 360, 0, 421, 420, + /* 700 */ 423, 422, 35, 367, 427, 428, 62, 58, 351, 164, + /* 710 */ 132, 133, 376, 329, 361, 430, 431, 432, 433, 70, + /* 720 */ 435, 436, 96, 360, 0, 368, 449, 96, 361, 190, + /* 730 */ 191, 368, 22, 194, 329, 196, 373, 70, 375, 329, + /* 740 */ 162, 163, 192, 193, 95, 35, 168, 169, 24, 25, + /* 750 */ 26, 27, 28, 29, 30, 31, 32, 373, 151, 62, + /* 760 */ 361, 183, 348, 185, 350, 402, 117, 160, 361, 406, + /* 770 */ 346, 347, 409, 410, 411, 412, 413, 414, 373, 416, + /* 780 */ 70, 132, 133, 373, 408, 329, 208, 209, 392, 211, + /* 790 */ 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, + /* 800 */ 222, 223, 224, 225, 226, 227, 228, 337, 368, 368, + /* 810 */ 434, 162, 163, 18, 451, 20, 337, 168, 169, 379, + /* 820 */ 379, 351, 27, 337, 341, 30, 408, 117, 33, 373, + /* 830 */ 351, 435, 183, 360, 185, 439, 352, 351, 368, 396, + /* 840 */ 14, 15, 16, 48, 360, 50, 363, 368, 53, 376, + /* 850 */ 454, 455, 434, 369, 368, 459, 460, 208, 209, 0, + /* 860 */ 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, + /* 870 */ 221, 222, 223, 224, 225, 226, 227, 228, 0, 352, + /* 880 */ 45, 46, 360, 18, 329, 337, 337, 360, 23, 94, + /* 890 */ 0, 369, 107, 183, 0, 185, 369, 20, 2, 351, + /* 900 */ 351, 106, 37, 38, 8, 9, 41, 48, 12, 13, + /* 910 */ 14, 15, 16, 352, 129, 349, 368, 368, 208, 209, + /* 920 */ 419, 360, 463, 422, 59, 60, 61, 383, 373, 370, + /* 930 */ 369, 136, 373, 4, 139, 140, 141, 142, 143, 144, + /* 940 */ 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, + /* 950 */ 155, 452, 157, 158, 159, 65, 66, 67, 68, 69, + /* 960 */ 95, 71, 72, 73, 74, 75, 76, 77, 78, 79, + /* 970 */ 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + /* 980 */ 90, 91, 329, 4, 20, 107, 8, 9, 337, 22, + /* 990 */ 12, 13, 14, 15, 16, 132, 133, 20, 19, 134, + /* 1000 */ 337, 107, 35, 337, 126, 127, 128, 129, 130, 131, + /* 1010 */ 244, 245, 33, 329, 351, 337, 329, 351, 44, 368, + /* 1020 */ 126, 127, 128, 129, 130, 131, 373, 48, 337, 351, + /* 1030 */ 337, 368, 53, 329, 368, 329, 337, 58, 44, 162, + /* 1040 */ 175, 176, 177, 446, 351, 180, 368, 360, 329, 370, + /* 1050 */ 351, 370, 373, 353, 373, 368, 356, 373, 0, 368, + /* 1060 */ 373, 368, 375, 198, 337, 329, 201, 368, 203, 204, + /* 1070 */ 205, 206, 207, 94, 329, 44, 97, 373, 351, 373, + /* 1080 */ 22, 35, 431, 432, 433, 329, 435, 436, 337, 402, + /* 1090 */ 96, 339, 373, 406, 106, 368, 409, 410, 411, 412, + /* 1100 */ 413, 414, 351, 416, 329, 360, 1, 2, 421, 373, + /* 1110 */ 423, 246, 44, 368, 427, 428, 339, 360, 373, 368, + /* 1120 */ 375, 329, 431, 432, 433, 438, 435, 436, 164, 373, + /* 1130 */ 329, 337, 8, 9, 336, 360, 12, 13, 14, 15, + /* 1140 */ 16, 164, 164, 368, 156, 351, 329, 402, 373, 337, + /* 1150 */ 375, 406, 360, 0, 409, 410, 411, 412, 413, 414, + /* 1160 */ 368, 416, 368, 351, 96, 373, 421, 375, 423, 42, + /* 1170 */ 0, 44, 427, 428, 373, 22, 247, 402, 329, 58, + /* 1180 */ 368, 406, 392, 438, 409, 410, 411, 412, 413, 414, + /* 1190 */ 373, 416, 22, 42, 402, 44, 99, 99, 406, 102, + /* 1200 */ 102, 409, 410, 411, 412, 413, 414, 35, 416, 360, + /* 1210 */ 383, 99, 372, 421, 102, 423, 164, 368, 97, 427, + /* 1220 */ 428, 392, 373, 171, 375, 435, 44, 63, 453, 439, + /* 1230 */ 438, 185, 99, 12, 13, 102, 329, 383, 44, 437, + /* 1240 */ 44, 267, 44, 22, 454, 455, 440, 44, 429, 459, + /* 1250 */ 460, 402, 44, 0, 33, 406, 35, 248, 409, 410, + /* 1260 */ 411, 412, 413, 414, 435, 416, 329, 360, 439, 456, + /* 1270 */ 421, 35, 423, 44, 44, 368, 427, 428, 96, 58, + /* 1280 */ 373, 44, 375, 454, 455, 95, 13, 438, 459, 460, + /* 1290 */ 96, 70, 96, 404, 96, 105, 48, 360, 329, 96, + /* 1300 */ 269, 403, 49, 44, 96, 368, 70, 44, 35, 402, + /* 1310 */ 373, 44, 375, 406, 394, 44, 409, 410, 411, 412, + /* 1320 */ 413, 414, 44, 416, 44, 96, 96, 181, 421, 360, + /* 1330 */ 423, 42, 13, 96, 427, 428, 44, 368, 117, 402, + /* 1340 */ 20, 380, 373, 406, 375, 438, 409, 410, 411, 412, + /* 1350 */ 413, 414, 392, 416, 35, 96, 383, 185, 421, 96, + /* 1360 */ 423, 380, 161, 96, 427, 428, 20, 96, 378, 337, + /* 1370 */ 378, 402, 208, 337, 96, 406, 96, 329, 409, 410, + /* 1380 */ 411, 412, 413, 414, 380, 416, 378, 93, 96, 345, + /* 1390 */ 421, 337, 423, 337, 337, 435, 427, 428, 20, 439, + /* 1400 */ 331, 331, 20, 398, 183, 343, 185, 20, 360, 375, + /* 1410 */ 343, 20, 338, 20, 454, 455, 368, 393, 343, 459, + /* 1420 */ 460, 373, 338, 375, 343, 392, 343, 337, 343, 208, + /* 1430 */ 209, 343, 52, 340, 340, 329, 360, 360, 331, 360, + /* 1440 */ 360, 337, 221, 222, 223, 224, 225, 226, 227, 360, + /* 1450 */ 402, 360, 360, 331, 406, 329, 360, 409, 410, 411, + /* 1460 */ 412, 413, 414, 360, 416, 197, 360, 401, 435, 421, + /* 1470 */ 360, 423, 439, 360, 368, 427, 428, 341, 95, 373, + /* 1480 */ 398, 375, 373, 188, 373, 341, 360, 454, 455, 375, + /* 1490 */ 256, 337, 459, 460, 368, 400, 397, 255, 383, 373, + /* 1500 */ 383, 375, 373, 373, 445, 445, 388, 388, 402, 262, + /* 1510 */ 373, 373, 406, 329, 174, 409, 410, 411, 412, 413, + /* 1520 */ 414, 448, 416, 249, 264, 447, 271, 263, 402, 423, + /* 1530 */ 443, 445, 406, 427, 428, 409, 410, 411, 412, 413, + /* 1540 */ 414, 266, 416, 329, 360, 444, 404, 268, 245, 423, + /* 1550 */ 442, 368, 368, 427, 428, 20, 408, 373, 338, 375, + /* 1560 */ 329, 337, 20, 341, 464, 386, 341, 166, 388, 388, + /* 1570 */ 373, 385, 373, 457, 360, 373, 341, 373, 458, 373, + /* 1580 */ 373, 341, 368, 356, 368, 95, 402, 373, 426, 375, + /* 1590 */ 406, 360, 95, 409, 410, 411, 412, 413, 414, 368, + /* 1600 */ 416, 373, 364, 337, 373, 36, 375, 423, 350, 332, + /* 1610 */ 341, 427, 428, 331, 395, 389, 402, 329, 342, 354, + /* 1620 */ 406, 389, 399, 409, 410, 411, 412, 413, 414, 415, + /* 1630 */ 416, 417, 418, 402, 327, 354, 0, 406, 354, 0, + /* 1640 */ 409, 410, 411, 412, 413, 414, 190, 416, 360, 0, + /* 1650 */ 0, 42, 0, 35, 202, 35, 368, 35, 35, 202, + /* 1660 */ 0, 373, 35, 375, 329, 35, 202, 0, 202, 0, + /* 1670 */ 35, 0, 0, 22, 185, 35, 183, 0, 0, 179, + /* 1680 */ 178, 329, 0, 0, 47, 0, 0, 0, 0, 42, + /* 1690 */ 402, 0, 461, 462, 406, 360, 0, 409, 410, 411, + /* 1700 */ 412, 413, 414, 368, 416, 0, 0, 0, 373, 151, + /* 1710 */ 375, 423, 360, 0, 0, 35, 428, 365, 0, 151, + /* 1720 */ 368, 0, 0, 0, 0, 373, 0, 375, 0, 0, + /* 1730 */ 0, 0, 0, 0, 0, 0, 329, 402, 0, 0, + /* 1740 */ 0, 406, 42, 0, 409, 410, 411, 412, 413, 414, + /* 1750 */ 0, 416, 0, 0, 402, 0, 0, 0, 406, 329, + /* 1760 */ 0, 409, 410, 411, 412, 413, 414, 360, 416, 22, + /* 1770 */ 0, 0, 0, 135, 0, 368, 35, 58, 0, 0, + /* 1780 */ 373, 0, 375, 58, 42, 39, 0, 14, 44, 14, + /* 1790 */ 360, 329, 47, 0, 0, 365, 47, 462, 368, 40, + /* 1800 */ 39, 47, 0, 373, 39, 375, 174, 0, 0, 402, + /* 1810 */ 0, 0, 64, 406, 0, 35, 409, 410, 411, 412, + /* 1820 */ 413, 414, 360, 416, 48, 418, 39, 365, 0, 39, + /* 1830 */ 368, 35, 402, 48, 0, 373, 406, 375, 329, 409, + /* 1840 */ 410, 411, 412, 413, 414, 48, 416, 35, 39, 0, + /* 1850 */ 35, 48, 19, 0, 0, 39, 0, 0, 35, 22, + /* 1860 */ 102, 0, 329, 44, 402, 35, 33, 104, 406, 360, + /* 1870 */ 35, 409, 410, 411, 412, 413, 414, 368, 416, 35, + /* 1880 */ 35, 48, 373, 44, 375, 35, 35, 54, 55, 56, + /* 1890 */ 57, 58, 22, 360, 35, 0, 22, 0, 22, 0, + /* 1900 */ 22, 368, 50, 35, 0, 0, 373, 35, 375, 0, + /* 1910 */ 35, 402, 22, 20, 35, 406, 35, 96, 409, 410, + /* 1920 */ 411, 412, 413, 414, 95, 416, 0, 94, 329, 35, + /* 1930 */ 97, 195, 164, 0, 22, 402, 0, 186, 0, 406, + /* 1940 */ 3, 164, 409, 410, 411, 412, 413, 414, 166, 416, + /* 1950 */ 329, 44, 164, 250, 96, 172, 229, 95, 254, 360, + /* 1960 */ 95, 44, 171, 130, 3, 96, 171, 368, 44, 96, + /* 1970 */ 47, 44, 373, 96, 375, 329, 47, 95, 95, 95, + /* 1980 */ 95, 360, 96, 96, 44, 35, 35, 35, 35, 368, + /* 1990 */ 35, 35, 96, 47, 373, 47, 375, 96, 165, 39, + /* 2000 */ 0, 402, 44, 170, 0, 406, 360, 0, 409, 410, + /* 2010 */ 411, 412, 413, 414, 368, 416, 250, 0, 0, 373, + /* 2020 */ 187, 375, 189, 402, 47, 95, 95, 406, 244, 95, + /* 2030 */ 409, 410, 411, 412, 413, 414, 95, 416, 329, 250, + /* 2040 */ 96, 96, 95, 47, 39, 44, 229, 167, 402, 95, + /* 2050 */ 2, 231, 406, 229, 165, 409, 410, 411, 412, 413, + /* 2060 */ 414, 22, 416, 208, 105, 95, 329, 96, 95, 360, + /* 2070 */ 96, 95, 47, 47, 96, 95, 95, 368, 106, 96, + /* 2080 */ 22, 210, 373, 95, 375, 96, 35, 35, 95, 35, + /* 2090 */ 96, 96, 95, 95, 35, 35, 329, 360, 96, 96, + /* 2100 */ 35, 95, 22, 95, 119, 368, 119, 95, 44, 119, + /* 2110 */ 373, 402, 375, 22, 95, 406, 107, 35, 409, 410, + /* 2120 */ 411, 412, 413, 414, 95, 416, 64, 360, 329, 119, + /* 2130 */ 63, 35, 35, 35, 35, 368, 35, 35, 35, 402, + /* 2140 */ 373, 35, 375, 406, 35, 35, 409, 410, 411, 412, + /* 2150 */ 413, 414, 44, 416, 70, 92, 35, 35, 35, 360, + /* 2160 */ 329, 22, 35, 35, 35, 70, 35, 368, 35, 402, + /* 2170 */ 35, 35, 373, 406, 375, 35, 409, 410, 411, 412, + /* 2180 */ 413, 414, 22, 416, 35, 0, 35, 39, 0, 48, + /* 2190 */ 35, 360, 329, 48, 0, 39, 35, 39, 48, 368, + /* 2200 */ 0, 402, 35, 39, 373, 406, 375, 48, 409, 410, + /* 2210 */ 411, 412, 413, 414, 0, 416, 35, 329, 35, 0, + /* 2220 */ 22, 22, 21, 360, 22, 21, 20, 465, 465, 465, + /* 2230 */ 465, 368, 465, 402, 465, 465, 373, 406, 375, 465, + /* 2240 */ 409, 410, 411, 412, 413, 414, 465, 416, 360, 329, + /* 2250 */ 465, 465, 465, 465, 465, 465, 368, 465, 465, 465, + /* 2260 */ 465, 373, 465, 375, 465, 402, 465, 465, 465, 406, + /* 2270 */ 465, 465, 409, 410, 411, 412, 413, 414, 465, 416, + /* 2280 */ 360, 465, 465, 465, 465, 465, 465, 465, 368, 465, + /* 2290 */ 402, 465, 465, 373, 406, 375, 465, 409, 410, 411, + /* 2300 */ 412, 413, 414, 465, 416, 465, 465, 465, 465, 465, + /* 2310 */ 465, 329, 465, 465, 465, 465, 465, 465, 465, 465, + /* 2320 */ 465, 465, 402, 465, 465, 465, 406, 465, 329, 409, + /* 2330 */ 410, 411, 412, 413, 414, 465, 416, 465, 465, 465, + /* 2340 */ 465, 465, 360, 465, 465, 465, 465, 465, 465, 465, + /* 2350 */ 368, 465, 465, 465, 465, 373, 465, 375, 465, 360, + /* 2360 */ 465, 465, 465, 465, 465, 465, 465, 368, 465, 465, + /* 2370 */ 465, 465, 373, 465, 375, 465, 465, 465, 465, 465, + /* 2380 */ 465, 465, 465, 465, 402, 329, 465, 465, 406, 465, + /* 2390 */ 465, 409, 410, 411, 412, 413, 414, 465, 416, 465, + /* 2400 */ 465, 402, 465, 465, 465, 406, 465, 329, 409, 410, + /* 2410 */ 411, 412, 413, 414, 465, 416, 360, 465, 465, 465, + /* 2420 */ 465, 465, 465, 465, 368, 465, 465, 465, 465, 373, + /* 2430 */ 465, 375, 465, 465, 465, 465, 465, 465, 360, 465, + /* 2440 */ 465, 465, 465, 465, 465, 465, 368, 465, 465, 465, + /* 2450 */ 465, 373, 465, 375, 465, 465, 465, 465, 402, 465, + /* 2460 */ 465, 465, 406, 465, 465, 409, 410, 411, 412, 413, + /* 2470 */ 414, 465, 416, 329, 465, 465, 465, 465, 465, 465, + /* 2480 */ 402, 465, 465, 465, 406, 465, 465, 409, 410, 411, + /* 2490 */ 412, 413, 414, 465, 416, 465, 465, 329, 465, 465, + /* 2500 */ 465, 465, 465, 465, 360, 465, 465, 465, 465, 465, + /* 2510 */ 465, 465, 368, 465, 465, 465, 465, 373, 465, 375, + /* 2520 */ 465, 465, 465, 465, 465, 465, 465, 465, 360, 465, + /* 2530 */ 465, 465, 465, 465, 465, 465, 368, 465, 465, 465, + /* 2540 */ 465, 373, 465, 375, 465, 465, 402, 465, 465, 465, + /* 2550 */ 406, 465, 465, 409, 410, 411, 412, 413, 414, 465, + /* 2560 */ 416, 329, 465, 465, 465, 465, 465, 465, 465, 465, + /* 2570 */ 402, 465, 465, 465, 406, 465, 465, 409, 410, 411, + /* 2580 */ 412, 413, 414, 465, 416, 465, 329, 465, 465, 465, + /* 2590 */ 465, 465, 360, 465, 465, 465, 465, 465, 465, 465, + /* 2600 */ 368, 465, 465, 465, 465, 373, 465, 375, 465, 465, + /* 2610 */ 465, 465, 465, 465, 465, 465, 465, 360, 329, 465, + /* 2620 */ 465, 465, 465, 465, 465, 368, 465, 465, 465, 465, + /* 2630 */ 373, 465, 375, 465, 402, 465, 465, 465, 406, 465, + /* 2640 */ 465, 409, 410, 411, 412, 413, 414, 465, 416, 360, + /* 2650 */ 465, 465, 465, 465, 465, 465, 465, 368, 465, 402, + /* 2660 */ 465, 465, 373, 406, 375, 465, 409, 410, 411, 412, + /* 2670 */ 413, 414, 465, 416, 465, 465, 465, 465, 465, 465, + /* 2680 */ 465, 465, 465, 465, 465, 465, 465, 465, 465, 465, + /* 2690 */ 465, 402, 465, 465, 465, 406, 465, 465, 409, 410, + /* 2700 */ 411, 412, 413, 414, 465, 416, }; -#define YY_SHIFT_COUNT (710) +#define YY_SHIFT_COUNT (729) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2065) +#define YY_SHIFT_MAX (2219) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 925, 0, 71, 0, 288, 288, 288, 288, 288, 288, - /* 10 */ 288, 288, 288, 359, 574, 574, 645, 574, 574, 574, - /* 20 */ 574, 574, 574, 574, 574, 574, 574, 574, 574, 574, - /* 30 */ 574, 574, 574, 574, 574, 574, 574, 574, 574, 574, - /* 40 */ 574, 574, 574, 574, 574, 574, 103, 227, 51, 169, - /* 50 */ 287, 134, 208, 134, 51, 51, 1182, 1182, 134, 1182, - /* 60 */ 1182, 203, 134, 5, 5, 772, 772, 14, 52, 45, - /* 70 */ 45, 5, 5, 5, 5, 5, 5, 5, 40, 5, - /* 80 */ 5, 28, 5, 5, 177, 5, 404, 5, 40, 543, - /* 90 */ 5, 5, 543, 5, 543, 543, 543, 5, 57, 783, - /* 100 */ 34, 34, 498, 471, 418, 418, 418, 418, 418, 418, - /* 110 */ 418, 418, 418, 418, 418, 418, 418, 418, 418, 418, - /* 120 */ 418, 418, 418, 213, 661, 14, 52, 312, 325, 172, - /* 130 */ 172, 172, 599, 607, 607, 325, 583, 583, 583, 784, - /* 140 */ 404, 546, 543, 869, 543, 869, 869, 784, 890, 216, - /* 150 */ 216, 216, 216, 216, 216, 216, 790, 434, 302, 711, - /* 160 */ 1055, 99, 96, 94, 295, 532, 339, 533, 899, 889, - /* 170 */ 853, 766, 804, 104, 766, 933, 724, 858, 1049, 1248, - /* 180 */ 1124, 1264, 1291, 1264, 1162, 1305, 1305, 1264, 1162, 1162, - /* 190 */ 1240, 1305, 1305, 1305, 1321, 1321, 1324, 28, 404, 28, - /* 200 */ 1329, 1339, 28, 1329, 28, 28, 28, 1305, 28, 1319, - /* 210 */ 1319, 1321, 543, 543, 543, 543, 543, 543, 543, 543, - /* 220 */ 543, 543, 543, 1305, 1321, 869, 869, 1200, 1324, 57, - /* 230 */ 1226, 404, 57, 1305, 1291, 1291, 869, 1189, 1188, 869, - /* 240 */ 1189, 1188, 869, 869, 543, 1186, 1293, 1189, 1206, 1208, - /* 250 */ 1218, 1049, 1210, 1215, 1207, 1239, 583, 1466, 1305, 1329, - /* 260 */ 57, 1472, 1188, 869, 869, 869, 869, 869, 1188, 869, - /* 270 */ 1330, 57, 784, 57, 583, 1427, 1443, 869, 890, 1305, - /* 280 */ 57, 1505, 1321, 2723, 2723, 2723, 2723, 2723, 2723, 2723, - /* 290 */ 2723, 2723, 826, 1187, 588, 656, 938, 998, 1064, 556, - /* 300 */ 686, 757, 867, 852, 979, 979, 979, 979, 979, 979, - /* 310 */ 979, 979, 979, 896, 641, 155, 155, 191, 534, 592, - /* 320 */ 572, 300, 435, 435, 1, 625, 573, 1, 1, 1, - /* 330 */ 1000, 547, 1031, 1007, 952, 839, 983, 984, 991, 1012, - /* 340 */ 1088, 1094, 1109, 959, 1060, 1071, 989, 1040, 384, 1010, - /* 350 */ 1075, 1086, 1104, 1126, 1130, 1204, 1136, 767, 1054, 823, - /* 360 */ 1155, 951, 1157, 1172, 1193, 1202, 1205, 1212, 1114, 1177, - /* 370 */ 1194, 1237, 732, 1556, 1557, 1371, 1562, 1563, 1518, 1565, - /* 380 */ 1533, 1374, 1535, 1539, 1541, 1379, 1577, 1544, 1546, 1384, - /* 390 */ 1584, 1386, 1586, 1552, 1588, 1567, 1590, 1558, 1410, 1414, - /* 400 */ 1596, 1604, 1430, 1433, 1609, 1610, 1566, 1612, 1620, 1622, - /* 410 */ 1581, 1624, 1625, 1626, 1627, 1637, 1638, 1644, 1645, 1478, - /* 420 */ 1595, 1631, 1481, 1633, 1634, 1635, 1646, 1647, 1648, 1649, - /* 430 */ 1650, 1651, 1652, 1653, 1654, 1655, 1657, 1658, 1617, 1660, - /* 440 */ 1661, 1662, 1663, 1664, 1643, 1666, 1667, 1669, 1536, 1670, - /* 450 */ 1683, 1639, 1684, 1632, 1685, 1636, 1687, 1689, 1656, 1665, - /* 460 */ 1668, 1659, 1677, 1671, 1678, 1681, 1693, 1675, 1672, 1695, - /* 470 */ 1696, 1697, 1690, 1528, 1700, 1701, 1702, 1641, 1707, 1708, - /* 480 */ 1679, 1682, 1692, 1709, 1698, 1688, 1699, 1710, 1705, 1703, - /* 490 */ 1704, 1716, 1706, 1711, 1713, 1732, 1735, 1737, 1744, 1642, - /* 500 */ 1673, 1715, 1723, 1748, 1718, 1719, 1720, 1721, 1714, 1717, - /* 510 */ 1725, 1728, 1740, 1729, 1766, 1745, 1768, 1747, 1726, 1771, - /* 520 */ 1755, 1754, 1779, 1756, 1780, 1757, 1790, 1772, 1773, 1760, - /* 530 */ 1761, 1606, 1724, 1730, 1799, 1640, 1765, 1802, 1676, 1792, - /* 540 */ 1680, 1686, 1810, 1812, 1691, 1712, 1813, 1775, 1573, 1733, - /* 550 */ 1731, 1734, 1736, 1778, 1738, 1741, 1742, 1746, 1739, 1786, - /* 560 */ 1777, 1791, 1749, 1795, 1594, 1750, 1751, 1842, 1805, 1602, - /* 570 */ 1817, 1820, 1822, 1825, 1829, 1838, 1781, 1782, 1827, 1694, - /* 580 */ 1831, 1833, 1815, 1876, 1882, 1659, 1836, 1794, 1797, 1798, - /* 590 */ 1796, 1808, 1762, 1809, 1887, 1851, 1753, 1811, 1800, 1659, - /* 600 */ 1860, 1864, 1722, 1758, 1763, 1907, 1888, 1752, 1818, 1816, - /* 610 */ 1819, 1830, 1832, 1834, 1886, 1839, 1840, 1890, 1843, 1916, - /* 620 */ 1764, 1846, 1837, 1848, 1911, 1913, 1847, 1854, 1917, 1856, - /* 630 */ 1859, 1918, 1863, 1865, 1924, 1867, 1873, 1938, 1879, 1858, - /* 640 */ 1869, 1870, 1872, 1953, 1871, 1897, 1950, 1900, 1961, 1906, - /* 650 */ 1950, 1950, 1975, 1939, 1941, 1967, 1970, 1971, 1972, 1973, - /* 660 */ 1974, 1976, 1977, 1978, 1979, 1940, 1923, 1980, 1981, 1982, - /* 670 */ 1983, 1997, 1985, 1986, 1987, 1956, 1714, 1988, 1717, 1992, - /* 680 */ 1993, 1996, 2005, 2019, 2007, 2044, 2011, 1999, 2009, 2049, - /* 690 */ 2015, 2003, 2014, 2054, 2020, 2008, 2018, 2058, 2024, 2012, - /* 700 */ 2022, 2062, 2028, 2029, 2065, 2045, 2048, 2050, 2051, 2055, - /* 710 */ 2046, + /* 0 */ 865, 0, 71, 0, 290, 290, 290, 290, 290, 290, + /* 10 */ 290, 290, 290, 290, 290, 361, 578, 578, 649, 578, + /* 20 */ 578, 578, 578, 578, 578, 578, 578, 578, 578, 578, + /* 30 */ 578, 578, 578, 578, 578, 578, 578, 578, 578, 578, + /* 40 */ 578, 578, 578, 578, 578, 578, 578, 578, 70, 346, + /* 50 */ 40, 140, 63, 225, 439, 225, 40, 40, 1221, 1221, + /* 60 */ 225, 1221, 1221, 110, 225, 435, 17, 17, 435, 85, + /* 70 */ 85, 68, 542, 167, 167, 17, 17, 17, 17, 17, + /* 80 */ 17, 17, 36, 17, 17, 174, 78, 17, 17, 117, + /* 90 */ 17, 78, 17, 36, 17, 36, 78, 17, 17, 78, + /* 100 */ 17, 78, 78, 78, 17, 204, 795, 34, 34, 217, + /* 110 */ 454, 710, 710, 710, 710, 710, 710, 710, 710, 710, + /* 120 */ 710, 710, 710, 710, 710, 710, 710, 710, 710, 710, + /* 130 */ 152, 444, 68, 542, 644, 667, 212, 212, 212, 697, + /* 140 */ 87, 87, 667, 345, 345, 345, 390, 303, 78, 372, + /* 150 */ 78, 372, 372, 390, 481, 218, 218, 218, 218, 218, + /* 160 */ 218, 218, 1833, 599, 304, 545, 184, 397, 286, 62, + /* 170 */ 330, 431, 341, 964, 835, 785, 977, 766, 145, 344, + /* 180 */ 766, 1127, 929, 877, 1009, 1248, 1146, 1289, 1320, 1289, + /* 190 */ 1201, 1346, 1346, 1289, 1201, 1201, 1294, 1346, 1346, 1346, + /* 200 */ 1378, 1378, 1382, 174, 1387, 174, 1391, 1393, 174, 1391, + /* 210 */ 174, 174, 174, 1346, 174, 1380, 1380, 1378, 78, 78, + /* 220 */ 78, 78, 78, 78, 78, 78, 78, 78, 78, 1346, + /* 230 */ 1378, 372, 372, 1268, 1383, 1382, 204, 1295, 1387, 204, + /* 240 */ 1346, 1320, 1320, 372, 1234, 1242, 372, 1234, 1242, 372, + /* 250 */ 372, 78, 1247, 1340, 1234, 1260, 1264, 1274, 1009, 1255, + /* 260 */ 1279, 1275, 1303, 345, 1535, 1346, 1391, 204, 204, 1542, + /* 270 */ 1242, 372, 372, 372, 372, 372, 1242, 372, 1401, 204, + /* 280 */ 390, 204, 345, 1490, 1497, 372, 481, 1346, 204, 1569, + /* 290 */ 1378, 2706, 2706, 2706, 2706, 2706, 2706, 2706, 2706, 2706, + /* 300 */ 890, 382, 724, 979, 249, 453, 626, 878, 412, 896, + /* 310 */ 978, 894, 1124, 1124, 1124, 1124, 1124, 1124, 1124, 1124, + /* 320 */ 1124, 488, 539, 130, 130, 404, 168, 607, 67, 436, + /* 330 */ 550, 44, 44, 826, 631, 257, 826, 826, 826, 994, + /* 340 */ 859, 967, 1151, 988, 1097, 1098, 1112, 1133, 1058, 1153, + /* 350 */ 1170, 1068, 1121, 1182, 1194, 863, 974, 1031, 1052, 1196, + /* 360 */ 1198, 1203, 1229, 1230, 1237, 1105, 1259, 1046, 1172, 1164, + /* 370 */ 1208, 337, 1263, 1267, 1271, 1278, 1280, 1292, 1190, 1273, + /* 380 */ 1319, 1236, 1253, 1636, 1639, 1456, 1649, 1650, 1609, 1652, + /* 390 */ 1618, 1452, 1620, 1622, 1623, 1457, 1660, 1627, 1630, 1464, + /* 400 */ 1667, 1466, 1669, 1635, 1671, 1651, 1672, 1640, 1489, 1493, + /* 410 */ 1677, 1678, 1500, 1502, 1682, 1683, 1637, 1685, 1686, 1687, + /* 420 */ 1647, 1688, 1691, 1696, 1705, 1706, 1707, 1713, 1714, 1558, + /* 430 */ 1680, 1718, 1568, 1721, 1722, 1723, 1724, 1726, 1728, 1729, + /* 440 */ 1730, 1731, 1732, 1733, 1734, 1735, 1738, 1739, 1740, 1700, + /* 450 */ 1743, 1750, 1752, 1753, 1755, 1756, 1747, 1757, 1760, 1770, + /* 460 */ 1638, 1771, 1772, 1741, 1774, 1719, 1778, 1725, 1779, 1781, + /* 470 */ 1742, 1746, 1744, 1745, 1773, 1749, 1775, 1754, 1786, 1759, + /* 480 */ 1761, 1793, 1794, 1802, 1765, 1632, 1807, 1808, 1810, 1748, + /* 490 */ 1811, 1814, 1780, 1776, 1787, 1828, 1796, 1785, 1790, 1834, + /* 500 */ 1812, 1797, 1809, 1849, 1815, 1803, 1816, 1853, 1854, 1856, + /* 510 */ 1857, 1763, 1758, 1823, 1837, 1861, 1830, 1835, 1844, 1845, + /* 520 */ 1819, 1839, 1850, 1851, 1870, 1859, 1895, 1874, 1897, 1876, + /* 530 */ 1852, 1899, 1878, 1868, 1904, 1872, 1905, 1875, 1909, 1890, + /* 540 */ 1893, 1879, 1881, 1736, 1821, 1829, 1926, 1768, 1894, 1933, + /* 550 */ 1751, 1912, 1777, 1782, 1936, 1938, 1788, 1783, 1937, 1907, + /* 560 */ 1703, 1862, 1858, 1865, 1791, 1727, 1795, 1704, 1869, 1917, + /* 570 */ 1873, 1882, 1883, 1884, 1877, 1924, 1923, 1929, 1885, 1927, + /* 580 */ 1766, 1886, 1887, 1961, 1940, 1789, 1950, 1951, 1952, 1953, + /* 590 */ 1955, 1956, 1896, 1901, 1946, 1784, 1958, 1948, 2004, 2007, + /* 600 */ 2017, 2018, 1930, 1960, 1745, 1977, 1931, 1944, 1945, 1934, + /* 610 */ 1941, 1880, 1947, 2000, 2005, 1889, 1954, 1959, 1745, 1996, + /* 620 */ 2001, 1817, 1820, 1824, 2048, 2039, 1855, 1970, 1971, 1973, + /* 630 */ 1974, 1976, 1978, 2025, 1980, 1981, 2026, 1983, 2058, 1871, + /* 640 */ 1988, 1972, 1989, 2051, 2052, 1993, 1994, 2054, 1997, 1995, + /* 650 */ 2059, 1998, 2002, 2060, 2006, 2003, 2065, 2008, 1985, 1987, + /* 660 */ 1990, 2010, 2080, 2009, 2012, 2064, 2019, 2082, 2029, 2064, + /* 670 */ 2064, 2091, 2062, 2067, 2096, 2097, 2098, 2099, 2101, 2102, + /* 680 */ 2103, 2106, 2109, 2110, 2084, 2063, 2108, 2121, 2122, 2123, + /* 690 */ 2139, 2127, 2128, 2129, 2095, 1819, 2131, 1839, 2133, 2135, + /* 700 */ 2136, 2140, 2160, 2149, 2185, 2151, 2141, 2148, 2188, 2155, + /* 710 */ 2145, 2156, 2194, 2161, 2150, 2158, 2200, 2167, 2159, 2164, + /* 720 */ 2214, 2181, 2183, 2219, 2198, 2201, 2199, 2202, 2204, 2206, }; -#define YY_REDUCE_COUNT (291) -#define YY_REDUCE_MIN (-425) -#define YY_REDUCE_MAX (2312) +#define YY_REDUCE_COUNT (299) +#define YY_REDUCE_MIN (-395) +#define YY_REDUCE_MAX (2289) static const short yy_reduce_ofst[] = { - /* 0 */ -17, -295, -7, 233, 633, 736, 781, 854, 909, 1021, - /* 10 */ 1051, 1107, 1127, -169, 1195, 77, 341, 1211, 1272, 1317, - /* 20 */ 1378, 1401, 1462, 1492, 1515, 1560, 1576, 1629, 1674, 1727, - /* 30 */ 1743, 1788, 1841, 1857, 1902, 1925, 1955, 2031, 2053, 2108, - /* 40 */ 2153, 2198, 2214, 2259, 2282, 2312, -276, -290, -168, -10, - /* 50 */ 834, 837, 907, 960, -327, -181, -348, -314, -425, 252, - /* 60 */ 755, -422, 289, -328, -219, -329, -291, -300, -303, -209, - /* 70 */ -144, -173, 78, 258, 278, 326, 668, 688, -288, 698, - /* 80 */ 727, 65, 748, 754, -275, 770, 86, 800, -152, -347, - /* 90 */ 802, 828, 123, 829, 186, 187, 192, 864, -158, -294, - /* 100 */ -282, -282, -182, -265, -170, -62, -6, 82, 105, 265, - /* 110 */ 276, 305, 310, 347, 383, 386, 392, 439, 442, 450, - /* 120 */ 480, 630, 631, 67, -150, 89, -116, 308, 173, -150, - /* 130 */ 311, 332, 309, 406, 426, 175, -58, 466, 521, 126, - /* 140 */ 306, 240, 456, 552, 516, 595, 601, 616, 627, 615, - /* 150 */ 648, 700, 718, 806, 815, 851, -337, 812, 814, 835, - /* 160 */ 726, 785, 895, 793, 882, 882, 904, 868, 916, 887, - /* 170 */ 898, 843, 843, 842, 843, 856, 859, 882, 902, 900, - /* 180 */ 915, 934, 939, 944, 949, 994, 995, 953, 958, 961, - /* 190 */ 993, 1004, 1005, 1006, 1015, 1016, 955, 1011, 974, 1014, - /* 200 */ 1024, 975, 1026, 1032, 1028, 1029, 1030, 1037, 1033, 1038, - /* 210 */ 1039, 1052, 1022, 1025, 1027, 1035, 1041, 1042, 1050, 1056, - /* 220 */ 1062, 1063, 1067, 1047, 1057, 1013, 1017, 1001, 1002, 1073, - /* 230 */ 1043, 1045, 1093, 1100, 1058, 1061, 1065, 1009, 1066, 1076, - /* 240 */ 1023, 1069, 1077, 1080, 882, 1018, 1034, 1046, 1036, 1044, - /* 250 */ 1048, 1072, 1068, 1053, 1070, 843, 1116, 1085, 1156, 1154, - /* 260 */ 1153, 1111, 1110, 1128, 1131, 1132, 1133, 1135, 1115, 1137, - /* 270 */ 1138, 1171, 1164, 1181, 1145, 1105, 1179, 1173, 1197, 1213, - /* 280 */ 1203, 1217, 1220, 1160, 1158, 1168, 1169, 1219, 1221, 1222, - /* 290 */ 1223, 1232, + /* 0 */ -253, -301, -11, 277, 687, 745, 792, 849, 907, 937, + /* 10 */ 969, 1048, 1106, 1126, 1184, 1214, 75, 1231, 1288, 363, + /* 20 */ 775, 1335, 1352, 1407, 1430, 1462, 1509, 1533, 1599, 1621, + /* 30 */ 1646, 1709, 1737, 1767, 1799, 1831, 1863, 1888, 1920, 1982, + /* 40 */ 1999, 2056, 2078, 2144, 2168, 2232, 2257, 2289, -282, -336, + /* 50 */ 285, 396, 790, 829, 960, 1033, 651, 691, -354, -341, + /* 60 */ -378, -337, -226, -26, 208, -364, -266, 21, -375, -332, + /* 70 */ -326, -343, -369, -293, -182, 112, 198, 309, 357, 470, + /* 80 */ 479, 486, -32, 548, 549, -106, -280, 663, 678, -201, + /* 90 */ 693, -260, 699, 60, 727, 226, 484, 751, 794, 305, + /* 100 */ 666, 527, 336, 561, 812, 119, -283, -395, -395, -154, + /* 110 */ 187, -295, -22, -18, 166, 221, 384, 405, 410, 456, + /* 120 */ 555, 653, 684, 704, 706, 719, 736, 756, 801, 817, + /* 130 */ 292, -28, -82, 222, 179, 255, -28, 376, 418, 483, + /* 140 */ 258, 279, 424, 291, 440, 441, -162, 501, 522, 559, + /* 150 */ 473, 679, 681, 700, 414, 195, 260, 288, 353, 367, + /* 160 */ 399, 407, 443, -312, 566, 544, 459, 499, 752, 597, + /* 170 */ 757, 757, 777, 827, 798, 840, 854, 802, 802, 813, + /* 180 */ 802, 819, 806, 757, 889, 898, 920, 961, 973, 981, + /* 190 */ 990, 1032, 1036, 1004, 992, 1008, 1044, 1054, 1056, 1057, + /* 200 */ 1069, 1070, 1005, 1062, 1034, 1067, 1074, 1024, 1075, 1084, + /* 210 */ 1081, 1083, 1085, 1090, 1088, 1093, 1094, 1107, 1076, 1077, + /* 220 */ 1079, 1080, 1089, 1091, 1092, 1096, 1103, 1110, 1113, 1104, + /* 230 */ 1122, 1109, 1111, 1066, 1095, 1082, 1136, 1099, 1114, 1144, + /* 240 */ 1154, 1115, 1117, 1129, 1059, 1118, 1130, 1060, 1119, 1137, + /* 250 */ 1138, 757, 1073, 1078, 1086, 1101, 1087, 1108, 1142, 1100, + /* 260 */ 1120, 1116, 802, 1183, 1148, 1224, 1220, 1222, 1225, 1179, + /* 270 */ 1180, 1197, 1199, 1202, 1204, 1206, 1181, 1207, 1186, 1235, + /* 280 */ 1227, 1240, 1216, 1162, 1238, 1228, 1258, 1266, 1269, 1277, + /* 290 */ 1282, 1219, 1223, 1226, 1232, 1265, 1281, 1284, 1276, 1307, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 10 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 20 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 30 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 40 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 50 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 60 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1850, 1595, 1595, - /* 70 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 80 */ 1595, 1673, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 90 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1671, 1843, - /* 100 */ 2043, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 110 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 120 */ 1595, 1595, 1595, 1595, 2055, 1595, 1595, 1673, 1595, 2055, - /* 130 */ 2055, 2055, 1671, 2015, 2015, 1595, 1595, 1595, 1595, 1780, - /* 140 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1780, 1595, 1595, - /* 150 */ 1595, 1595, 1595, 1595, 1595, 1595, 1889, 1595, 1595, 2080, - /* 160 */ 2133, 1595, 1595, 2083, 1595, 1595, 1595, 1855, 1595, 1733, - /* 170 */ 2070, 2047, 2061, 2117, 2048, 2045, 2064, 1595, 2074, 1595, - /* 180 */ 1882, 1848, 1595, 1848, 1845, 1595, 1595, 1848, 1845, 1845, - /* 190 */ 1724, 1595, 1595, 1595, 1595, 1595, 1595, 1673, 1595, 1673, - /* 200 */ 1595, 1595, 1673, 1595, 1673, 1673, 1673, 1595, 1673, 1652, - /* 210 */ 1652, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 220 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1902, 1595, 1671, - /* 230 */ 1891, 1595, 1671, 1595, 1595, 1595, 1595, 2090, 2088, 1595, - /* 240 */ 2090, 2088, 1595, 1595, 1595, 2102, 2098, 2090, 2106, 2104, - /* 250 */ 2076, 2074, 2136, 2123, 2119, 2061, 1595, 1595, 1595, 1595, - /* 260 */ 1671, 1595, 2088, 1595, 1595, 1595, 1595, 1595, 2088, 1595, - /* 270 */ 1595, 1671, 1595, 1671, 1595, 1595, 1749, 1595, 1595, 1595, - /* 280 */ 1671, 1627, 1595, 1884, 1895, 1867, 1867, 1783, 1783, 1783, - /* 290 */ 1674, 1600, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 300 */ 1595, 1595, 1595, 1595, 2101, 2100, 1971, 1595, 2019, 2018, - /* 310 */ 2017, 2008, 1970, 1745, 1595, 1969, 1968, 1595, 1595, 1595, - /* 320 */ 1595, 1595, 1863, 1862, 1962, 1595, 1595, 1963, 1961, 1960, - /* 330 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 340 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 2120, 2124, 1595, - /* 350 */ 1595, 1595, 1595, 1595, 1595, 2044, 1595, 1595, 1595, 1595, - /* 360 */ 1595, 1944, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 370 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 380 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 390 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 400 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 410 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 420 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 430 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 440 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 450 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 460 */ 1632, 1949, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 470 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 480 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 490 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 500 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1712, 1711, - /* 510 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 520 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 530 */ 1595, 1595, 1953, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 540 */ 1595, 1595, 1595, 1595, 1595, 1595, 2116, 2077, 1595, 1595, - /* 550 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 560 */ 1595, 1944, 1595, 2099, 1595, 1595, 2114, 1595, 2118, 1595, - /* 570 */ 1595, 1595, 1595, 1595, 1595, 1595, 2054, 2050, 1595, 1595, - /* 580 */ 2046, 1595, 1595, 1595, 1595, 1952, 1595, 1595, 1595, 1595, - /* 590 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1943, - /* 600 */ 1595, 2005, 1595, 1595, 1595, 2039, 1595, 1595, 1990, 1595, - /* 610 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1953, 1595, - /* 620 */ 1956, 1595, 1595, 1595, 1595, 1595, 1777, 1595, 1595, 1595, - /* 630 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1762, - /* 640 */ 1760, 1759, 1758, 1595, 1755, 1595, 1790, 1595, 1595, 1595, - /* 650 */ 1786, 1785, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 660 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1692, 1595, 1595, - /* 670 */ 1595, 1595, 1595, 1595, 1595, 1595, 1684, 1595, 1683, 1595, - /* 680 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 690 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 700 */ 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, 1595, - /* 710 */ 1595, + /* 0 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 10 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 20 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 30 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 40 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 50 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 60 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 70 */ 1634, 1891, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 80 */ 1634, 1634, 1634, 1634, 1634, 1712, 1634, 1634, 1634, 1634, + /* 90 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 100 */ 1634, 1634, 1634, 1634, 1634, 1710, 1884, 2093, 1634, 1634, + /* 110 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 120 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 130 */ 1634, 2105, 1634, 1634, 1712, 1634, 2105, 2105, 2105, 1710, + /* 140 */ 2065, 2065, 1634, 1634, 1634, 1634, 1819, 1634, 1634, 1634, + /* 150 */ 1634, 1634, 1634, 1819, 1634, 1634, 1634, 1634, 1634, 1634, + /* 160 */ 1634, 1634, 1936, 1634, 1634, 2130, 2184, 1634, 1634, 2133, + /* 170 */ 1634, 1634, 1634, 1896, 1634, 1772, 2120, 2097, 2111, 2168, + /* 180 */ 2098, 2095, 2114, 1634, 2124, 1634, 1929, 1889, 1634, 1889, + /* 190 */ 1886, 1634, 1634, 1889, 1886, 1886, 1763, 1634, 1634, 1634, + /* 200 */ 1634, 1634, 1634, 1712, 1634, 1712, 1634, 1634, 1712, 1634, + /* 210 */ 1712, 1712, 1712, 1634, 1712, 1691, 1691, 1634, 1634, 1634, + /* 220 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 230 */ 1634, 1634, 1634, 1951, 1942, 1634, 1710, 1938, 1634, 1710, + /* 240 */ 1634, 1634, 1634, 1634, 2141, 2139, 1634, 2141, 2139, 1634, + /* 250 */ 1634, 1634, 2153, 2149, 2141, 2157, 2155, 2126, 2124, 2187, + /* 260 */ 2174, 2170, 2111, 1634, 1634, 1634, 1634, 1710, 1710, 1634, + /* 270 */ 2139, 1634, 1634, 1634, 1634, 1634, 2139, 1634, 1634, 1710, + /* 280 */ 1634, 1710, 1634, 1634, 1788, 1634, 1634, 1634, 1710, 1666, + /* 290 */ 1634, 1931, 1944, 1914, 1914, 1822, 1822, 1822, 1713, 1639, + /* 300 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 310 */ 1634, 1634, 2152, 2151, 2020, 1634, 2069, 2068, 2067, 2058, + /* 320 */ 2019, 1784, 1634, 2018, 2017, 1634, 1634, 1634, 1634, 1634, + /* 330 */ 1634, 1905, 1904, 2011, 1634, 1634, 2012, 2010, 2009, 1634, + /* 340 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 350 */ 1634, 1634, 1634, 1634, 1634, 1634, 2171, 2175, 1634, 1634, + /* 360 */ 1634, 1634, 1634, 1634, 1634, 2094, 1634, 1634, 1634, 1634, + /* 370 */ 1634, 1993, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 380 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 390 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 400 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 410 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 420 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 430 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 440 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 450 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 460 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 470 */ 1634, 1634, 1671, 1998, 1634, 1634, 1634, 1634, 1634, 1634, + /* 480 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 490 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 500 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 510 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 520 */ 1751, 1750, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 530 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 540 */ 1634, 1634, 1634, 1634, 2002, 1634, 1634, 1634, 1634, 1634, + /* 550 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 2167, 2127, + /* 560 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 570 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1993, 1634, 2150, + /* 580 */ 1634, 1634, 2165, 1634, 2169, 1634, 1634, 1634, 1634, 1634, + /* 590 */ 1634, 1634, 2104, 2100, 1634, 1634, 2096, 1634, 1634, 1634, + /* 600 */ 1634, 1634, 1634, 1634, 2001, 1634, 1634, 1634, 1634, 1634, + /* 610 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1992, 1634, + /* 620 */ 2055, 1634, 1634, 1634, 2089, 1634, 1634, 2040, 1634, 1634, + /* 630 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 2002, 1634, 2005, + /* 640 */ 1634, 1634, 1634, 1634, 1634, 1816, 1634, 1634, 1634, 1634, + /* 650 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1801, 1799, + /* 660 */ 1798, 1797, 1634, 1794, 1634, 1829, 1634, 1634, 1634, 1825, + /* 670 */ 1824, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 680 */ 1634, 1634, 1634, 1634, 1634, 1634, 1731, 1634, 1634, 1634, + /* 690 */ 1634, 1634, 1634, 1634, 1634, 1723, 1634, 1722, 1634, 1634, + /* 700 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 710 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 720 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1131,6 +1129,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* CONSUMERS => nothing */ 0, /* SUBSCRIPTIONS => nothing */ 0, /* VNODES => nothing */ + 0, /* ALIVE => nothing */ 0, /* LIKE => nothing */ 0, /* TBNAME => nothing */ 0, /* QTAGS => nothing */ @@ -1138,6 +1137,8 @@ static const YYCODETYPE yyFallback[] = { 0, /* INDEX => nothing */ 0, /* FUNCTION => nothing */ 0, /* INTERVAL => nothing */ + 0, /* COUNT => nothing */ + 0, /* LAST_ROW => nothing */ 0, /* TOPIC => nothing */ 0, /* WITH => nothing */ 0, /* META => nothing */ @@ -1187,6 +1188,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* WEND => nothing */ 0, /* WDURATION => nothing */ 0, /* IROWTS => nothing */ + 0, /* ISFILLED => nothing */ 0, /* CAST => nothing */ 0, /* NOW => nothing */ 0, /* TODAY => nothing */ @@ -1195,10 +1197,8 @@ static const YYCODETYPE yyFallback[] = { 0, /* SERVER_VERSION => nothing */ 0, /* SERVER_STATUS => nothing */ 0, /* CURRENT_USER => nothing */ - 0, /* COUNT => nothing */ - 0, /* LAST_ROW => nothing */ 0, /* CASE => nothing */ - 268, /* END => ABORT */ + 272, /* END => ABORT */ 0, /* WHEN => nothing */ 0, /* THEN => nothing */ 0, /* ELSE => nothing */ @@ -1222,6 +1222,8 @@ static const YYCODETYPE yyFallback[] = { 0, /* BY => nothing */ 0, /* SESSION => nothing */ 0, /* STATE_WINDOW => nothing */ + 0, /* EVENT_WINDOW => nothing */ + 0, /* START => nothing */ 0, /* SLIDING => nothing */ 0, /* FILL => nothing */ 0, /* VALUE => nothing */ @@ -1240,59 +1242,59 @@ static const YYCODETYPE yyFallback[] = { 0, /* ASC => nothing */ 0, /* NULLS => nothing */ 0, /* ABORT => nothing */ - 268, /* AFTER => ABORT */ - 268, /* ATTACH => ABORT */ - 268, /* BEFORE => ABORT */ - 268, /* BEGIN => ABORT */ - 268, /* BITAND => ABORT */ - 268, /* BITNOT => ABORT */ - 268, /* BITOR => ABORT */ - 268, /* BLOCKS => ABORT */ - 268, /* CHANGE => ABORT */ - 268, /* COMMA => ABORT */ - 268, /* COMPACT => ABORT */ - 268, /* CONCAT => ABORT */ - 268, /* CONFLICT => ABORT */ - 268, /* COPY => ABORT */ - 268, /* DEFERRED => ABORT */ - 268, /* DELIMITERS => ABORT */ - 268, /* DETACH => ABORT */ - 268, /* DIVIDE => ABORT */ - 268, /* DOT => ABORT */ - 268, /* EACH => ABORT */ - 268, /* FAIL => ABORT */ - 268, /* FILE => ABORT */ - 268, /* FOR => ABORT */ - 268, /* GLOB => ABORT */ - 268, /* ID => ABORT */ - 268, /* IMMEDIATE => ABORT */ - 268, /* IMPORT => ABORT */ - 268, /* INITIALLY => ABORT */ - 268, /* INSTEAD => ABORT */ - 268, /* ISNULL => ABORT */ - 268, /* KEY => ABORT */ - 268, /* MODULES => ABORT */ - 268, /* NK_BITNOT => ABORT */ - 268, /* NK_SEMI => ABORT */ - 268, /* NOTNULL => ABORT */ - 268, /* OF => ABORT */ - 268, /* PLUS => ABORT */ - 268, /* PRIVILEGE => ABORT */ - 268, /* RAISE => ABORT */ - 268, /* REPLACE => ABORT */ - 268, /* RESTRICT => ABORT */ - 268, /* ROW => ABORT */ - 268, /* SEMI => ABORT */ - 268, /* STAR => ABORT */ - 268, /* STATEMENT => ABORT */ - 268, /* STRICT => ABORT */ - 268, /* STRING => ABORT */ - 268, /* TIMES => ABORT */ - 268, /* UPDATE => ABORT */ - 268, /* VALUES => ABORT */ - 268, /* VARIABLE => ABORT */ - 268, /* VIEW => ABORT */ - 268, /* WAL => ABORT */ + 272, /* AFTER => ABORT */ + 272, /* ATTACH => ABORT */ + 272, /* BEFORE => ABORT */ + 272, /* BEGIN => ABORT */ + 272, /* BITAND => ABORT */ + 272, /* BITNOT => ABORT */ + 272, /* BITOR => ABORT */ + 272, /* BLOCKS => ABORT */ + 272, /* CHANGE => ABORT */ + 272, /* COMMA => ABORT */ + 272, /* COMPACT => ABORT */ + 272, /* CONCAT => ABORT */ + 272, /* CONFLICT => ABORT */ + 272, /* COPY => ABORT */ + 272, /* DEFERRED => ABORT */ + 272, /* DELIMITERS => ABORT */ + 272, /* DETACH => ABORT */ + 272, /* DIVIDE => ABORT */ + 272, /* DOT => ABORT */ + 272, /* EACH => ABORT */ + 272, /* FAIL => ABORT */ + 272, /* FILE => ABORT */ + 272, /* FOR => ABORT */ + 272, /* GLOB => ABORT */ + 272, /* ID => ABORT */ + 272, /* IMMEDIATE => ABORT */ + 272, /* IMPORT => ABORT */ + 272, /* INITIALLY => ABORT */ + 272, /* INSTEAD => ABORT */ + 272, /* ISNULL => ABORT */ + 272, /* KEY => ABORT */ + 272, /* MODULES => ABORT */ + 272, /* NK_BITNOT => ABORT */ + 272, /* NK_SEMI => ABORT */ + 272, /* NOTNULL => ABORT */ + 272, /* OF => ABORT */ + 272, /* PLUS => ABORT */ + 272, /* PRIVILEGE => ABORT */ + 272, /* RAISE => ABORT */ + 272, /* REPLACE => ABORT */ + 272, /* RESTRICT => ABORT */ + 272, /* ROW => ABORT */ + 272, /* SEMI => ABORT */ + 272, /* STAR => ABORT */ + 272, /* STATEMENT => ABORT */ + 272, /* STRICT => ABORT */ + 272, /* STRING => ABORT */ + 272, /* TIMES => ABORT */ + 272, /* UPDATE => ABORT */ + 272, /* VALUES => ABORT */ + 272, /* VARIABLE => ABORT */ + 272, /* VIEW => ABORT */ + 272, /* WAL => ABORT */ }; #endif /* YYFALLBACK */ @@ -1540,305 +1542,311 @@ static const char *const yyTokenName[] = { /* 157 */ "CONSUMERS", /* 158 */ "SUBSCRIPTIONS", /* 159 */ "VNODES", - /* 160 */ "LIKE", - /* 161 */ "TBNAME", - /* 162 */ "QTAGS", - /* 163 */ "AS", - /* 164 */ "INDEX", - /* 165 */ "FUNCTION", - /* 166 */ "INTERVAL", - /* 167 */ "TOPIC", - /* 168 */ "WITH", - /* 169 */ "META", - /* 170 */ "CONSUMER", - /* 171 */ "GROUP", - /* 172 */ "DESC", - /* 173 */ "DESCRIBE", - /* 174 */ "RESET", - /* 175 */ "QUERY", - /* 176 */ "CACHE", - /* 177 */ "EXPLAIN", - /* 178 */ "ANALYZE", - /* 179 */ "VERBOSE", - /* 180 */ "NK_BOOL", - /* 181 */ "RATIO", - /* 182 */ "NK_FLOAT", - /* 183 */ "OUTPUTTYPE", - /* 184 */ "AGGREGATE", - /* 185 */ "BUFSIZE", - /* 186 */ "STREAM", - /* 187 */ "INTO", - /* 188 */ "TRIGGER", - /* 189 */ "AT_ONCE", - /* 190 */ "WINDOW_CLOSE", - /* 191 */ "IGNORE", - /* 192 */ "EXPIRED", - /* 193 */ "FILL_HISTORY", - /* 194 */ "SUBTABLE", - /* 195 */ "KILL", - /* 196 */ "CONNECTION", - /* 197 */ "TRANSACTION", - /* 198 */ "BALANCE", - /* 199 */ "VGROUP", - /* 200 */ "MERGE", - /* 201 */ "REDISTRIBUTE", - /* 202 */ "SPLIT", - /* 203 */ "DELETE", - /* 204 */ "INSERT", - /* 205 */ "NULL", - /* 206 */ "NK_QUESTION", - /* 207 */ "NK_ARROW", - /* 208 */ "ROWTS", - /* 209 */ "QSTART", - /* 210 */ "QEND", - /* 211 */ "QDURATION", - /* 212 */ "WSTART", - /* 213 */ "WEND", - /* 214 */ "WDURATION", - /* 215 */ "IROWTS", - /* 216 */ "CAST", - /* 217 */ "NOW", - /* 218 */ "TODAY", - /* 219 */ "TIMEZONE", - /* 220 */ "CLIENT_VERSION", - /* 221 */ "SERVER_VERSION", - /* 222 */ "SERVER_STATUS", - /* 223 */ "CURRENT_USER", - /* 224 */ "COUNT", - /* 225 */ "LAST_ROW", - /* 226 */ "CASE", - /* 227 */ "END", - /* 228 */ "WHEN", - /* 229 */ "THEN", - /* 230 */ "ELSE", - /* 231 */ "BETWEEN", - /* 232 */ "IS", - /* 233 */ "NK_LT", - /* 234 */ "NK_GT", - /* 235 */ "NK_LE", - /* 236 */ "NK_GE", - /* 237 */ "NK_NE", - /* 238 */ "MATCH", - /* 239 */ "NMATCH", - /* 240 */ "CONTAINS", - /* 241 */ "IN", - /* 242 */ "JOIN", - /* 243 */ "INNER", - /* 244 */ "SELECT", - /* 245 */ "DISTINCT", - /* 246 */ "WHERE", - /* 247 */ "PARTITION", - /* 248 */ "BY", - /* 249 */ "SESSION", - /* 250 */ "STATE_WINDOW", - /* 251 */ "SLIDING", - /* 252 */ "FILL", - /* 253 */ "VALUE", - /* 254 */ "NONE", - /* 255 */ "PREV", - /* 256 */ "LINEAR", - /* 257 */ "NEXT", - /* 258 */ "HAVING", - /* 259 */ "RANGE", - /* 260 */ "EVERY", - /* 261 */ "ORDER", - /* 262 */ "SLIMIT", - /* 263 */ "SOFFSET", - /* 264 */ "LIMIT", - /* 265 */ "OFFSET", - /* 266 */ "ASC", - /* 267 */ "NULLS", - /* 268 */ "ABORT", - /* 269 */ "AFTER", - /* 270 */ "ATTACH", - /* 271 */ "BEFORE", - /* 272 */ "BEGIN", - /* 273 */ "BITAND", - /* 274 */ "BITNOT", - /* 275 */ "BITOR", - /* 276 */ "BLOCKS", - /* 277 */ "CHANGE", - /* 278 */ "COMMA", - /* 279 */ "COMPACT", - /* 280 */ "CONCAT", - /* 281 */ "CONFLICT", - /* 282 */ "COPY", - /* 283 */ "DEFERRED", - /* 284 */ "DELIMITERS", - /* 285 */ "DETACH", - /* 286 */ "DIVIDE", - /* 287 */ "DOT", - /* 288 */ "EACH", - /* 289 */ "FAIL", - /* 290 */ "FILE", - /* 291 */ "FOR", - /* 292 */ "GLOB", - /* 293 */ "ID", - /* 294 */ "IMMEDIATE", - /* 295 */ "IMPORT", - /* 296 */ "INITIALLY", - /* 297 */ "INSTEAD", - /* 298 */ "ISNULL", - /* 299 */ "KEY", - /* 300 */ "MODULES", - /* 301 */ "NK_BITNOT", - /* 302 */ "NK_SEMI", - /* 303 */ "NOTNULL", - /* 304 */ "OF", - /* 305 */ "PLUS", - /* 306 */ "PRIVILEGE", - /* 307 */ "RAISE", - /* 308 */ "REPLACE", - /* 309 */ "RESTRICT", - /* 310 */ "ROW", - /* 311 */ "SEMI", - /* 312 */ "STAR", - /* 313 */ "STATEMENT", - /* 314 */ "STRICT", - /* 315 */ "STRING", - /* 316 */ "TIMES", - /* 317 */ "UPDATE", - /* 318 */ "VALUES", - /* 319 */ "VARIABLE", - /* 320 */ "VIEW", - /* 321 */ "WAL", - /* 322 */ "cmd", - /* 323 */ "account_options", - /* 324 */ "alter_account_options", - /* 325 */ "literal", - /* 326 */ "alter_account_option", - /* 327 */ "user_name", - /* 328 */ "sysinfo_opt", - /* 329 */ "privileges", - /* 330 */ "priv_level", - /* 331 */ "priv_type_list", - /* 332 */ "priv_type", - /* 333 */ "db_name", - /* 334 */ "topic_name", - /* 335 */ "dnode_endpoint", - /* 336 */ "force_opt", - /* 337 */ "not_exists_opt", - /* 338 */ "db_options", - /* 339 */ "exists_opt", - /* 340 */ "alter_db_options", - /* 341 */ "speed_opt", - /* 342 */ "integer_list", - /* 343 */ "variable_list", - /* 344 */ "retention_list", - /* 345 */ "alter_db_option", - /* 346 */ "retention", - /* 347 */ "full_table_name", - /* 348 */ "column_def_list", - /* 349 */ "tags_def_opt", - /* 350 */ "table_options", - /* 351 */ "multi_create_clause", - /* 352 */ "tags_def", - /* 353 */ "multi_drop_clause", - /* 354 */ "alter_table_clause", - /* 355 */ "alter_table_options", - /* 356 */ "column_name", - /* 357 */ "type_name", - /* 358 */ "signed_literal", - /* 359 */ "create_subtable_clause", - /* 360 */ "specific_cols_opt", - /* 361 */ "expression_list", - /* 362 */ "drop_table_clause", - /* 363 */ "col_name_list", - /* 364 */ "table_name", - /* 365 */ "column_def", - /* 366 */ "duration_list", - /* 367 */ "rollup_func_list", - /* 368 */ "alter_table_option", - /* 369 */ "duration_literal", - /* 370 */ "rollup_func_name", - /* 371 */ "function_name", - /* 372 */ "col_name", - /* 373 */ "db_name_cond_opt", - /* 374 */ "like_pattern_opt", - /* 375 */ "table_name_cond", - /* 376 */ "from_db_opt", - /* 377 */ "tag_list_opt", - /* 378 */ "tag_item", - /* 379 */ "column_alias", - /* 380 */ "full_index_name", - /* 381 */ "index_options", - /* 382 */ "index_name", - /* 383 */ "func_list", - /* 384 */ "sliding_opt", - /* 385 */ "sma_stream_opt", - /* 386 */ "func", - /* 387 */ "query_or_subquery", - /* 388 */ "cgroup_name", - /* 389 */ "analyze_opt", - /* 390 */ "explain_options", - /* 391 */ "agg_func_opt", - /* 392 */ "bufsize_opt", - /* 393 */ "stream_name", - /* 394 */ "stream_options", - /* 395 */ "subtable_opt", - /* 396 */ "expression", - /* 397 */ "dnode_list", - /* 398 */ "where_clause_opt", - /* 399 */ "signed", - /* 400 */ "literal_func", - /* 401 */ "literal_list", - /* 402 */ "table_alias", - /* 403 */ "expr_or_subquery", - /* 404 */ "pseudo_column", - /* 405 */ "column_reference", - /* 406 */ "function_expression", - /* 407 */ "case_when_expression", - /* 408 */ "star_func", - /* 409 */ "star_func_para_list", - /* 410 */ "noarg_func", - /* 411 */ "other_para_list", - /* 412 */ "star_func_para", - /* 413 */ "when_then_list", - /* 414 */ "case_when_else_opt", - /* 415 */ "common_expression", - /* 416 */ "when_then_expr", - /* 417 */ "predicate", - /* 418 */ "compare_op", - /* 419 */ "in_op", - /* 420 */ "in_predicate_value", - /* 421 */ "boolean_value_expression", - /* 422 */ "boolean_primary", - /* 423 */ "from_clause_opt", - /* 424 */ "table_reference_list", - /* 425 */ "table_reference", - /* 426 */ "table_primary", - /* 427 */ "joined_table", - /* 428 */ "alias_opt", - /* 429 */ "subquery", - /* 430 */ "parenthesized_joined_table", - /* 431 */ "join_type", - /* 432 */ "search_condition", - /* 433 */ "query_specification", - /* 434 */ "set_quantifier_opt", - /* 435 */ "select_list", - /* 436 */ "partition_by_clause_opt", - /* 437 */ "range_opt", - /* 438 */ "every_opt", - /* 439 */ "fill_opt", - /* 440 */ "twindow_clause_opt", - /* 441 */ "group_by_clause_opt", - /* 442 */ "having_clause_opt", - /* 443 */ "select_item", - /* 444 */ "partition_list", - /* 445 */ "partition_item", - /* 446 */ "fill_mode", - /* 447 */ "group_by_list", - /* 448 */ "query_expression", - /* 449 */ "query_simple", - /* 450 */ "order_by_clause_opt", - /* 451 */ "slimit_clause_opt", - /* 452 */ "limit_clause_opt", - /* 453 */ "union_query_expression", - /* 454 */ "query_simple_or_subquery", - /* 455 */ "sort_specification_list", - /* 456 */ "sort_specification", - /* 457 */ "ordering_specification_opt", - /* 458 */ "null_ordering_opt", + /* 160 */ "ALIVE", + /* 161 */ "LIKE", + /* 162 */ "TBNAME", + /* 163 */ "QTAGS", + /* 164 */ "AS", + /* 165 */ "INDEX", + /* 166 */ "FUNCTION", + /* 167 */ "INTERVAL", + /* 168 */ "COUNT", + /* 169 */ "LAST_ROW", + /* 170 */ "TOPIC", + /* 171 */ "WITH", + /* 172 */ "META", + /* 173 */ "CONSUMER", + /* 174 */ "GROUP", + /* 175 */ "DESC", + /* 176 */ "DESCRIBE", + /* 177 */ "RESET", + /* 178 */ "QUERY", + /* 179 */ "CACHE", + /* 180 */ "EXPLAIN", + /* 181 */ "ANALYZE", + /* 182 */ "VERBOSE", + /* 183 */ "NK_BOOL", + /* 184 */ "RATIO", + /* 185 */ "NK_FLOAT", + /* 186 */ "OUTPUTTYPE", + /* 187 */ "AGGREGATE", + /* 188 */ "BUFSIZE", + /* 189 */ "STREAM", + /* 190 */ "INTO", + /* 191 */ "TRIGGER", + /* 192 */ "AT_ONCE", + /* 193 */ "WINDOW_CLOSE", + /* 194 */ "IGNORE", + /* 195 */ "EXPIRED", + /* 196 */ "FILL_HISTORY", + /* 197 */ "SUBTABLE", + /* 198 */ "KILL", + /* 199 */ "CONNECTION", + /* 200 */ "TRANSACTION", + /* 201 */ "BALANCE", + /* 202 */ "VGROUP", + /* 203 */ "MERGE", + /* 204 */ "REDISTRIBUTE", + /* 205 */ "SPLIT", + /* 206 */ "DELETE", + /* 207 */ "INSERT", + /* 208 */ "NULL", + /* 209 */ "NK_QUESTION", + /* 210 */ "NK_ARROW", + /* 211 */ "ROWTS", + /* 212 */ "QSTART", + /* 213 */ "QEND", + /* 214 */ "QDURATION", + /* 215 */ "WSTART", + /* 216 */ "WEND", + /* 217 */ "WDURATION", + /* 218 */ "IROWTS", + /* 219 */ "ISFILLED", + /* 220 */ "CAST", + /* 221 */ "NOW", + /* 222 */ "TODAY", + /* 223 */ "TIMEZONE", + /* 224 */ "CLIENT_VERSION", + /* 225 */ "SERVER_VERSION", + /* 226 */ "SERVER_STATUS", + /* 227 */ "CURRENT_USER", + /* 228 */ "CASE", + /* 229 */ "END", + /* 230 */ "WHEN", + /* 231 */ "THEN", + /* 232 */ "ELSE", + /* 233 */ "BETWEEN", + /* 234 */ "IS", + /* 235 */ "NK_LT", + /* 236 */ "NK_GT", + /* 237 */ "NK_LE", + /* 238 */ "NK_GE", + /* 239 */ "NK_NE", + /* 240 */ "MATCH", + /* 241 */ "NMATCH", + /* 242 */ "CONTAINS", + /* 243 */ "IN", + /* 244 */ "JOIN", + /* 245 */ "INNER", + /* 246 */ "SELECT", + /* 247 */ "DISTINCT", + /* 248 */ "WHERE", + /* 249 */ "PARTITION", + /* 250 */ "BY", + /* 251 */ "SESSION", + /* 252 */ "STATE_WINDOW", + /* 253 */ "EVENT_WINDOW", + /* 254 */ "START", + /* 255 */ "SLIDING", + /* 256 */ "FILL", + /* 257 */ "VALUE", + /* 258 */ "NONE", + /* 259 */ "PREV", + /* 260 */ "LINEAR", + /* 261 */ "NEXT", + /* 262 */ "HAVING", + /* 263 */ "RANGE", + /* 264 */ "EVERY", + /* 265 */ "ORDER", + /* 266 */ "SLIMIT", + /* 267 */ "SOFFSET", + /* 268 */ "LIMIT", + /* 269 */ "OFFSET", + /* 270 */ "ASC", + /* 271 */ "NULLS", + /* 272 */ "ABORT", + /* 273 */ "AFTER", + /* 274 */ "ATTACH", + /* 275 */ "BEFORE", + /* 276 */ "BEGIN", + /* 277 */ "BITAND", + /* 278 */ "BITNOT", + /* 279 */ "BITOR", + /* 280 */ "BLOCKS", + /* 281 */ "CHANGE", + /* 282 */ "COMMA", + /* 283 */ "COMPACT", + /* 284 */ "CONCAT", + /* 285 */ "CONFLICT", + /* 286 */ "COPY", + /* 287 */ "DEFERRED", + /* 288 */ "DELIMITERS", + /* 289 */ "DETACH", + /* 290 */ "DIVIDE", + /* 291 */ "DOT", + /* 292 */ "EACH", + /* 293 */ "FAIL", + /* 294 */ "FILE", + /* 295 */ "FOR", + /* 296 */ "GLOB", + /* 297 */ "ID", + /* 298 */ "IMMEDIATE", + /* 299 */ "IMPORT", + /* 300 */ "INITIALLY", + /* 301 */ "INSTEAD", + /* 302 */ "ISNULL", + /* 303 */ "KEY", + /* 304 */ "MODULES", + /* 305 */ "NK_BITNOT", + /* 306 */ "NK_SEMI", + /* 307 */ "NOTNULL", + /* 308 */ "OF", + /* 309 */ "PLUS", + /* 310 */ "PRIVILEGE", + /* 311 */ "RAISE", + /* 312 */ "REPLACE", + /* 313 */ "RESTRICT", + /* 314 */ "ROW", + /* 315 */ "SEMI", + /* 316 */ "STAR", + /* 317 */ "STATEMENT", + /* 318 */ "STRICT", + /* 319 */ "STRING", + /* 320 */ "TIMES", + /* 321 */ "UPDATE", + /* 322 */ "VALUES", + /* 323 */ "VARIABLE", + /* 324 */ "VIEW", + /* 325 */ "WAL", + /* 326 */ "cmd", + /* 327 */ "account_options", + /* 328 */ "alter_account_options", + /* 329 */ "literal", + /* 330 */ "alter_account_option", + /* 331 */ "user_name", + /* 332 */ "sysinfo_opt", + /* 333 */ "privileges", + /* 334 */ "priv_level", + /* 335 */ "priv_type_list", + /* 336 */ "priv_type", + /* 337 */ "db_name", + /* 338 */ "topic_name", + /* 339 */ "dnode_endpoint", + /* 340 */ "force_opt", + /* 341 */ "not_exists_opt", + /* 342 */ "db_options", + /* 343 */ "exists_opt", + /* 344 */ "alter_db_options", + /* 345 */ "speed_opt", + /* 346 */ "integer_list", + /* 347 */ "variable_list", + /* 348 */ "retention_list", + /* 349 */ "alter_db_option", + /* 350 */ "retention", + /* 351 */ "full_table_name", + /* 352 */ "column_def_list", + /* 353 */ "tags_def_opt", + /* 354 */ "table_options", + /* 355 */ "multi_create_clause", + /* 356 */ "tags_def", + /* 357 */ "multi_drop_clause", + /* 358 */ "alter_table_clause", + /* 359 */ "alter_table_options", + /* 360 */ "column_name", + /* 361 */ "type_name", + /* 362 */ "signed_literal", + /* 363 */ "create_subtable_clause", + /* 364 */ "specific_cols_opt", + /* 365 */ "expression_list", + /* 366 */ "drop_table_clause", + /* 367 */ "col_name_list", + /* 368 */ "table_name", + /* 369 */ "column_def", + /* 370 */ "duration_list", + /* 371 */ "rollup_func_list", + /* 372 */ "alter_table_option", + /* 373 */ "duration_literal", + /* 374 */ "rollup_func_name", + /* 375 */ "function_name", + /* 376 */ "col_name", + /* 377 */ "db_name_cond_opt", + /* 378 */ "like_pattern_opt", + /* 379 */ "table_name_cond", + /* 380 */ "from_db_opt", + /* 381 */ "tag_list_opt", + /* 382 */ "tag_item", + /* 383 */ "column_alias", + /* 384 */ "full_index_name", + /* 385 */ "index_options", + /* 386 */ "index_name", + /* 387 */ "func_list", + /* 388 */ "sliding_opt", + /* 389 */ "sma_stream_opt", + /* 390 */ "func", + /* 391 */ "sma_func_name", + /* 392 */ "query_or_subquery", + /* 393 */ "cgroup_name", + /* 394 */ "analyze_opt", + /* 395 */ "explain_options", + /* 396 */ "agg_func_opt", + /* 397 */ "bufsize_opt", + /* 398 */ "stream_name", + /* 399 */ "stream_options", + /* 400 */ "col_list_opt", + /* 401 */ "subtable_opt", + /* 402 */ "expression", + /* 403 */ "dnode_list", + /* 404 */ "where_clause_opt", + /* 405 */ "signed", + /* 406 */ "literal_func", + /* 407 */ "literal_list", + /* 408 */ "table_alias", + /* 409 */ "expr_or_subquery", + /* 410 */ "pseudo_column", + /* 411 */ "column_reference", + /* 412 */ "function_expression", + /* 413 */ "case_when_expression", + /* 414 */ "star_func", + /* 415 */ "star_func_para_list", + /* 416 */ "noarg_func", + /* 417 */ "other_para_list", + /* 418 */ "star_func_para", + /* 419 */ "when_then_list", + /* 420 */ "case_when_else_opt", + /* 421 */ "common_expression", + /* 422 */ "when_then_expr", + /* 423 */ "predicate", + /* 424 */ "compare_op", + /* 425 */ "in_op", + /* 426 */ "in_predicate_value", + /* 427 */ "boolean_value_expression", + /* 428 */ "boolean_primary", + /* 429 */ "from_clause_opt", + /* 430 */ "table_reference_list", + /* 431 */ "table_reference", + /* 432 */ "table_primary", + /* 433 */ "joined_table", + /* 434 */ "alias_opt", + /* 435 */ "subquery", + /* 436 */ "parenthesized_joined_table", + /* 437 */ "join_type", + /* 438 */ "search_condition", + /* 439 */ "query_specification", + /* 440 */ "set_quantifier_opt", + /* 441 */ "select_list", + /* 442 */ "partition_by_clause_opt", + /* 443 */ "range_opt", + /* 444 */ "every_opt", + /* 445 */ "fill_opt", + /* 446 */ "twindow_clause_opt", + /* 447 */ "group_by_clause_opt", + /* 448 */ "having_clause_opt", + /* 449 */ "select_item", + /* 450 */ "partition_list", + /* 451 */ "partition_item", + /* 452 */ "fill_mode", + /* 453 */ "group_by_list", + /* 454 */ "query_expression", + /* 455 */ "query_simple", + /* 456 */ "order_by_clause_opt", + /* 457 */ "slimit_clause_opt", + /* 458 */ "limit_clause_opt", + /* 459 */ "union_query_expression", + /* 460 */ "query_simple_or_subquery", + /* 461 */ "sort_specification_list", + /* 462 */ "sort_specification", + /* 463 */ "ordering_specification_opt", + /* 464 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -2091,302 +2099,314 @@ static const char *const yyRuleName[] = { /* 242 */ "cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt", /* 243 */ "cmd ::= SHOW VNODES NK_INTEGER", /* 244 */ "cmd ::= SHOW VNODES NK_STRING", - /* 245 */ "db_name_cond_opt ::=", - /* 246 */ "db_name_cond_opt ::= db_name NK_DOT", - /* 247 */ "like_pattern_opt ::=", - /* 248 */ "like_pattern_opt ::= LIKE NK_STRING", - /* 249 */ "table_name_cond ::= table_name", - /* 250 */ "from_db_opt ::=", - /* 251 */ "from_db_opt ::= FROM db_name", - /* 252 */ "tag_list_opt ::=", - /* 253 */ "tag_list_opt ::= tag_item", - /* 254 */ "tag_list_opt ::= tag_list_opt NK_COMMA tag_item", - /* 255 */ "tag_item ::= TBNAME", - /* 256 */ "tag_item ::= QTAGS", - /* 257 */ "tag_item ::= column_name", - /* 258 */ "tag_item ::= column_name column_alias", - /* 259 */ "tag_item ::= column_name AS column_alias", - /* 260 */ "cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options", - /* 261 */ "cmd ::= DROP INDEX exists_opt full_index_name", - /* 262 */ "full_index_name ::= index_name", - /* 263 */ "full_index_name ::= db_name NK_DOT index_name", - /* 264 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", - /* 265 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", - /* 266 */ "func_list ::= func", - /* 267 */ "func_list ::= func_list NK_COMMA func", - /* 268 */ "func ::= function_name NK_LP expression_list NK_RP", - /* 269 */ "sma_stream_opt ::=", - /* 270 */ "sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal", - /* 271 */ "sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal", - /* 272 */ "sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal", - /* 273 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery", - /* 274 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", - /* 275 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", - /* 276 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name", - /* 277 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name", - /* 278 */ "cmd ::= DROP TOPIC exists_opt topic_name", - /* 279 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", - /* 280 */ "cmd ::= DESC full_table_name", - /* 281 */ "cmd ::= DESCRIBE full_table_name", - /* 282 */ "cmd ::= RESET QUERY CACHE", - /* 283 */ "cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery", - /* 284 */ "analyze_opt ::=", - /* 285 */ "analyze_opt ::= ANALYZE", - /* 286 */ "explain_options ::=", - /* 287 */ "explain_options ::= explain_options VERBOSE NK_BOOL", - /* 288 */ "explain_options ::= explain_options RATIO NK_FLOAT", - /* 289 */ "cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt", - /* 290 */ "cmd ::= DROP FUNCTION exists_opt function_name", - /* 291 */ "agg_func_opt ::=", - /* 292 */ "agg_func_opt ::= AGGREGATE", - /* 293 */ "bufsize_opt ::=", - /* 294 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", - /* 295 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name tags_def_opt subtable_opt AS query_or_subquery", - /* 296 */ "cmd ::= DROP STREAM exists_opt stream_name", - /* 297 */ "stream_options ::=", - /* 298 */ "stream_options ::= stream_options TRIGGER AT_ONCE", - /* 299 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", - /* 300 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", - /* 301 */ "stream_options ::= stream_options WATERMARK duration_literal", - /* 302 */ "stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER", - /* 303 */ "stream_options ::= stream_options FILL_HISTORY NK_INTEGER", - /* 304 */ "subtable_opt ::=", - /* 305 */ "subtable_opt ::= SUBTABLE NK_LP expression NK_RP", - /* 306 */ "cmd ::= KILL CONNECTION NK_INTEGER", - /* 307 */ "cmd ::= KILL QUERY NK_STRING", - /* 308 */ "cmd ::= KILL TRANSACTION NK_INTEGER", - /* 309 */ "cmd ::= BALANCE VGROUP", - /* 310 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", - /* 311 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", - /* 312 */ "cmd ::= SPLIT VGROUP NK_INTEGER", - /* 313 */ "dnode_list ::= DNODE NK_INTEGER", - /* 314 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", - /* 315 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", - /* 316 */ "cmd ::= query_or_subquery", - /* 317 */ "cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery", - /* 318 */ "cmd ::= INSERT INTO full_table_name query_or_subquery", - /* 319 */ "literal ::= NK_INTEGER", - /* 320 */ "literal ::= NK_FLOAT", - /* 321 */ "literal ::= NK_STRING", - /* 322 */ "literal ::= NK_BOOL", - /* 323 */ "literal ::= TIMESTAMP NK_STRING", - /* 324 */ "literal ::= duration_literal", - /* 325 */ "literal ::= NULL", - /* 326 */ "literal ::= NK_QUESTION", - /* 327 */ "duration_literal ::= NK_VARIABLE", - /* 328 */ "signed ::= NK_INTEGER", - /* 329 */ "signed ::= NK_PLUS NK_INTEGER", - /* 330 */ "signed ::= NK_MINUS NK_INTEGER", - /* 331 */ "signed ::= NK_FLOAT", - /* 332 */ "signed ::= NK_PLUS NK_FLOAT", - /* 333 */ "signed ::= NK_MINUS NK_FLOAT", - /* 334 */ "signed_literal ::= signed", - /* 335 */ "signed_literal ::= NK_STRING", - /* 336 */ "signed_literal ::= NK_BOOL", - /* 337 */ "signed_literal ::= TIMESTAMP NK_STRING", - /* 338 */ "signed_literal ::= duration_literal", - /* 339 */ "signed_literal ::= NULL", - /* 340 */ "signed_literal ::= literal_func", - /* 341 */ "signed_literal ::= NK_QUESTION", - /* 342 */ "literal_list ::= signed_literal", - /* 343 */ "literal_list ::= literal_list NK_COMMA signed_literal", - /* 344 */ "db_name ::= NK_ID", - /* 345 */ "table_name ::= NK_ID", - /* 346 */ "column_name ::= NK_ID", - /* 347 */ "function_name ::= NK_ID", - /* 348 */ "table_alias ::= NK_ID", - /* 349 */ "column_alias ::= NK_ID", - /* 350 */ "user_name ::= NK_ID", - /* 351 */ "topic_name ::= NK_ID", - /* 352 */ "stream_name ::= NK_ID", - /* 353 */ "cgroup_name ::= NK_ID", - /* 354 */ "index_name ::= NK_ID", - /* 355 */ "expr_or_subquery ::= expression", - /* 356 */ "expression ::= literal", - /* 357 */ "expression ::= pseudo_column", - /* 358 */ "expression ::= column_reference", - /* 359 */ "expression ::= function_expression", - /* 360 */ "expression ::= case_when_expression", - /* 361 */ "expression ::= NK_LP expression NK_RP", - /* 362 */ "expression ::= NK_PLUS expr_or_subquery", - /* 363 */ "expression ::= NK_MINUS expr_or_subquery", - /* 364 */ "expression ::= expr_or_subquery NK_PLUS expr_or_subquery", - /* 365 */ "expression ::= expr_or_subquery NK_MINUS expr_or_subquery", - /* 366 */ "expression ::= expr_or_subquery NK_STAR expr_or_subquery", - /* 367 */ "expression ::= expr_or_subquery NK_SLASH expr_or_subquery", - /* 368 */ "expression ::= expr_or_subquery NK_REM expr_or_subquery", - /* 369 */ "expression ::= column_reference NK_ARROW NK_STRING", - /* 370 */ "expression ::= expr_or_subquery NK_BITAND expr_or_subquery", - /* 371 */ "expression ::= expr_or_subquery NK_BITOR expr_or_subquery", - /* 372 */ "expression_list ::= expr_or_subquery", - /* 373 */ "expression_list ::= expression_list NK_COMMA expr_or_subquery", - /* 374 */ "column_reference ::= column_name", - /* 375 */ "column_reference ::= table_name NK_DOT column_name", - /* 376 */ "pseudo_column ::= ROWTS", - /* 377 */ "pseudo_column ::= TBNAME", - /* 378 */ "pseudo_column ::= table_name NK_DOT TBNAME", - /* 379 */ "pseudo_column ::= QSTART", - /* 380 */ "pseudo_column ::= QEND", - /* 381 */ "pseudo_column ::= QDURATION", - /* 382 */ "pseudo_column ::= WSTART", - /* 383 */ "pseudo_column ::= WEND", - /* 384 */ "pseudo_column ::= WDURATION", - /* 385 */ "pseudo_column ::= IROWTS", - /* 386 */ "pseudo_column ::= QTAGS", - /* 387 */ "function_expression ::= function_name NK_LP expression_list NK_RP", - /* 388 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", - /* 389 */ "function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP", - /* 390 */ "function_expression ::= literal_func", - /* 391 */ "literal_func ::= noarg_func NK_LP NK_RP", - /* 392 */ "literal_func ::= NOW", - /* 393 */ "noarg_func ::= NOW", - /* 394 */ "noarg_func ::= TODAY", - /* 395 */ "noarg_func ::= TIMEZONE", - /* 396 */ "noarg_func ::= DATABASE", - /* 397 */ "noarg_func ::= CLIENT_VERSION", - /* 398 */ "noarg_func ::= SERVER_VERSION", - /* 399 */ "noarg_func ::= SERVER_STATUS", - /* 400 */ "noarg_func ::= CURRENT_USER", - /* 401 */ "noarg_func ::= USER", - /* 402 */ "star_func ::= COUNT", - /* 403 */ "star_func ::= FIRST", - /* 404 */ "star_func ::= LAST", - /* 405 */ "star_func ::= LAST_ROW", - /* 406 */ "star_func_para_list ::= NK_STAR", - /* 407 */ "star_func_para_list ::= other_para_list", - /* 408 */ "other_para_list ::= star_func_para", - /* 409 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", - /* 410 */ "star_func_para ::= expr_or_subquery", - /* 411 */ "star_func_para ::= table_name NK_DOT NK_STAR", - /* 412 */ "case_when_expression ::= CASE when_then_list case_when_else_opt END", - /* 413 */ "case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END", - /* 414 */ "when_then_list ::= when_then_expr", - /* 415 */ "when_then_list ::= when_then_list when_then_expr", - /* 416 */ "when_then_expr ::= WHEN common_expression THEN common_expression", - /* 417 */ "case_when_else_opt ::=", - /* 418 */ "case_when_else_opt ::= ELSE common_expression", - /* 419 */ "predicate ::= expr_or_subquery compare_op expr_or_subquery", - /* 420 */ "predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery", - /* 421 */ "predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery", - /* 422 */ "predicate ::= expr_or_subquery IS NULL", - /* 423 */ "predicate ::= expr_or_subquery IS NOT NULL", - /* 424 */ "predicate ::= expr_or_subquery in_op in_predicate_value", - /* 425 */ "compare_op ::= NK_LT", - /* 426 */ "compare_op ::= NK_GT", - /* 427 */ "compare_op ::= NK_LE", - /* 428 */ "compare_op ::= NK_GE", - /* 429 */ "compare_op ::= NK_NE", - /* 430 */ "compare_op ::= NK_EQ", - /* 431 */ "compare_op ::= LIKE", - /* 432 */ "compare_op ::= NOT LIKE", - /* 433 */ "compare_op ::= MATCH", - /* 434 */ "compare_op ::= NMATCH", - /* 435 */ "compare_op ::= CONTAINS", - /* 436 */ "in_op ::= IN", - /* 437 */ "in_op ::= NOT IN", - /* 438 */ "in_predicate_value ::= NK_LP literal_list NK_RP", - /* 439 */ "boolean_value_expression ::= boolean_primary", - /* 440 */ "boolean_value_expression ::= NOT boolean_primary", - /* 441 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", - /* 442 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", - /* 443 */ "boolean_primary ::= predicate", - /* 444 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 445 */ "common_expression ::= expr_or_subquery", - /* 446 */ "common_expression ::= boolean_value_expression", - /* 447 */ "from_clause_opt ::=", - /* 448 */ "from_clause_opt ::= FROM table_reference_list", - /* 449 */ "table_reference_list ::= table_reference", - /* 450 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 451 */ "table_reference ::= table_primary", - /* 452 */ "table_reference ::= joined_table", - /* 453 */ "table_primary ::= table_name alias_opt", - /* 454 */ "table_primary ::= db_name NK_DOT table_name alias_opt", - /* 455 */ "table_primary ::= subquery alias_opt", - /* 456 */ "table_primary ::= parenthesized_joined_table", - /* 457 */ "alias_opt ::=", - /* 458 */ "alias_opt ::= table_alias", - /* 459 */ "alias_opt ::= AS table_alias", - /* 460 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 461 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 462 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 463 */ "join_type ::=", - /* 464 */ "join_type ::= INNER", - /* 465 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 466 */ "set_quantifier_opt ::=", - /* 467 */ "set_quantifier_opt ::= DISTINCT", - /* 468 */ "set_quantifier_opt ::= ALL", - /* 469 */ "select_list ::= select_item", - /* 470 */ "select_list ::= select_list NK_COMMA select_item", - /* 471 */ "select_item ::= NK_STAR", - /* 472 */ "select_item ::= common_expression", - /* 473 */ "select_item ::= common_expression column_alias", - /* 474 */ "select_item ::= common_expression AS column_alias", - /* 475 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 476 */ "where_clause_opt ::=", - /* 477 */ "where_clause_opt ::= WHERE search_condition", - /* 478 */ "partition_by_clause_opt ::=", - /* 479 */ "partition_by_clause_opt ::= PARTITION BY partition_list", - /* 480 */ "partition_list ::= partition_item", - /* 481 */ "partition_list ::= partition_list NK_COMMA partition_item", - /* 482 */ "partition_item ::= expr_or_subquery", - /* 483 */ "partition_item ::= expr_or_subquery column_alias", - /* 484 */ "partition_item ::= expr_or_subquery AS column_alias", - /* 485 */ "twindow_clause_opt ::=", - /* 486 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", - /* 487 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP", - /* 488 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 489 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 490 */ "sliding_opt ::=", - /* 491 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 492 */ "fill_opt ::=", - /* 493 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 494 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 495 */ "fill_mode ::= NONE", - /* 496 */ "fill_mode ::= PREV", - /* 497 */ "fill_mode ::= NULL", - /* 498 */ "fill_mode ::= LINEAR", - /* 499 */ "fill_mode ::= NEXT", - /* 500 */ "group_by_clause_opt ::=", - /* 501 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 502 */ "group_by_list ::= expr_or_subquery", - /* 503 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", - /* 504 */ "having_clause_opt ::=", - /* 505 */ "having_clause_opt ::= HAVING search_condition", - /* 506 */ "range_opt ::=", - /* 507 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", - /* 508 */ "every_opt ::=", - /* 509 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", - /* 510 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 511 */ "query_simple ::= query_specification", - /* 512 */ "query_simple ::= union_query_expression", - /* 513 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", - /* 514 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", - /* 515 */ "query_simple_or_subquery ::= query_simple", - /* 516 */ "query_simple_or_subquery ::= subquery", - /* 517 */ "query_or_subquery ::= query_expression", - /* 518 */ "query_or_subquery ::= subquery", - /* 519 */ "order_by_clause_opt ::=", - /* 520 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 521 */ "slimit_clause_opt ::=", - /* 522 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 523 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 524 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 525 */ "limit_clause_opt ::=", - /* 526 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 527 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 528 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 529 */ "subquery ::= NK_LP query_expression NK_RP", - /* 530 */ "subquery ::= NK_LP subquery NK_RP", - /* 531 */ "search_condition ::= common_expression", - /* 532 */ "sort_specification_list ::= sort_specification", - /* 533 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 534 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", - /* 535 */ "ordering_specification_opt ::=", - /* 536 */ "ordering_specification_opt ::= ASC", - /* 537 */ "ordering_specification_opt ::= DESC", - /* 538 */ "null_ordering_opt ::=", - /* 539 */ "null_ordering_opt ::= NULLS FIRST", - /* 540 */ "null_ordering_opt ::= NULLS LAST", + /* 245 */ "cmd ::= SHOW db_name_cond_opt ALIVE", + /* 246 */ "cmd ::= SHOW CLUSTER ALIVE", + /* 247 */ "db_name_cond_opt ::=", + /* 248 */ "db_name_cond_opt ::= db_name NK_DOT", + /* 249 */ "like_pattern_opt ::=", + /* 250 */ "like_pattern_opt ::= LIKE NK_STRING", + /* 251 */ "table_name_cond ::= table_name", + /* 252 */ "from_db_opt ::=", + /* 253 */ "from_db_opt ::= FROM db_name", + /* 254 */ "tag_list_opt ::=", + /* 255 */ "tag_list_opt ::= tag_item", + /* 256 */ "tag_list_opt ::= tag_list_opt NK_COMMA tag_item", + /* 257 */ "tag_item ::= TBNAME", + /* 258 */ "tag_item ::= QTAGS", + /* 259 */ "tag_item ::= column_name", + /* 260 */ "tag_item ::= column_name column_alias", + /* 261 */ "tag_item ::= column_name AS column_alias", + /* 262 */ "cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options", + /* 263 */ "cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP", + /* 264 */ "cmd ::= DROP INDEX exists_opt full_index_name", + /* 265 */ "full_index_name ::= index_name", + /* 266 */ "full_index_name ::= db_name NK_DOT index_name", + /* 267 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", + /* 268 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", + /* 269 */ "func_list ::= func", + /* 270 */ "func_list ::= func_list NK_COMMA func", + /* 271 */ "func ::= sma_func_name NK_LP expression_list NK_RP", + /* 272 */ "sma_func_name ::= function_name", + /* 273 */ "sma_func_name ::= COUNT", + /* 274 */ "sma_func_name ::= FIRST", + /* 275 */ "sma_func_name ::= LAST", + /* 276 */ "sma_func_name ::= LAST_ROW", + /* 277 */ "sma_stream_opt ::=", + /* 278 */ "sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal", + /* 279 */ "sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal", + /* 280 */ "sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal", + /* 281 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery", + /* 282 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", + /* 283 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", + /* 284 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name", + /* 285 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name", + /* 286 */ "cmd ::= DROP TOPIC exists_opt topic_name", + /* 287 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", + /* 288 */ "cmd ::= DESC full_table_name", + /* 289 */ "cmd ::= DESCRIBE full_table_name", + /* 290 */ "cmd ::= RESET QUERY CACHE", + /* 291 */ "cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery", + /* 292 */ "analyze_opt ::=", + /* 293 */ "analyze_opt ::= ANALYZE", + /* 294 */ "explain_options ::=", + /* 295 */ "explain_options ::= explain_options VERBOSE NK_BOOL", + /* 296 */ "explain_options ::= explain_options RATIO NK_FLOAT", + /* 297 */ "cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt", + /* 298 */ "cmd ::= DROP FUNCTION exists_opt function_name", + /* 299 */ "agg_func_opt ::=", + /* 300 */ "agg_func_opt ::= AGGREGATE", + /* 301 */ "bufsize_opt ::=", + /* 302 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", + /* 303 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tags_def_opt subtable_opt AS query_or_subquery", + /* 304 */ "cmd ::= DROP STREAM exists_opt stream_name", + /* 305 */ "col_list_opt ::=", + /* 306 */ "col_list_opt ::= NK_LP col_name_list NK_RP", + /* 307 */ "stream_options ::=", + /* 308 */ "stream_options ::= stream_options TRIGGER AT_ONCE", + /* 309 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", + /* 310 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", + /* 311 */ "stream_options ::= stream_options WATERMARK duration_literal", + /* 312 */ "stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER", + /* 313 */ "stream_options ::= stream_options FILL_HISTORY NK_INTEGER", + /* 314 */ "subtable_opt ::=", + /* 315 */ "subtable_opt ::= SUBTABLE NK_LP expression NK_RP", + /* 316 */ "cmd ::= KILL CONNECTION NK_INTEGER", + /* 317 */ "cmd ::= KILL QUERY NK_STRING", + /* 318 */ "cmd ::= KILL TRANSACTION NK_INTEGER", + /* 319 */ "cmd ::= BALANCE VGROUP", + /* 320 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", + /* 321 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", + /* 322 */ "cmd ::= SPLIT VGROUP NK_INTEGER", + /* 323 */ "dnode_list ::= DNODE NK_INTEGER", + /* 324 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", + /* 325 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", + /* 326 */ "cmd ::= query_or_subquery", + /* 327 */ "cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery", + /* 328 */ "cmd ::= INSERT INTO full_table_name query_or_subquery", + /* 329 */ "literal ::= NK_INTEGER", + /* 330 */ "literal ::= NK_FLOAT", + /* 331 */ "literal ::= NK_STRING", + /* 332 */ "literal ::= NK_BOOL", + /* 333 */ "literal ::= TIMESTAMP NK_STRING", + /* 334 */ "literal ::= duration_literal", + /* 335 */ "literal ::= NULL", + /* 336 */ "literal ::= NK_QUESTION", + /* 337 */ "duration_literal ::= NK_VARIABLE", + /* 338 */ "signed ::= NK_INTEGER", + /* 339 */ "signed ::= NK_PLUS NK_INTEGER", + /* 340 */ "signed ::= NK_MINUS NK_INTEGER", + /* 341 */ "signed ::= NK_FLOAT", + /* 342 */ "signed ::= NK_PLUS NK_FLOAT", + /* 343 */ "signed ::= NK_MINUS NK_FLOAT", + /* 344 */ "signed_literal ::= signed", + /* 345 */ "signed_literal ::= NK_STRING", + /* 346 */ "signed_literal ::= NK_BOOL", + /* 347 */ "signed_literal ::= TIMESTAMP NK_STRING", + /* 348 */ "signed_literal ::= duration_literal", + /* 349 */ "signed_literal ::= NULL", + /* 350 */ "signed_literal ::= literal_func", + /* 351 */ "signed_literal ::= NK_QUESTION", + /* 352 */ "literal_list ::= signed_literal", + /* 353 */ "literal_list ::= literal_list NK_COMMA signed_literal", + /* 354 */ "db_name ::= NK_ID", + /* 355 */ "table_name ::= NK_ID", + /* 356 */ "column_name ::= NK_ID", + /* 357 */ "function_name ::= NK_ID", + /* 358 */ "table_alias ::= NK_ID", + /* 359 */ "column_alias ::= NK_ID", + /* 360 */ "user_name ::= NK_ID", + /* 361 */ "topic_name ::= NK_ID", + /* 362 */ "stream_name ::= NK_ID", + /* 363 */ "cgroup_name ::= NK_ID", + /* 364 */ "index_name ::= NK_ID", + /* 365 */ "expr_or_subquery ::= expression", + /* 366 */ "expression ::= literal", + /* 367 */ "expression ::= pseudo_column", + /* 368 */ "expression ::= column_reference", + /* 369 */ "expression ::= function_expression", + /* 370 */ "expression ::= case_when_expression", + /* 371 */ "expression ::= NK_LP expression NK_RP", + /* 372 */ "expression ::= NK_PLUS expr_or_subquery", + /* 373 */ "expression ::= NK_MINUS expr_or_subquery", + /* 374 */ "expression ::= expr_or_subquery NK_PLUS expr_or_subquery", + /* 375 */ "expression ::= expr_or_subquery NK_MINUS expr_or_subquery", + /* 376 */ "expression ::= expr_or_subquery NK_STAR expr_or_subquery", + /* 377 */ "expression ::= expr_or_subquery NK_SLASH expr_or_subquery", + /* 378 */ "expression ::= expr_or_subquery NK_REM expr_or_subquery", + /* 379 */ "expression ::= column_reference NK_ARROW NK_STRING", + /* 380 */ "expression ::= expr_or_subquery NK_BITAND expr_or_subquery", + /* 381 */ "expression ::= expr_or_subquery NK_BITOR expr_or_subquery", + /* 382 */ "expression_list ::= expr_or_subquery", + /* 383 */ "expression_list ::= expression_list NK_COMMA expr_or_subquery", + /* 384 */ "column_reference ::= column_name", + /* 385 */ "column_reference ::= table_name NK_DOT column_name", + /* 386 */ "pseudo_column ::= ROWTS", + /* 387 */ "pseudo_column ::= TBNAME", + /* 388 */ "pseudo_column ::= table_name NK_DOT TBNAME", + /* 389 */ "pseudo_column ::= QSTART", + /* 390 */ "pseudo_column ::= QEND", + /* 391 */ "pseudo_column ::= QDURATION", + /* 392 */ "pseudo_column ::= WSTART", + /* 393 */ "pseudo_column ::= WEND", + /* 394 */ "pseudo_column ::= WDURATION", + /* 395 */ "pseudo_column ::= IROWTS", + /* 396 */ "pseudo_column ::= ISFILLED", + /* 397 */ "pseudo_column ::= QTAGS", + /* 398 */ "function_expression ::= function_name NK_LP expression_list NK_RP", + /* 399 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", + /* 400 */ "function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP", + /* 401 */ "function_expression ::= literal_func", + /* 402 */ "literal_func ::= noarg_func NK_LP NK_RP", + /* 403 */ "literal_func ::= NOW", + /* 404 */ "noarg_func ::= NOW", + /* 405 */ "noarg_func ::= TODAY", + /* 406 */ "noarg_func ::= TIMEZONE", + /* 407 */ "noarg_func ::= DATABASE", + /* 408 */ "noarg_func ::= CLIENT_VERSION", + /* 409 */ "noarg_func ::= SERVER_VERSION", + /* 410 */ "noarg_func ::= SERVER_STATUS", + /* 411 */ "noarg_func ::= CURRENT_USER", + /* 412 */ "noarg_func ::= USER", + /* 413 */ "star_func ::= COUNT", + /* 414 */ "star_func ::= FIRST", + /* 415 */ "star_func ::= LAST", + /* 416 */ "star_func ::= LAST_ROW", + /* 417 */ "star_func_para_list ::= NK_STAR", + /* 418 */ "star_func_para_list ::= other_para_list", + /* 419 */ "other_para_list ::= star_func_para", + /* 420 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", + /* 421 */ "star_func_para ::= expr_or_subquery", + /* 422 */ "star_func_para ::= table_name NK_DOT NK_STAR", + /* 423 */ "case_when_expression ::= CASE when_then_list case_when_else_opt END", + /* 424 */ "case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END", + /* 425 */ "when_then_list ::= when_then_expr", + /* 426 */ "when_then_list ::= when_then_list when_then_expr", + /* 427 */ "when_then_expr ::= WHEN common_expression THEN common_expression", + /* 428 */ "case_when_else_opt ::=", + /* 429 */ "case_when_else_opt ::= ELSE common_expression", + /* 430 */ "predicate ::= expr_or_subquery compare_op expr_or_subquery", + /* 431 */ "predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery", + /* 432 */ "predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery", + /* 433 */ "predicate ::= expr_or_subquery IS NULL", + /* 434 */ "predicate ::= expr_or_subquery IS NOT NULL", + /* 435 */ "predicate ::= expr_or_subquery in_op in_predicate_value", + /* 436 */ "compare_op ::= NK_LT", + /* 437 */ "compare_op ::= NK_GT", + /* 438 */ "compare_op ::= NK_LE", + /* 439 */ "compare_op ::= NK_GE", + /* 440 */ "compare_op ::= NK_NE", + /* 441 */ "compare_op ::= NK_EQ", + /* 442 */ "compare_op ::= LIKE", + /* 443 */ "compare_op ::= NOT LIKE", + /* 444 */ "compare_op ::= MATCH", + /* 445 */ "compare_op ::= NMATCH", + /* 446 */ "compare_op ::= CONTAINS", + /* 447 */ "in_op ::= IN", + /* 448 */ "in_op ::= NOT IN", + /* 449 */ "in_predicate_value ::= NK_LP literal_list NK_RP", + /* 450 */ "boolean_value_expression ::= boolean_primary", + /* 451 */ "boolean_value_expression ::= NOT boolean_primary", + /* 452 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 453 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 454 */ "boolean_primary ::= predicate", + /* 455 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 456 */ "common_expression ::= expr_or_subquery", + /* 457 */ "common_expression ::= boolean_value_expression", + /* 458 */ "from_clause_opt ::=", + /* 459 */ "from_clause_opt ::= FROM table_reference_list", + /* 460 */ "table_reference_list ::= table_reference", + /* 461 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 462 */ "table_reference ::= table_primary", + /* 463 */ "table_reference ::= joined_table", + /* 464 */ "table_primary ::= table_name alias_opt", + /* 465 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 466 */ "table_primary ::= subquery alias_opt", + /* 467 */ "table_primary ::= parenthesized_joined_table", + /* 468 */ "alias_opt ::=", + /* 469 */ "alias_opt ::= table_alias", + /* 470 */ "alias_opt ::= AS table_alias", + /* 471 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 472 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 473 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 474 */ "join_type ::=", + /* 475 */ "join_type ::= INNER", + /* 476 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 477 */ "set_quantifier_opt ::=", + /* 478 */ "set_quantifier_opt ::= DISTINCT", + /* 479 */ "set_quantifier_opt ::= ALL", + /* 480 */ "select_list ::= select_item", + /* 481 */ "select_list ::= select_list NK_COMMA select_item", + /* 482 */ "select_item ::= NK_STAR", + /* 483 */ "select_item ::= common_expression", + /* 484 */ "select_item ::= common_expression column_alias", + /* 485 */ "select_item ::= common_expression AS column_alias", + /* 486 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 487 */ "where_clause_opt ::=", + /* 488 */ "where_clause_opt ::= WHERE search_condition", + /* 489 */ "partition_by_clause_opt ::=", + /* 490 */ "partition_by_clause_opt ::= PARTITION BY partition_list", + /* 491 */ "partition_list ::= partition_item", + /* 492 */ "partition_list ::= partition_list NK_COMMA partition_item", + /* 493 */ "partition_item ::= expr_or_subquery", + /* 494 */ "partition_item ::= expr_or_subquery column_alias", + /* 495 */ "partition_item ::= expr_or_subquery AS column_alias", + /* 496 */ "twindow_clause_opt ::=", + /* 497 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", + /* 498 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP", + /* 499 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 500 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 501 */ "twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition", + /* 502 */ "sliding_opt ::=", + /* 503 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 504 */ "fill_opt ::=", + /* 505 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 506 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 507 */ "fill_mode ::= NONE", + /* 508 */ "fill_mode ::= PREV", + /* 509 */ "fill_mode ::= NULL", + /* 510 */ "fill_mode ::= LINEAR", + /* 511 */ "fill_mode ::= NEXT", + /* 512 */ "group_by_clause_opt ::=", + /* 513 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 514 */ "group_by_list ::= expr_or_subquery", + /* 515 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", + /* 516 */ "having_clause_opt ::=", + /* 517 */ "having_clause_opt ::= HAVING search_condition", + /* 518 */ "range_opt ::=", + /* 519 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", + /* 520 */ "every_opt ::=", + /* 521 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", + /* 522 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 523 */ "query_simple ::= query_specification", + /* 524 */ "query_simple ::= union_query_expression", + /* 525 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", + /* 526 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", + /* 527 */ "query_simple_or_subquery ::= query_simple", + /* 528 */ "query_simple_or_subquery ::= subquery", + /* 529 */ "query_or_subquery ::= query_expression", + /* 530 */ "query_or_subquery ::= subquery", + /* 531 */ "order_by_clause_opt ::=", + /* 532 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 533 */ "slimit_clause_opt ::=", + /* 534 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 535 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 536 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 537 */ "limit_clause_opt ::=", + /* 538 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 539 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 540 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 541 */ "subquery ::= NK_LP query_expression NK_RP", + /* 542 */ "subquery ::= NK_LP subquery NK_RP", + /* 543 */ "search_condition ::= common_expression", + /* 544 */ "sort_specification_list ::= sort_specification", + /* 545 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 546 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", + /* 547 */ "ordering_specification_opt ::=", + /* 548 */ "ordering_specification_opt ::= ASC", + /* 549 */ "ordering_specification_opt ::= DESC", + /* 550 */ "null_ordering_opt ::=", + /* 551 */ "null_ordering_opt ::= NULLS FIRST", + /* 552 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -2513,195 +2533,197 @@ static void yy_destructor( */ /********* Begin destructor definitions ***************************************/ /* Default NON-TERMINAL Destructor */ - case 322: /* cmd */ - case 325: /* literal */ - case 338: /* db_options */ - case 340: /* alter_db_options */ - case 346: /* retention */ - case 347: /* full_table_name */ - case 350: /* table_options */ - case 354: /* alter_table_clause */ - case 355: /* alter_table_options */ - case 358: /* signed_literal */ - case 359: /* create_subtable_clause */ - case 362: /* drop_table_clause */ - case 365: /* column_def */ - case 369: /* duration_literal */ - case 370: /* rollup_func_name */ - case 372: /* col_name */ - case 373: /* db_name_cond_opt */ - case 374: /* like_pattern_opt */ - case 375: /* table_name_cond */ - case 376: /* from_db_opt */ - case 378: /* tag_item */ - case 380: /* full_index_name */ - case 381: /* index_options */ - case 384: /* sliding_opt */ - case 385: /* sma_stream_opt */ - case 386: /* func */ - case 387: /* query_or_subquery */ - case 390: /* explain_options */ - case 394: /* stream_options */ - case 395: /* subtable_opt */ - case 396: /* expression */ - case 398: /* where_clause_opt */ - case 399: /* signed */ - case 400: /* literal_func */ - case 403: /* expr_or_subquery */ - case 404: /* pseudo_column */ - case 405: /* column_reference */ - case 406: /* function_expression */ - case 407: /* case_when_expression */ - case 412: /* star_func_para */ - case 414: /* case_when_else_opt */ - case 415: /* common_expression */ - case 416: /* when_then_expr */ - case 417: /* predicate */ - case 420: /* in_predicate_value */ - case 421: /* boolean_value_expression */ - case 422: /* boolean_primary */ - case 423: /* from_clause_opt */ - case 424: /* table_reference_list */ - case 425: /* table_reference */ - case 426: /* table_primary */ - case 427: /* joined_table */ - case 429: /* subquery */ - case 430: /* parenthesized_joined_table */ - case 432: /* search_condition */ - case 433: /* query_specification */ - case 437: /* range_opt */ - case 438: /* every_opt */ - case 439: /* fill_opt */ - case 440: /* twindow_clause_opt */ - case 442: /* having_clause_opt */ - case 443: /* select_item */ - case 445: /* partition_item */ - case 448: /* query_expression */ - case 449: /* query_simple */ - case 451: /* slimit_clause_opt */ - case 452: /* limit_clause_opt */ - case 453: /* union_query_expression */ - case 454: /* query_simple_or_subquery */ - case 456: /* sort_specification */ + case 326: /* cmd */ + case 329: /* literal */ + case 342: /* db_options */ + case 344: /* alter_db_options */ + case 350: /* retention */ + case 351: /* full_table_name */ + case 354: /* table_options */ + case 358: /* alter_table_clause */ + case 359: /* alter_table_options */ + case 362: /* signed_literal */ + case 363: /* create_subtable_clause */ + case 366: /* drop_table_clause */ + case 369: /* column_def */ + case 373: /* duration_literal */ + case 374: /* rollup_func_name */ + case 376: /* col_name */ + case 377: /* db_name_cond_opt */ + case 378: /* like_pattern_opt */ + case 379: /* table_name_cond */ + case 380: /* from_db_opt */ + case 382: /* tag_item */ + case 384: /* full_index_name */ + case 385: /* index_options */ + case 388: /* sliding_opt */ + case 389: /* sma_stream_opt */ + case 390: /* func */ + case 392: /* query_or_subquery */ + case 395: /* explain_options */ + case 399: /* stream_options */ + case 401: /* subtable_opt */ + case 402: /* expression */ + case 404: /* where_clause_opt */ + case 405: /* signed */ + case 406: /* literal_func */ + case 409: /* expr_or_subquery */ + case 410: /* pseudo_column */ + case 411: /* column_reference */ + case 412: /* function_expression */ + case 413: /* case_when_expression */ + case 418: /* star_func_para */ + case 420: /* case_when_else_opt */ + case 421: /* common_expression */ + case 422: /* when_then_expr */ + case 423: /* predicate */ + case 426: /* in_predicate_value */ + case 427: /* boolean_value_expression */ + case 428: /* boolean_primary */ + case 429: /* from_clause_opt */ + case 430: /* table_reference_list */ + case 431: /* table_reference */ + case 432: /* table_primary */ + case 433: /* joined_table */ + case 435: /* subquery */ + case 436: /* parenthesized_joined_table */ + case 438: /* search_condition */ + case 439: /* query_specification */ + case 443: /* range_opt */ + case 444: /* every_opt */ + case 445: /* fill_opt */ + case 446: /* twindow_clause_opt */ + case 448: /* having_clause_opt */ + case 449: /* select_item */ + case 451: /* partition_item */ + case 454: /* query_expression */ + case 455: /* query_simple */ + case 457: /* slimit_clause_opt */ + case 458: /* limit_clause_opt */ + case 459: /* union_query_expression */ + case 460: /* query_simple_or_subquery */ + case 462: /* sort_specification */ { - nodesDestroyNode((yypminor->yy74)); + nodesDestroyNode((yypminor->yy602)); } break; - case 323: /* account_options */ - case 324: /* alter_account_options */ - case 326: /* alter_account_option */ - case 341: /* speed_opt */ - case 392: /* bufsize_opt */ + case 327: /* account_options */ + case 328: /* alter_account_options */ + case 330: /* alter_account_option */ + case 345: /* speed_opt */ + case 397: /* bufsize_opt */ { } break; - case 327: /* user_name */ - case 330: /* priv_level */ - case 333: /* db_name */ - case 334: /* topic_name */ - case 335: /* dnode_endpoint */ - case 356: /* column_name */ - case 364: /* table_name */ - case 371: /* function_name */ - case 379: /* column_alias */ - case 382: /* index_name */ - case 388: /* cgroup_name */ - case 393: /* stream_name */ - case 402: /* table_alias */ - case 408: /* star_func */ - case 410: /* noarg_func */ - case 428: /* alias_opt */ + case 331: /* user_name */ + case 334: /* priv_level */ + case 337: /* db_name */ + case 338: /* topic_name */ + case 339: /* dnode_endpoint */ + case 360: /* column_name */ + case 368: /* table_name */ + case 375: /* function_name */ + case 383: /* column_alias */ + case 386: /* index_name */ + case 391: /* sma_func_name */ + case 393: /* cgroup_name */ + case 398: /* stream_name */ + case 408: /* table_alias */ + case 414: /* star_func */ + case 416: /* noarg_func */ + case 434: /* alias_opt */ { } break; - case 328: /* sysinfo_opt */ + case 332: /* sysinfo_opt */ { } break; - case 329: /* privileges */ - case 331: /* priv_type_list */ - case 332: /* priv_type */ + case 333: /* privileges */ + case 335: /* priv_type_list */ + case 336: /* priv_type */ { } break; - case 336: /* force_opt */ - case 337: /* not_exists_opt */ - case 339: /* exists_opt */ - case 389: /* analyze_opt */ - case 391: /* agg_func_opt */ - case 434: /* set_quantifier_opt */ + case 340: /* force_opt */ + case 341: /* not_exists_opt */ + case 343: /* exists_opt */ + case 394: /* analyze_opt */ + case 396: /* agg_func_opt */ + case 440: /* set_quantifier_opt */ { } break; - case 342: /* integer_list */ - case 343: /* variable_list */ - case 344: /* retention_list */ - case 348: /* column_def_list */ - case 349: /* tags_def_opt */ - case 351: /* multi_create_clause */ - case 352: /* tags_def */ - case 353: /* multi_drop_clause */ - case 360: /* specific_cols_opt */ - case 361: /* expression_list */ - case 363: /* col_name_list */ - case 366: /* duration_list */ - case 367: /* rollup_func_list */ - case 377: /* tag_list_opt */ - case 383: /* func_list */ - case 397: /* dnode_list */ - case 401: /* literal_list */ - case 409: /* star_func_para_list */ - case 411: /* other_para_list */ - case 413: /* when_then_list */ - case 435: /* select_list */ - case 436: /* partition_by_clause_opt */ - case 441: /* group_by_clause_opt */ - case 444: /* partition_list */ - case 447: /* group_by_list */ - case 450: /* order_by_clause_opt */ - case 455: /* sort_specification_list */ + case 346: /* integer_list */ + case 347: /* variable_list */ + case 348: /* retention_list */ + case 352: /* column_def_list */ + case 353: /* tags_def_opt */ + case 355: /* multi_create_clause */ + case 356: /* tags_def */ + case 357: /* multi_drop_clause */ + case 364: /* specific_cols_opt */ + case 365: /* expression_list */ + case 367: /* col_name_list */ + case 370: /* duration_list */ + case 371: /* rollup_func_list */ + case 381: /* tag_list_opt */ + case 387: /* func_list */ + case 400: /* col_list_opt */ + case 403: /* dnode_list */ + case 407: /* literal_list */ + case 415: /* star_func_para_list */ + case 417: /* other_para_list */ + case 419: /* when_then_list */ + case 441: /* select_list */ + case 442: /* partition_by_clause_opt */ + case 447: /* group_by_clause_opt */ + case 450: /* partition_list */ + case 453: /* group_by_list */ + case 456: /* order_by_clause_opt */ + case 461: /* sort_specification_list */ { nodesDestroyList((yypminor->yy874)); } break; - case 345: /* alter_db_option */ - case 368: /* alter_table_option */ + case 349: /* alter_db_option */ + case 372: /* alter_table_option */ { } break; - case 357: /* type_name */ + case 361: /* type_name */ { } break; - case 418: /* compare_op */ - case 419: /* in_op */ + case 424: /* compare_op */ + case 425: /* in_op */ { } break; - case 431: /* join_type */ + case 437: /* join_type */ { } break; - case 446: /* fill_mode */ + case 452: /* fill_mode */ { } break; - case 457: /* ordering_specification_opt */ + case 463: /* ordering_specification_opt */ { } break; - case 458: /* null_ordering_opt */ + case 464: /* null_ordering_opt */ { } @@ -3000,547 +3022,559 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ signed char nrhs; /* Negative of the number of RHS symbols in the rule */ } yyRuleInfo[] = { - { 322, -6 }, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ - { 322, -4 }, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ - { 323, 0 }, /* (2) account_options ::= */ - { 323, -3 }, /* (3) account_options ::= account_options PPS literal */ - { 323, -3 }, /* (4) account_options ::= account_options TSERIES literal */ - { 323, -3 }, /* (5) account_options ::= account_options STORAGE literal */ - { 323, -3 }, /* (6) account_options ::= account_options STREAMS literal */ - { 323, -3 }, /* (7) account_options ::= account_options QTIME literal */ - { 323, -3 }, /* (8) account_options ::= account_options DBS literal */ - { 323, -3 }, /* (9) account_options ::= account_options USERS literal */ - { 323, -3 }, /* (10) account_options ::= account_options CONNS literal */ - { 323, -3 }, /* (11) account_options ::= account_options STATE literal */ - { 324, -1 }, /* (12) alter_account_options ::= alter_account_option */ - { 324, -2 }, /* (13) alter_account_options ::= alter_account_options alter_account_option */ - { 326, -2 }, /* (14) alter_account_option ::= PASS literal */ - { 326, -2 }, /* (15) alter_account_option ::= PPS literal */ - { 326, -2 }, /* (16) alter_account_option ::= TSERIES literal */ - { 326, -2 }, /* (17) alter_account_option ::= STORAGE literal */ - { 326, -2 }, /* (18) alter_account_option ::= STREAMS literal */ - { 326, -2 }, /* (19) alter_account_option ::= QTIME literal */ - { 326, -2 }, /* (20) alter_account_option ::= DBS literal */ - { 326, -2 }, /* (21) alter_account_option ::= USERS literal */ - { 326, -2 }, /* (22) alter_account_option ::= CONNS literal */ - { 326, -2 }, /* (23) alter_account_option ::= STATE literal */ - { 322, -6 }, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ - { 322, -5 }, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ - { 322, -5 }, /* (26) cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ - { 322, -5 }, /* (27) cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ - { 322, -3 }, /* (28) cmd ::= DROP USER user_name */ - { 328, 0 }, /* (29) sysinfo_opt ::= */ - { 328, -2 }, /* (30) sysinfo_opt ::= SYSINFO NK_INTEGER */ - { 322, -6 }, /* (31) cmd ::= GRANT privileges ON priv_level TO user_name */ - { 322, -6 }, /* (32) cmd ::= REVOKE privileges ON priv_level FROM user_name */ - { 329, -1 }, /* (33) privileges ::= ALL */ - { 329, -1 }, /* (34) privileges ::= priv_type_list */ - { 329, -1 }, /* (35) privileges ::= SUBSCRIBE */ - { 331, -1 }, /* (36) priv_type_list ::= priv_type */ - { 331, -3 }, /* (37) priv_type_list ::= priv_type_list NK_COMMA priv_type */ - { 332, -1 }, /* (38) priv_type ::= READ */ - { 332, -1 }, /* (39) priv_type ::= WRITE */ - { 330, -3 }, /* (40) priv_level ::= NK_STAR NK_DOT NK_STAR */ - { 330, -3 }, /* (41) priv_level ::= db_name NK_DOT NK_STAR */ - { 330, -1 }, /* (42) priv_level ::= topic_name */ - { 322, -3 }, /* (43) cmd ::= CREATE DNODE dnode_endpoint */ - { 322, -5 }, /* (44) cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ - { 322, -4 }, /* (45) cmd ::= DROP DNODE NK_INTEGER force_opt */ - { 322, -4 }, /* (46) cmd ::= DROP DNODE dnode_endpoint force_opt */ - { 322, -4 }, /* (47) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ - { 322, -5 }, /* (48) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ - { 322, -4 }, /* (49) cmd ::= ALTER ALL DNODES NK_STRING */ - { 322, -5 }, /* (50) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ - { 335, -1 }, /* (51) dnode_endpoint ::= NK_STRING */ - { 335, -1 }, /* (52) dnode_endpoint ::= NK_ID */ - { 335, -1 }, /* (53) dnode_endpoint ::= NK_IPTOKEN */ - { 336, 0 }, /* (54) force_opt ::= */ - { 336, -1 }, /* (55) force_opt ::= FORCE */ - { 322, -3 }, /* (56) cmd ::= ALTER LOCAL NK_STRING */ - { 322, -4 }, /* (57) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ - { 322, -5 }, /* (58) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ - { 322, -5 }, /* (59) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ - { 322, -5 }, /* (60) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ - { 322, -5 }, /* (61) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ - { 322, -5 }, /* (62) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ - { 322, -5 }, /* (63) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ - { 322, -5 }, /* (64) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ - { 322, -5 }, /* (65) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ - { 322, -5 }, /* (66) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ - { 322, -4 }, /* (67) cmd ::= DROP DATABASE exists_opt db_name */ - { 322, -2 }, /* (68) cmd ::= USE db_name */ - { 322, -4 }, /* (69) cmd ::= ALTER DATABASE db_name alter_db_options */ - { 322, -3 }, /* (70) cmd ::= FLUSH DATABASE db_name */ - { 322, -4 }, /* (71) cmd ::= TRIM DATABASE db_name speed_opt */ - { 337, -3 }, /* (72) not_exists_opt ::= IF NOT EXISTS */ - { 337, 0 }, /* (73) not_exists_opt ::= */ - { 339, -2 }, /* (74) exists_opt ::= IF EXISTS */ - { 339, 0 }, /* (75) exists_opt ::= */ - { 338, 0 }, /* (76) db_options ::= */ - { 338, -3 }, /* (77) db_options ::= db_options BUFFER NK_INTEGER */ - { 338, -3 }, /* (78) db_options ::= db_options CACHEMODEL NK_STRING */ - { 338, -3 }, /* (79) db_options ::= db_options CACHESIZE NK_INTEGER */ - { 338, -3 }, /* (80) db_options ::= db_options COMP NK_INTEGER */ - { 338, -3 }, /* (81) db_options ::= db_options DURATION NK_INTEGER */ - { 338, -3 }, /* (82) db_options ::= db_options DURATION NK_VARIABLE */ - { 338, -3 }, /* (83) db_options ::= db_options MAXROWS NK_INTEGER */ - { 338, -3 }, /* (84) db_options ::= db_options MINROWS NK_INTEGER */ - { 338, -3 }, /* (85) db_options ::= db_options KEEP integer_list */ - { 338, -3 }, /* (86) db_options ::= db_options KEEP variable_list */ - { 338, -3 }, /* (87) db_options ::= db_options PAGES NK_INTEGER */ - { 338, -3 }, /* (88) db_options ::= db_options PAGESIZE NK_INTEGER */ - { 338, -3 }, /* (89) db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ - { 338, -3 }, /* (90) db_options ::= db_options PRECISION NK_STRING */ - { 338, -3 }, /* (91) db_options ::= db_options REPLICA NK_INTEGER */ - { 338, -3 }, /* (92) db_options ::= db_options VGROUPS NK_INTEGER */ - { 338, -3 }, /* (93) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ - { 338, -3 }, /* (94) db_options ::= db_options RETENTIONS retention_list */ - { 338, -3 }, /* (95) db_options ::= db_options SCHEMALESS NK_INTEGER */ - { 338, -3 }, /* (96) db_options ::= db_options WAL_LEVEL NK_INTEGER */ - { 338, -3 }, /* (97) db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ - { 338, -3 }, /* (98) db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ - { 338, -4 }, /* (99) db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ - { 338, -3 }, /* (100) db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ - { 338, -4 }, /* (101) db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ - { 338, -3 }, /* (102) db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ - { 338, -3 }, /* (103) db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ - { 338, -3 }, /* (104) db_options ::= db_options STT_TRIGGER NK_INTEGER */ - { 338, -3 }, /* (105) db_options ::= db_options TABLE_PREFIX NK_INTEGER */ - { 338, -3 }, /* (106) db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ - { 340, -1 }, /* (107) alter_db_options ::= alter_db_option */ - { 340, -2 }, /* (108) alter_db_options ::= alter_db_options alter_db_option */ - { 345, -2 }, /* (109) alter_db_option ::= BUFFER NK_INTEGER */ - { 345, -2 }, /* (110) alter_db_option ::= CACHEMODEL NK_STRING */ - { 345, -2 }, /* (111) alter_db_option ::= CACHESIZE NK_INTEGER */ - { 345, -2 }, /* (112) alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ - { 345, -2 }, /* (113) alter_db_option ::= KEEP integer_list */ - { 345, -2 }, /* (114) alter_db_option ::= KEEP variable_list */ - { 345, -2 }, /* (115) alter_db_option ::= PAGES NK_INTEGER */ - { 345, -2 }, /* (116) alter_db_option ::= REPLICA NK_INTEGER */ - { 345, -2 }, /* (117) alter_db_option ::= WAL_LEVEL NK_INTEGER */ - { 345, -2 }, /* (118) alter_db_option ::= STT_TRIGGER NK_INTEGER */ - { 342, -1 }, /* (119) integer_list ::= NK_INTEGER */ - { 342, -3 }, /* (120) integer_list ::= integer_list NK_COMMA NK_INTEGER */ - { 343, -1 }, /* (121) variable_list ::= NK_VARIABLE */ - { 343, -3 }, /* (122) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ - { 344, -1 }, /* (123) retention_list ::= retention */ - { 344, -3 }, /* (124) retention_list ::= retention_list NK_COMMA retention */ - { 346, -3 }, /* (125) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ - { 341, 0 }, /* (126) speed_opt ::= */ - { 341, -2 }, /* (127) speed_opt ::= MAX_SPEED NK_INTEGER */ - { 322, -9 }, /* (128) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - { 322, -3 }, /* (129) cmd ::= CREATE TABLE multi_create_clause */ - { 322, -9 }, /* (130) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ - { 322, -3 }, /* (131) cmd ::= DROP TABLE multi_drop_clause */ - { 322, -4 }, /* (132) cmd ::= DROP STABLE exists_opt full_table_name */ - { 322, -3 }, /* (133) cmd ::= ALTER TABLE alter_table_clause */ - { 322, -3 }, /* (134) cmd ::= ALTER STABLE alter_table_clause */ - { 354, -2 }, /* (135) alter_table_clause ::= full_table_name alter_table_options */ - { 354, -5 }, /* (136) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ - { 354, -4 }, /* (137) alter_table_clause ::= full_table_name DROP COLUMN column_name */ - { 354, -5 }, /* (138) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ - { 354, -5 }, /* (139) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ - { 354, -5 }, /* (140) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ - { 354, -4 }, /* (141) alter_table_clause ::= full_table_name DROP TAG column_name */ - { 354, -5 }, /* (142) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ - { 354, -5 }, /* (143) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ - { 354, -6 }, /* (144) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ - { 351, -1 }, /* (145) multi_create_clause ::= create_subtable_clause */ - { 351, -2 }, /* (146) multi_create_clause ::= multi_create_clause create_subtable_clause */ - { 359, -10 }, /* (147) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ - { 353, -1 }, /* (148) multi_drop_clause ::= drop_table_clause */ - { 353, -2 }, /* (149) multi_drop_clause ::= multi_drop_clause drop_table_clause */ - { 362, -2 }, /* (150) drop_table_clause ::= exists_opt full_table_name */ - { 360, 0 }, /* (151) specific_cols_opt ::= */ - { 360, -3 }, /* (152) specific_cols_opt ::= NK_LP col_name_list NK_RP */ - { 347, -1 }, /* (153) full_table_name ::= table_name */ - { 347, -3 }, /* (154) full_table_name ::= db_name NK_DOT table_name */ - { 348, -1 }, /* (155) column_def_list ::= column_def */ - { 348, -3 }, /* (156) column_def_list ::= column_def_list NK_COMMA column_def */ - { 365, -2 }, /* (157) column_def ::= column_name type_name */ - { 365, -4 }, /* (158) column_def ::= column_name type_name COMMENT NK_STRING */ - { 357, -1 }, /* (159) type_name ::= BOOL */ - { 357, -1 }, /* (160) type_name ::= TINYINT */ - { 357, -1 }, /* (161) type_name ::= SMALLINT */ - { 357, -1 }, /* (162) type_name ::= INT */ - { 357, -1 }, /* (163) type_name ::= INTEGER */ - { 357, -1 }, /* (164) type_name ::= BIGINT */ - { 357, -1 }, /* (165) type_name ::= FLOAT */ - { 357, -1 }, /* (166) type_name ::= DOUBLE */ - { 357, -4 }, /* (167) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ - { 357, -1 }, /* (168) type_name ::= TIMESTAMP */ - { 357, -4 }, /* (169) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ - { 357, -2 }, /* (170) type_name ::= TINYINT UNSIGNED */ - { 357, -2 }, /* (171) type_name ::= SMALLINT UNSIGNED */ - { 357, -2 }, /* (172) type_name ::= INT UNSIGNED */ - { 357, -2 }, /* (173) type_name ::= BIGINT UNSIGNED */ - { 357, -1 }, /* (174) type_name ::= JSON */ - { 357, -4 }, /* (175) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ - { 357, -1 }, /* (176) type_name ::= MEDIUMBLOB */ - { 357, -1 }, /* (177) type_name ::= BLOB */ - { 357, -4 }, /* (178) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ - { 357, -1 }, /* (179) type_name ::= DECIMAL */ - { 357, -4 }, /* (180) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ - { 357, -6 }, /* (181) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ - { 349, 0 }, /* (182) tags_def_opt ::= */ - { 349, -1 }, /* (183) tags_def_opt ::= tags_def */ - { 352, -4 }, /* (184) tags_def ::= TAGS NK_LP column_def_list NK_RP */ - { 350, 0 }, /* (185) table_options ::= */ - { 350, -3 }, /* (186) table_options ::= table_options COMMENT NK_STRING */ - { 350, -3 }, /* (187) table_options ::= table_options MAX_DELAY duration_list */ - { 350, -3 }, /* (188) table_options ::= table_options WATERMARK duration_list */ - { 350, -5 }, /* (189) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ - { 350, -3 }, /* (190) table_options ::= table_options TTL NK_INTEGER */ - { 350, -5 }, /* (191) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ - { 350, -3 }, /* (192) table_options ::= table_options DELETE_MARK duration_list */ - { 355, -1 }, /* (193) alter_table_options ::= alter_table_option */ - { 355, -2 }, /* (194) alter_table_options ::= alter_table_options alter_table_option */ - { 368, -2 }, /* (195) alter_table_option ::= COMMENT NK_STRING */ - { 368, -2 }, /* (196) alter_table_option ::= TTL NK_INTEGER */ - { 366, -1 }, /* (197) duration_list ::= duration_literal */ - { 366, -3 }, /* (198) duration_list ::= duration_list NK_COMMA duration_literal */ - { 367, -1 }, /* (199) rollup_func_list ::= rollup_func_name */ - { 367, -3 }, /* (200) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ - { 370, -1 }, /* (201) rollup_func_name ::= function_name */ - { 370, -1 }, /* (202) rollup_func_name ::= FIRST */ - { 370, -1 }, /* (203) rollup_func_name ::= LAST */ - { 363, -1 }, /* (204) col_name_list ::= col_name */ - { 363, -3 }, /* (205) col_name_list ::= col_name_list NK_COMMA col_name */ - { 372, -1 }, /* (206) col_name ::= column_name */ - { 322, -2 }, /* (207) cmd ::= SHOW DNODES */ - { 322, -2 }, /* (208) cmd ::= SHOW USERS */ - { 322, -3 }, /* (209) cmd ::= SHOW USER PRIVILEGES */ - { 322, -2 }, /* (210) cmd ::= SHOW DATABASES */ - { 322, -4 }, /* (211) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ - { 322, -4 }, /* (212) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ - { 322, -3 }, /* (213) cmd ::= SHOW db_name_cond_opt VGROUPS */ - { 322, -2 }, /* (214) cmd ::= SHOW MNODES */ - { 322, -2 }, /* (215) cmd ::= SHOW QNODES */ - { 322, -2 }, /* (216) cmd ::= SHOW FUNCTIONS */ - { 322, -5 }, /* (217) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ - { 322, -2 }, /* (218) cmd ::= SHOW STREAMS */ - { 322, -2 }, /* (219) cmd ::= SHOW ACCOUNTS */ - { 322, -2 }, /* (220) cmd ::= SHOW APPS */ - { 322, -2 }, /* (221) cmd ::= SHOW CONNECTIONS */ - { 322, -2 }, /* (222) cmd ::= SHOW LICENCES */ - { 322, -2 }, /* (223) cmd ::= SHOW GRANTS */ - { 322, -4 }, /* (224) cmd ::= SHOW CREATE DATABASE db_name */ - { 322, -4 }, /* (225) cmd ::= SHOW CREATE TABLE full_table_name */ - { 322, -4 }, /* (226) cmd ::= SHOW CREATE STABLE full_table_name */ - { 322, -2 }, /* (227) cmd ::= SHOW QUERIES */ - { 322, -2 }, /* (228) cmd ::= SHOW SCORES */ - { 322, -2 }, /* (229) cmd ::= SHOW TOPICS */ - { 322, -2 }, /* (230) cmd ::= SHOW VARIABLES */ - { 322, -3 }, /* (231) cmd ::= SHOW CLUSTER VARIABLES */ - { 322, -3 }, /* (232) cmd ::= SHOW LOCAL VARIABLES */ - { 322, -5 }, /* (233) cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ - { 322, -2 }, /* (234) cmd ::= SHOW BNODES */ - { 322, -2 }, /* (235) cmd ::= SHOW SNODES */ - { 322, -2 }, /* (236) cmd ::= SHOW CLUSTER */ - { 322, -2 }, /* (237) cmd ::= SHOW TRANSACTIONS */ - { 322, -4 }, /* (238) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ - { 322, -2 }, /* (239) cmd ::= SHOW CONSUMERS */ - { 322, -2 }, /* (240) cmd ::= SHOW SUBSCRIPTIONS */ - { 322, -5 }, /* (241) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ - { 322, -7 }, /* (242) cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ - { 322, -3 }, /* (243) cmd ::= SHOW VNODES NK_INTEGER */ - { 322, -3 }, /* (244) cmd ::= SHOW VNODES NK_STRING */ - { 373, 0 }, /* (245) db_name_cond_opt ::= */ - { 373, -2 }, /* (246) db_name_cond_opt ::= db_name NK_DOT */ - { 374, 0 }, /* (247) like_pattern_opt ::= */ - { 374, -2 }, /* (248) like_pattern_opt ::= LIKE NK_STRING */ - { 375, -1 }, /* (249) table_name_cond ::= table_name */ - { 376, 0 }, /* (250) from_db_opt ::= */ - { 376, -2 }, /* (251) from_db_opt ::= FROM db_name */ - { 377, 0 }, /* (252) tag_list_opt ::= */ - { 377, -1 }, /* (253) tag_list_opt ::= tag_item */ - { 377, -3 }, /* (254) tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ - { 378, -1 }, /* (255) tag_item ::= TBNAME */ - { 378, -1 }, /* (256) tag_item ::= QTAGS */ - { 378, -1 }, /* (257) tag_item ::= column_name */ - { 378, -2 }, /* (258) tag_item ::= column_name column_alias */ - { 378, -3 }, /* (259) tag_item ::= column_name AS column_alias */ - { 322, -8 }, /* (260) cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ - { 322, -4 }, /* (261) cmd ::= DROP INDEX exists_opt full_index_name */ - { 380, -1 }, /* (262) full_index_name ::= index_name */ - { 380, -3 }, /* (263) full_index_name ::= db_name NK_DOT index_name */ - { 381, -10 }, /* (264) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ - { 381, -12 }, /* (265) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ - { 383, -1 }, /* (266) func_list ::= func */ - { 383, -3 }, /* (267) func_list ::= func_list NK_COMMA func */ - { 386, -4 }, /* (268) func ::= function_name NK_LP expression_list NK_RP */ - { 385, 0 }, /* (269) sma_stream_opt ::= */ - { 385, -3 }, /* (270) sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ - { 385, -3 }, /* (271) sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ - { 385, -3 }, /* (272) sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ - { 322, -6 }, /* (273) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ - { 322, -7 }, /* (274) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ - { 322, -9 }, /* (275) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ - { 322, -7 }, /* (276) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ - { 322, -9 }, /* (277) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ - { 322, -4 }, /* (278) cmd ::= DROP TOPIC exists_opt topic_name */ - { 322, -7 }, /* (279) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ - { 322, -2 }, /* (280) cmd ::= DESC full_table_name */ - { 322, -2 }, /* (281) cmd ::= DESCRIBE full_table_name */ - { 322, -3 }, /* (282) cmd ::= RESET QUERY CACHE */ - { 322, -4 }, /* (283) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ - { 389, 0 }, /* (284) analyze_opt ::= */ - { 389, -1 }, /* (285) analyze_opt ::= ANALYZE */ - { 390, 0 }, /* (286) explain_options ::= */ - { 390, -3 }, /* (287) explain_options ::= explain_options VERBOSE NK_BOOL */ - { 390, -3 }, /* (288) explain_options ::= explain_options RATIO NK_FLOAT */ - { 322, -10 }, /* (289) cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ - { 322, -4 }, /* (290) cmd ::= DROP FUNCTION exists_opt function_name */ - { 391, 0 }, /* (291) agg_func_opt ::= */ - { 391, -1 }, /* (292) agg_func_opt ::= AGGREGATE */ - { 392, 0 }, /* (293) bufsize_opt ::= */ - { 392, -2 }, /* (294) bufsize_opt ::= BUFSIZE NK_INTEGER */ - { 322, -11 }, /* (295) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name tags_def_opt subtable_opt AS query_or_subquery */ - { 322, -4 }, /* (296) cmd ::= DROP STREAM exists_opt stream_name */ - { 394, 0 }, /* (297) stream_options ::= */ - { 394, -3 }, /* (298) stream_options ::= stream_options TRIGGER AT_ONCE */ - { 394, -3 }, /* (299) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ - { 394, -4 }, /* (300) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ - { 394, -3 }, /* (301) stream_options ::= stream_options WATERMARK duration_literal */ - { 394, -4 }, /* (302) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ - { 394, -3 }, /* (303) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ - { 395, 0 }, /* (304) subtable_opt ::= */ - { 395, -4 }, /* (305) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ - { 322, -3 }, /* (306) cmd ::= KILL CONNECTION NK_INTEGER */ - { 322, -3 }, /* (307) cmd ::= KILL QUERY NK_STRING */ - { 322, -3 }, /* (308) cmd ::= KILL TRANSACTION NK_INTEGER */ - { 322, -2 }, /* (309) cmd ::= BALANCE VGROUP */ - { 322, -4 }, /* (310) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ - { 322, -4 }, /* (311) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ - { 322, -3 }, /* (312) cmd ::= SPLIT VGROUP NK_INTEGER */ - { 397, -2 }, /* (313) dnode_list ::= DNODE NK_INTEGER */ - { 397, -3 }, /* (314) dnode_list ::= dnode_list DNODE NK_INTEGER */ - { 322, -4 }, /* (315) cmd ::= DELETE FROM full_table_name where_clause_opt */ - { 322, -1 }, /* (316) cmd ::= query_or_subquery */ - { 322, -7 }, /* (317) cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ - { 322, -4 }, /* (318) cmd ::= INSERT INTO full_table_name query_or_subquery */ - { 325, -1 }, /* (319) literal ::= NK_INTEGER */ - { 325, -1 }, /* (320) literal ::= NK_FLOAT */ - { 325, -1 }, /* (321) literal ::= NK_STRING */ - { 325, -1 }, /* (322) literal ::= NK_BOOL */ - { 325, -2 }, /* (323) literal ::= TIMESTAMP NK_STRING */ - { 325, -1 }, /* (324) literal ::= duration_literal */ - { 325, -1 }, /* (325) literal ::= NULL */ - { 325, -1 }, /* (326) literal ::= NK_QUESTION */ - { 369, -1 }, /* (327) duration_literal ::= NK_VARIABLE */ - { 399, -1 }, /* (328) signed ::= NK_INTEGER */ - { 399, -2 }, /* (329) signed ::= NK_PLUS NK_INTEGER */ - { 399, -2 }, /* (330) signed ::= NK_MINUS NK_INTEGER */ - { 399, -1 }, /* (331) signed ::= NK_FLOAT */ - { 399, -2 }, /* (332) signed ::= NK_PLUS NK_FLOAT */ - { 399, -2 }, /* (333) signed ::= NK_MINUS NK_FLOAT */ - { 358, -1 }, /* (334) signed_literal ::= signed */ - { 358, -1 }, /* (335) signed_literal ::= NK_STRING */ - { 358, -1 }, /* (336) signed_literal ::= NK_BOOL */ - { 358, -2 }, /* (337) signed_literal ::= TIMESTAMP NK_STRING */ - { 358, -1 }, /* (338) signed_literal ::= duration_literal */ - { 358, -1 }, /* (339) signed_literal ::= NULL */ - { 358, -1 }, /* (340) signed_literal ::= literal_func */ - { 358, -1 }, /* (341) signed_literal ::= NK_QUESTION */ - { 401, -1 }, /* (342) literal_list ::= signed_literal */ - { 401, -3 }, /* (343) literal_list ::= literal_list NK_COMMA signed_literal */ - { 333, -1 }, /* (344) db_name ::= NK_ID */ - { 364, -1 }, /* (345) table_name ::= NK_ID */ - { 356, -1 }, /* (346) column_name ::= NK_ID */ - { 371, -1 }, /* (347) function_name ::= NK_ID */ - { 402, -1 }, /* (348) table_alias ::= NK_ID */ - { 379, -1 }, /* (349) column_alias ::= NK_ID */ - { 327, -1 }, /* (350) user_name ::= NK_ID */ - { 334, -1 }, /* (351) topic_name ::= NK_ID */ - { 393, -1 }, /* (352) stream_name ::= NK_ID */ - { 388, -1 }, /* (353) cgroup_name ::= NK_ID */ - { 382, -1 }, /* (354) index_name ::= NK_ID */ - { 403, -1 }, /* (355) expr_or_subquery ::= expression */ - { 396, -1 }, /* (356) expression ::= literal */ - { 396, -1 }, /* (357) expression ::= pseudo_column */ - { 396, -1 }, /* (358) expression ::= column_reference */ - { 396, -1 }, /* (359) expression ::= function_expression */ - { 396, -1 }, /* (360) expression ::= case_when_expression */ - { 396, -3 }, /* (361) expression ::= NK_LP expression NK_RP */ - { 396, -2 }, /* (362) expression ::= NK_PLUS expr_or_subquery */ - { 396, -2 }, /* (363) expression ::= NK_MINUS expr_or_subquery */ - { 396, -3 }, /* (364) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ - { 396, -3 }, /* (365) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ - { 396, -3 }, /* (366) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ - { 396, -3 }, /* (367) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ - { 396, -3 }, /* (368) expression ::= expr_or_subquery NK_REM expr_or_subquery */ - { 396, -3 }, /* (369) expression ::= column_reference NK_ARROW NK_STRING */ - { 396, -3 }, /* (370) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ - { 396, -3 }, /* (371) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ - { 361, -1 }, /* (372) expression_list ::= expr_or_subquery */ - { 361, -3 }, /* (373) expression_list ::= expression_list NK_COMMA expr_or_subquery */ - { 405, -1 }, /* (374) column_reference ::= column_name */ - { 405, -3 }, /* (375) column_reference ::= table_name NK_DOT column_name */ - { 404, -1 }, /* (376) pseudo_column ::= ROWTS */ - { 404, -1 }, /* (377) pseudo_column ::= TBNAME */ - { 404, -3 }, /* (378) pseudo_column ::= table_name NK_DOT TBNAME */ - { 404, -1 }, /* (379) pseudo_column ::= QSTART */ - { 404, -1 }, /* (380) pseudo_column ::= QEND */ - { 404, -1 }, /* (381) pseudo_column ::= QDURATION */ - { 404, -1 }, /* (382) pseudo_column ::= WSTART */ - { 404, -1 }, /* (383) pseudo_column ::= WEND */ - { 404, -1 }, /* (384) pseudo_column ::= WDURATION */ - { 404, -1 }, /* (385) pseudo_column ::= IROWTS */ - { 404, -1 }, /* (386) pseudo_column ::= QTAGS */ - { 406, -4 }, /* (387) function_expression ::= function_name NK_LP expression_list NK_RP */ - { 406, -4 }, /* (388) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ - { 406, -6 }, /* (389) function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ - { 406, -1 }, /* (390) function_expression ::= literal_func */ - { 400, -3 }, /* (391) literal_func ::= noarg_func NK_LP NK_RP */ - { 400, -1 }, /* (392) literal_func ::= NOW */ - { 410, -1 }, /* (393) noarg_func ::= NOW */ - { 410, -1 }, /* (394) noarg_func ::= TODAY */ - { 410, -1 }, /* (395) noarg_func ::= TIMEZONE */ - { 410, -1 }, /* (396) noarg_func ::= DATABASE */ - { 410, -1 }, /* (397) noarg_func ::= CLIENT_VERSION */ - { 410, -1 }, /* (398) noarg_func ::= SERVER_VERSION */ - { 410, -1 }, /* (399) noarg_func ::= SERVER_STATUS */ - { 410, -1 }, /* (400) noarg_func ::= CURRENT_USER */ - { 410, -1 }, /* (401) noarg_func ::= USER */ - { 408, -1 }, /* (402) star_func ::= COUNT */ - { 408, -1 }, /* (403) star_func ::= FIRST */ - { 408, -1 }, /* (404) star_func ::= LAST */ - { 408, -1 }, /* (405) star_func ::= LAST_ROW */ - { 409, -1 }, /* (406) star_func_para_list ::= NK_STAR */ - { 409, -1 }, /* (407) star_func_para_list ::= other_para_list */ - { 411, -1 }, /* (408) other_para_list ::= star_func_para */ - { 411, -3 }, /* (409) other_para_list ::= other_para_list NK_COMMA star_func_para */ - { 412, -1 }, /* (410) star_func_para ::= expr_or_subquery */ - { 412, -3 }, /* (411) star_func_para ::= table_name NK_DOT NK_STAR */ - { 407, -4 }, /* (412) case_when_expression ::= CASE when_then_list case_when_else_opt END */ - { 407, -5 }, /* (413) case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ - { 413, -1 }, /* (414) when_then_list ::= when_then_expr */ - { 413, -2 }, /* (415) when_then_list ::= when_then_list when_then_expr */ - { 416, -4 }, /* (416) when_then_expr ::= WHEN common_expression THEN common_expression */ - { 414, 0 }, /* (417) case_when_else_opt ::= */ - { 414, -2 }, /* (418) case_when_else_opt ::= ELSE common_expression */ - { 417, -3 }, /* (419) predicate ::= expr_or_subquery compare_op expr_or_subquery */ - { 417, -5 }, /* (420) predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ - { 417, -6 }, /* (421) predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ - { 417, -3 }, /* (422) predicate ::= expr_or_subquery IS NULL */ - { 417, -4 }, /* (423) predicate ::= expr_or_subquery IS NOT NULL */ - { 417, -3 }, /* (424) predicate ::= expr_or_subquery in_op in_predicate_value */ - { 418, -1 }, /* (425) compare_op ::= NK_LT */ - { 418, -1 }, /* (426) compare_op ::= NK_GT */ - { 418, -1 }, /* (427) compare_op ::= NK_LE */ - { 418, -1 }, /* (428) compare_op ::= NK_GE */ - { 418, -1 }, /* (429) compare_op ::= NK_NE */ - { 418, -1 }, /* (430) compare_op ::= NK_EQ */ - { 418, -1 }, /* (431) compare_op ::= LIKE */ - { 418, -2 }, /* (432) compare_op ::= NOT LIKE */ - { 418, -1 }, /* (433) compare_op ::= MATCH */ - { 418, -1 }, /* (434) compare_op ::= NMATCH */ - { 418, -1 }, /* (435) compare_op ::= CONTAINS */ - { 419, -1 }, /* (436) in_op ::= IN */ - { 419, -2 }, /* (437) in_op ::= NOT IN */ - { 420, -3 }, /* (438) in_predicate_value ::= NK_LP literal_list NK_RP */ - { 421, -1 }, /* (439) boolean_value_expression ::= boolean_primary */ - { 421, -2 }, /* (440) boolean_value_expression ::= NOT boolean_primary */ - { 421, -3 }, /* (441) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ - { 421, -3 }, /* (442) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ - { 422, -1 }, /* (443) boolean_primary ::= predicate */ - { 422, -3 }, /* (444) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - { 415, -1 }, /* (445) common_expression ::= expr_or_subquery */ - { 415, -1 }, /* (446) common_expression ::= boolean_value_expression */ - { 423, 0 }, /* (447) from_clause_opt ::= */ - { 423, -2 }, /* (448) from_clause_opt ::= FROM table_reference_list */ - { 424, -1 }, /* (449) table_reference_list ::= table_reference */ - { 424, -3 }, /* (450) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - { 425, -1 }, /* (451) table_reference ::= table_primary */ - { 425, -1 }, /* (452) table_reference ::= joined_table */ - { 426, -2 }, /* (453) table_primary ::= table_name alias_opt */ - { 426, -4 }, /* (454) table_primary ::= db_name NK_DOT table_name alias_opt */ - { 426, -2 }, /* (455) table_primary ::= subquery alias_opt */ - { 426, -1 }, /* (456) table_primary ::= parenthesized_joined_table */ - { 428, 0 }, /* (457) alias_opt ::= */ - { 428, -1 }, /* (458) alias_opt ::= table_alias */ - { 428, -2 }, /* (459) alias_opt ::= AS table_alias */ - { 430, -3 }, /* (460) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - { 430, -3 }, /* (461) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - { 427, -6 }, /* (462) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - { 431, 0 }, /* (463) join_type ::= */ - { 431, -1 }, /* (464) join_type ::= INNER */ - { 433, -12 }, /* (465) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - { 434, 0 }, /* (466) set_quantifier_opt ::= */ - { 434, -1 }, /* (467) set_quantifier_opt ::= DISTINCT */ - { 434, -1 }, /* (468) set_quantifier_opt ::= ALL */ - { 435, -1 }, /* (469) select_list ::= select_item */ - { 435, -3 }, /* (470) select_list ::= select_list NK_COMMA select_item */ - { 443, -1 }, /* (471) select_item ::= NK_STAR */ - { 443, -1 }, /* (472) select_item ::= common_expression */ - { 443, -2 }, /* (473) select_item ::= common_expression column_alias */ - { 443, -3 }, /* (474) select_item ::= common_expression AS column_alias */ - { 443, -3 }, /* (475) select_item ::= table_name NK_DOT NK_STAR */ - { 398, 0 }, /* (476) where_clause_opt ::= */ - { 398, -2 }, /* (477) where_clause_opt ::= WHERE search_condition */ - { 436, 0 }, /* (478) partition_by_clause_opt ::= */ - { 436, -3 }, /* (479) partition_by_clause_opt ::= PARTITION BY partition_list */ - { 444, -1 }, /* (480) partition_list ::= partition_item */ - { 444, -3 }, /* (481) partition_list ::= partition_list NK_COMMA partition_item */ - { 445, -1 }, /* (482) partition_item ::= expr_or_subquery */ - { 445, -2 }, /* (483) partition_item ::= expr_or_subquery column_alias */ - { 445, -3 }, /* (484) partition_item ::= expr_or_subquery AS column_alias */ - { 440, 0 }, /* (485) twindow_clause_opt ::= */ - { 440, -6 }, /* (486) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ - { 440, -4 }, /* (487) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ - { 440, -6 }, /* (488) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - { 440, -8 }, /* (489) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - { 384, 0 }, /* (490) sliding_opt ::= */ - { 384, -4 }, /* (491) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - { 439, 0 }, /* (492) fill_opt ::= */ - { 439, -4 }, /* (493) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - { 439, -6 }, /* (494) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ - { 446, -1 }, /* (495) fill_mode ::= NONE */ - { 446, -1 }, /* (496) fill_mode ::= PREV */ - { 446, -1 }, /* (497) fill_mode ::= NULL */ - { 446, -1 }, /* (498) fill_mode ::= LINEAR */ - { 446, -1 }, /* (499) fill_mode ::= NEXT */ - { 441, 0 }, /* (500) group_by_clause_opt ::= */ - { 441, -3 }, /* (501) group_by_clause_opt ::= GROUP BY group_by_list */ - { 447, -1 }, /* (502) group_by_list ::= expr_or_subquery */ - { 447, -3 }, /* (503) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ - { 442, 0 }, /* (504) having_clause_opt ::= */ - { 442, -2 }, /* (505) having_clause_opt ::= HAVING search_condition */ - { 437, 0 }, /* (506) range_opt ::= */ - { 437, -6 }, /* (507) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ - { 438, 0 }, /* (508) every_opt ::= */ - { 438, -4 }, /* (509) every_opt ::= EVERY NK_LP duration_literal NK_RP */ - { 448, -4 }, /* (510) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 449, -1 }, /* (511) query_simple ::= query_specification */ - { 449, -1 }, /* (512) query_simple ::= union_query_expression */ - { 453, -4 }, /* (513) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ - { 453, -3 }, /* (514) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ - { 454, -1 }, /* (515) query_simple_or_subquery ::= query_simple */ - { 454, -1 }, /* (516) query_simple_or_subquery ::= subquery */ - { 387, -1 }, /* (517) query_or_subquery ::= query_expression */ - { 387, -1 }, /* (518) query_or_subquery ::= subquery */ - { 450, 0 }, /* (519) order_by_clause_opt ::= */ - { 450, -3 }, /* (520) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 451, 0 }, /* (521) slimit_clause_opt ::= */ - { 451, -2 }, /* (522) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - { 451, -4 }, /* (523) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 451, -4 }, /* (524) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 452, 0 }, /* (525) limit_clause_opt ::= */ - { 452, -2 }, /* (526) limit_clause_opt ::= LIMIT NK_INTEGER */ - { 452, -4 }, /* (527) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 452, -4 }, /* (528) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 429, -3 }, /* (529) subquery ::= NK_LP query_expression NK_RP */ - { 429, -3 }, /* (530) subquery ::= NK_LP subquery NK_RP */ - { 432, -1 }, /* (531) search_condition ::= common_expression */ - { 455, -1 }, /* (532) sort_specification_list ::= sort_specification */ - { 455, -3 }, /* (533) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 456, -3 }, /* (534) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ - { 457, 0 }, /* (535) ordering_specification_opt ::= */ - { 457, -1 }, /* (536) ordering_specification_opt ::= ASC */ - { 457, -1 }, /* (537) ordering_specification_opt ::= DESC */ - { 458, 0 }, /* (538) null_ordering_opt ::= */ - { 458, -2 }, /* (539) null_ordering_opt ::= NULLS FIRST */ - { 458, -2 }, /* (540) null_ordering_opt ::= NULLS LAST */ + { 326, -6 }, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ + { 326, -4 }, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ + { 327, 0 }, /* (2) account_options ::= */ + { 327, -3 }, /* (3) account_options ::= account_options PPS literal */ + { 327, -3 }, /* (4) account_options ::= account_options TSERIES literal */ + { 327, -3 }, /* (5) account_options ::= account_options STORAGE literal */ + { 327, -3 }, /* (6) account_options ::= account_options STREAMS literal */ + { 327, -3 }, /* (7) account_options ::= account_options QTIME literal */ + { 327, -3 }, /* (8) account_options ::= account_options DBS literal */ + { 327, -3 }, /* (9) account_options ::= account_options USERS literal */ + { 327, -3 }, /* (10) account_options ::= account_options CONNS literal */ + { 327, -3 }, /* (11) account_options ::= account_options STATE literal */ + { 328, -1 }, /* (12) alter_account_options ::= alter_account_option */ + { 328, -2 }, /* (13) alter_account_options ::= alter_account_options alter_account_option */ + { 330, -2 }, /* (14) alter_account_option ::= PASS literal */ + { 330, -2 }, /* (15) alter_account_option ::= PPS literal */ + { 330, -2 }, /* (16) alter_account_option ::= TSERIES literal */ + { 330, -2 }, /* (17) alter_account_option ::= STORAGE literal */ + { 330, -2 }, /* (18) alter_account_option ::= STREAMS literal */ + { 330, -2 }, /* (19) alter_account_option ::= QTIME literal */ + { 330, -2 }, /* (20) alter_account_option ::= DBS literal */ + { 330, -2 }, /* (21) alter_account_option ::= USERS literal */ + { 330, -2 }, /* (22) alter_account_option ::= CONNS literal */ + { 330, -2 }, /* (23) alter_account_option ::= STATE literal */ + { 326, -6 }, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ + { 326, -5 }, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ + { 326, -5 }, /* (26) cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ + { 326, -5 }, /* (27) cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ + { 326, -3 }, /* (28) cmd ::= DROP USER user_name */ + { 332, 0 }, /* (29) sysinfo_opt ::= */ + { 332, -2 }, /* (30) sysinfo_opt ::= SYSINFO NK_INTEGER */ + { 326, -6 }, /* (31) cmd ::= GRANT privileges ON priv_level TO user_name */ + { 326, -6 }, /* (32) cmd ::= REVOKE privileges ON priv_level FROM user_name */ + { 333, -1 }, /* (33) privileges ::= ALL */ + { 333, -1 }, /* (34) privileges ::= priv_type_list */ + { 333, -1 }, /* (35) privileges ::= SUBSCRIBE */ + { 335, -1 }, /* (36) priv_type_list ::= priv_type */ + { 335, -3 }, /* (37) priv_type_list ::= priv_type_list NK_COMMA priv_type */ + { 336, -1 }, /* (38) priv_type ::= READ */ + { 336, -1 }, /* (39) priv_type ::= WRITE */ + { 334, -3 }, /* (40) priv_level ::= NK_STAR NK_DOT NK_STAR */ + { 334, -3 }, /* (41) priv_level ::= db_name NK_DOT NK_STAR */ + { 334, -1 }, /* (42) priv_level ::= topic_name */ + { 326, -3 }, /* (43) cmd ::= CREATE DNODE dnode_endpoint */ + { 326, -5 }, /* (44) cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ + { 326, -4 }, /* (45) cmd ::= DROP DNODE NK_INTEGER force_opt */ + { 326, -4 }, /* (46) cmd ::= DROP DNODE dnode_endpoint force_opt */ + { 326, -4 }, /* (47) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ + { 326, -5 }, /* (48) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ + { 326, -4 }, /* (49) cmd ::= ALTER ALL DNODES NK_STRING */ + { 326, -5 }, /* (50) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ + { 339, -1 }, /* (51) dnode_endpoint ::= NK_STRING */ + { 339, -1 }, /* (52) dnode_endpoint ::= NK_ID */ + { 339, -1 }, /* (53) dnode_endpoint ::= NK_IPTOKEN */ + { 340, 0 }, /* (54) force_opt ::= */ + { 340, -1 }, /* (55) force_opt ::= FORCE */ + { 326, -3 }, /* (56) cmd ::= ALTER LOCAL NK_STRING */ + { 326, -4 }, /* (57) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ + { 326, -5 }, /* (58) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ + { 326, -5 }, /* (59) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ + { 326, -5 }, /* (60) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ + { 326, -5 }, /* (61) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ + { 326, -5 }, /* (62) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ + { 326, -5 }, /* (63) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ + { 326, -5 }, /* (64) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ + { 326, -5 }, /* (65) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ + { 326, -5 }, /* (66) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ + { 326, -4 }, /* (67) cmd ::= DROP DATABASE exists_opt db_name */ + { 326, -2 }, /* (68) cmd ::= USE db_name */ + { 326, -4 }, /* (69) cmd ::= ALTER DATABASE db_name alter_db_options */ + { 326, -3 }, /* (70) cmd ::= FLUSH DATABASE db_name */ + { 326, -4 }, /* (71) cmd ::= TRIM DATABASE db_name speed_opt */ + { 341, -3 }, /* (72) not_exists_opt ::= IF NOT EXISTS */ + { 341, 0 }, /* (73) not_exists_opt ::= */ + { 343, -2 }, /* (74) exists_opt ::= IF EXISTS */ + { 343, 0 }, /* (75) exists_opt ::= */ + { 342, 0 }, /* (76) db_options ::= */ + { 342, -3 }, /* (77) db_options ::= db_options BUFFER NK_INTEGER */ + { 342, -3 }, /* (78) db_options ::= db_options CACHEMODEL NK_STRING */ + { 342, -3 }, /* (79) db_options ::= db_options CACHESIZE NK_INTEGER */ + { 342, -3 }, /* (80) db_options ::= db_options COMP NK_INTEGER */ + { 342, -3 }, /* (81) db_options ::= db_options DURATION NK_INTEGER */ + { 342, -3 }, /* (82) db_options ::= db_options DURATION NK_VARIABLE */ + { 342, -3 }, /* (83) db_options ::= db_options MAXROWS NK_INTEGER */ + { 342, -3 }, /* (84) db_options ::= db_options MINROWS NK_INTEGER */ + { 342, -3 }, /* (85) db_options ::= db_options KEEP integer_list */ + { 342, -3 }, /* (86) db_options ::= db_options KEEP variable_list */ + { 342, -3 }, /* (87) db_options ::= db_options PAGES NK_INTEGER */ + { 342, -3 }, /* (88) db_options ::= db_options PAGESIZE NK_INTEGER */ + { 342, -3 }, /* (89) db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ + { 342, -3 }, /* (90) db_options ::= db_options PRECISION NK_STRING */ + { 342, -3 }, /* (91) db_options ::= db_options REPLICA NK_INTEGER */ + { 342, -3 }, /* (92) db_options ::= db_options VGROUPS NK_INTEGER */ + { 342, -3 }, /* (93) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ + { 342, -3 }, /* (94) db_options ::= db_options RETENTIONS retention_list */ + { 342, -3 }, /* (95) db_options ::= db_options SCHEMALESS NK_INTEGER */ + { 342, -3 }, /* (96) db_options ::= db_options WAL_LEVEL NK_INTEGER */ + { 342, -3 }, /* (97) db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ + { 342, -3 }, /* (98) db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ + { 342, -4 }, /* (99) db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ + { 342, -3 }, /* (100) db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ + { 342, -4 }, /* (101) db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ + { 342, -3 }, /* (102) db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ + { 342, -3 }, /* (103) db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ + { 342, -3 }, /* (104) db_options ::= db_options STT_TRIGGER NK_INTEGER */ + { 342, -3 }, /* (105) db_options ::= db_options TABLE_PREFIX NK_INTEGER */ + { 342, -3 }, /* (106) db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ + { 344, -1 }, /* (107) alter_db_options ::= alter_db_option */ + { 344, -2 }, /* (108) alter_db_options ::= alter_db_options alter_db_option */ + { 349, -2 }, /* (109) alter_db_option ::= BUFFER NK_INTEGER */ + { 349, -2 }, /* (110) alter_db_option ::= CACHEMODEL NK_STRING */ + { 349, -2 }, /* (111) alter_db_option ::= CACHESIZE NK_INTEGER */ + { 349, -2 }, /* (112) alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ + { 349, -2 }, /* (113) alter_db_option ::= KEEP integer_list */ + { 349, -2 }, /* (114) alter_db_option ::= KEEP variable_list */ + { 349, -2 }, /* (115) alter_db_option ::= PAGES NK_INTEGER */ + { 349, -2 }, /* (116) alter_db_option ::= REPLICA NK_INTEGER */ + { 349, -2 }, /* (117) alter_db_option ::= WAL_LEVEL NK_INTEGER */ + { 349, -2 }, /* (118) alter_db_option ::= STT_TRIGGER NK_INTEGER */ + { 346, -1 }, /* (119) integer_list ::= NK_INTEGER */ + { 346, -3 }, /* (120) integer_list ::= integer_list NK_COMMA NK_INTEGER */ + { 347, -1 }, /* (121) variable_list ::= NK_VARIABLE */ + { 347, -3 }, /* (122) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ + { 348, -1 }, /* (123) retention_list ::= retention */ + { 348, -3 }, /* (124) retention_list ::= retention_list NK_COMMA retention */ + { 350, -3 }, /* (125) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ + { 345, 0 }, /* (126) speed_opt ::= */ + { 345, -2 }, /* (127) speed_opt ::= MAX_SPEED NK_INTEGER */ + { 326, -9 }, /* (128) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + { 326, -3 }, /* (129) cmd ::= CREATE TABLE multi_create_clause */ + { 326, -9 }, /* (130) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ + { 326, -3 }, /* (131) cmd ::= DROP TABLE multi_drop_clause */ + { 326, -4 }, /* (132) cmd ::= DROP STABLE exists_opt full_table_name */ + { 326, -3 }, /* (133) cmd ::= ALTER TABLE alter_table_clause */ + { 326, -3 }, /* (134) cmd ::= ALTER STABLE alter_table_clause */ + { 358, -2 }, /* (135) alter_table_clause ::= full_table_name alter_table_options */ + { 358, -5 }, /* (136) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ + { 358, -4 }, /* (137) alter_table_clause ::= full_table_name DROP COLUMN column_name */ + { 358, -5 }, /* (138) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ + { 358, -5 }, /* (139) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ + { 358, -5 }, /* (140) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ + { 358, -4 }, /* (141) alter_table_clause ::= full_table_name DROP TAG column_name */ + { 358, -5 }, /* (142) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ + { 358, -5 }, /* (143) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ + { 358, -6 }, /* (144) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ + { 355, -1 }, /* (145) multi_create_clause ::= create_subtable_clause */ + { 355, -2 }, /* (146) multi_create_clause ::= multi_create_clause create_subtable_clause */ + { 363, -10 }, /* (147) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ + { 357, -1 }, /* (148) multi_drop_clause ::= drop_table_clause */ + { 357, -2 }, /* (149) multi_drop_clause ::= multi_drop_clause drop_table_clause */ + { 366, -2 }, /* (150) drop_table_clause ::= exists_opt full_table_name */ + { 364, 0 }, /* (151) specific_cols_opt ::= */ + { 364, -3 }, /* (152) specific_cols_opt ::= NK_LP col_name_list NK_RP */ + { 351, -1 }, /* (153) full_table_name ::= table_name */ + { 351, -3 }, /* (154) full_table_name ::= db_name NK_DOT table_name */ + { 352, -1 }, /* (155) column_def_list ::= column_def */ + { 352, -3 }, /* (156) column_def_list ::= column_def_list NK_COMMA column_def */ + { 369, -2 }, /* (157) column_def ::= column_name type_name */ + { 369, -4 }, /* (158) column_def ::= column_name type_name COMMENT NK_STRING */ + { 361, -1 }, /* (159) type_name ::= BOOL */ + { 361, -1 }, /* (160) type_name ::= TINYINT */ + { 361, -1 }, /* (161) type_name ::= SMALLINT */ + { 361, -1 }, /* (162) type_name ::= INT */ + { 361, -1 }, /* (163) type_name ::= INTEGER */ + { 361, -1 }, /* (164) type_name ::= BIGINT */ + { 361, -1 }, /* (165) type_name ::= FLOAT */ + { 361, -1 }, /* (166) type_name ::= DOUBLE */ + { 361, -4 }, /* (167) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ + { 361, -1 }, /* (168) type_name ::= TIMESTAMP */ + { 361, -4 }, /* (169) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ + { 361, -2 }, /* (170) type_name ::= TINYINT UNSIGNED */ + { 361, -2 }, /* (171) type_name ::= SMALLINT UNSIGNED */ + { 361, -2 }, /* (172) type_name ::= INT UNSIGNED */ + { 361, -2 }, /* (173) type_name ::= BIGINT UNSIGNED */ + { 361, -1 }, /* (174) type_name ::= JSON */ + { 361, -4 }, /* (175) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ + { 361, -1 }, /* (176) type_name ::= MEDIUMBLOB */ + { 361, -1 }, /* (177) type_name ::= BLOB */ + { 361, -4 }, /* (178) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ + { 361, -1 }, /* (179) type_name ::= DECIMAL */ + { 361, -4 }, /* (180) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ + { 361, -6 }, /* (181) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ + { 353, 0 }, /* (182) tags_def_opt ::= */ + { 353, -1 }, /* (183) tags_def_opt ::= tags_def */ + { 356, -4 }, /* (184) tags_def ::= TAGS NK_LP column_def_list NK_RP */ + { 354, 0 }, /* (185) table_options ::= */ + { 354, -3 }, /* (186) table_options ::= table_options COMMENT NK_STRING */ + { 354, -3 }, /* (187) table_options ::= table_options MAX_DELAY duration_list */ + { 354, -3 }, /* (188) table_options ::= table_options WATERMARK duration_list */ + { 354, -5 }, /* (189) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ + { 354, -3 }, /* (190) table_options ::= table_options TTL NK_INTEGER */ + { 354, -5 }, /* (191) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ + { 354, -3 }, /* (192) table_options ::= table_options DELETE_MARK duration_list */ + { 359, -1 }, /* (193) alter_table_options ::= alter_table_option */ + { 359, -2 }, /* (194) alter_table_options ::= alter_table_options alter_table_option */ + { 372, -2 }, /* (195) alter_table_option ::= COMMENT NK_STRING */ + { 372, -2 }, /* (196) alter_table_option ::= TTL NK_INTEGER */ + { 370, -1 }, /* (197) duration_list ::= duration_literal */ + { 370, -3 }, /* (198) duration_list ::= duration_list NK_COMMA duration_literal */ + { 371, -1 }, /* (199) rollup_func_list ::= rollup_func_name */ + { 371, -3 }, /* (200) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ + { 374, -1 }, /* (201) rollup_func_name ::= function_name */ + { 374, -1 }, /* (202) rollup_func_name ::= FIRST */ + { 374, -1 }, /* (203) rollup_func_name ::= LAST */ + { 367, -1 }, /* (204) col_name_list ::= col_name */ + { 367, -3 }, /* (205) col_name_list ::= col_name_list NK_COMMA col_name */ + { 376, -1 }, /* (206) col_name ::= column_name */ + { 326, -2 }, /* (207) cmd ::= SHOW DNODES */ + { 326, -2 }, /* (208) cmd ::= SHOW USERS */ + { 326, -3 }, /* (209) cmd ::= SHOW USER PRIVILEGES */ + { 326, -2 }, /* (210) cmd ::= SHOW DATABASES */ + { 326, -4 }, /* (211) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ + { 326, -4 }, /* (212) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ + { 326, -3 }, /* (213) cmd ::= SHOW db_name_cond_opt VGROUPS */ + { 326, -2 }, /* (214) cmd ::= SHOW MNODES */ + { 326, -2 }, /* (215) cmd ::= SHOW QNODES */ + { 326, -2 }, /* (216) cmd ::= SHOW FUNCTIONS */ + { 326, -5 }, /* (217) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ + { 326, -2 }, /* (218) cmd ::= SHOW STREAMS */ + { 326, -2 }, /* (219) cmd ::= SHOW ACCOUNTS */ + { 326, -2 }, /* (220) cmd ::= SHOW APPS */ + { 326, -2 }, /* (221) cmd ::= SHOW CONNECTIONS */ + { 326, -2 }, /* (222) cmd ::= SHOW LICENCES */ + { 326, -2 }, /* (223) cmd ::= SHOW GRANTS */ + { 326, -4 }, /* (224) cmd ::= SHOW CREATE DATABASE db_name */ + { 326, -4 }, /* (225) cmd ::= SHOW CREATE TABLE full_table_name */ + { 326, -4 }, /* (226) cmd ::= SHOW CREATE STABLE full_table_name */ + { 326, -2 }, /* (227) cmd ::= SHOW QUERIES */ + { 326, -2 }, /* (228) cmd ::= SHOW SCORES */ + { 326, -2 }, /* (229) cmd ::= SHOW TOPICS */ + { 326, -2 }, /* (230) cmd ::= SHOW VARIABLES */ + { 326, -3 }, /* (231) cmd ::= SHOW CLUSTER VARIABLES */ + { 326, -3 }, /* (232) cmd ::= SHOW LOCAL VARIABLES */ + { 326, -5 }, /* (233) cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ + { 326, -2 }, /* (234) cmd ::= SHOW BNODES */ + { 326, -2 }, /* (235) cmd ::= SHOW SNODES */ + { 326, -2 }, /* (236) cmd ::= SHOW CLUSTER */ + { 326, -2 }, /* (237) cmd ::= SHOW TRANSACTIONS */ + { 326, -4 }, /* (238) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ + { 326, -2 }, /* (239) cmd ::= SHOW CONSUMERS */ + { 326, -2 }, /* (240) cmd ::= SHOW SUBSCRIPTIONS */ + { 326, -5 }, /* (241) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ + { 326, -7 }, /* (242) cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ + { 326, -3 }, /* (243) cmd ::= SHOW VNODES NK_INTEGER */ + { 326, -3 }, /* (244) cmd ::= SHOW VNODES NK_STRING */ + { 326, -3 }, /* (245) cmd ::= SHOW db_name_cond_opt ALIVE */ + { 326, -3 }, /* (246) cmd ::= SHOW CLUSTER ALIVE */ + { 377, 0 }, /* (247) db_name_cond_opt ::= */ + { 377, -2 }, /* (248) db_name_cond_opt ::= db_name NK_DOT */ + { 378, 0 }, /* (249) like_pattern_opt ::= */ + { 378, -2 }, /* (250) like_pattern_opt ::= LIKE NK_STRING */ + { 379, -1 }, /* (251) table_name_cond ::= table_name */ + { 380, 0 }, /* (252) from_db_opt ::= */ + { 380, -2 }, /* (253) from_db_opt ::= FROM db_name */ + { 381, 0 }, /* (254) tag_list_opt ::= */ + { 381, -1 }, /* (255) tag_list_opt ::= tag_item */ + { 381, -3 }, /* (256) tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ + { 382, -1 }, /* (257) tag_item ::= TBNAME */ + { 382, -1 }, /* (258) tag_item ::= QTAGS */ + { 382, -1 }, /* (259) tag_item ::= column_name */ + { 382, -2 }, /* (260) tag_item ::= column_name column_alias */ + { 382, -3 }, /* (261) tag_item ::= column_name AS column_alias */ + { 326, -8 }, /* (262) cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ + { 326, -9 }, /* (263) cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ + { 326, -4 }, /* (264) cmd ::= DROP INDEX exists_opt full_index_name */ + { 384, -1 }, /* (265) full_index_name ::= index_name */ + { 384, -3 }, /* (266) full_index_name ::= db_name NK_DOT index_name */ + { 385, -10 }, /* (267) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ + { 385, -12 }, /* (268) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ + { 387, -1 }, /* (269) func_list ::= func */ + { 387, -3 }, /* (270) func_list ::= func_list NK_COMMA func */ + { 390, -4 }, /* (271) func ::= sma_func_name NK_LP expression_list NK_RP */ + { 391, -1 }, /* (272) sma_func_name ::= function_name */ + { 391, -1 }, /* (273) sma_func_name ::= COUNT */ + { 391, -1 }, /* (274) sma_func_name ::= FIRST */ + { 391, -1 }, /* (275) sma_func_name ::= LAST */ + { 391, -1 }, /* (276) sma_func_name ::= LAST_ROW */ + { 389, 0 }, /* (277) sma_stream_opt ::= */ + { 389, -3 }, /* (278) sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ + { 389, -3 }, /* (279) sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ + { 389, -3 }, /* (280) sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ + { 326, -6 }, /* (281) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ + { 326, -7 }, /* (282) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ + { 326, -9 }, /* (283) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ + { 326, -7 }, /* (284) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ + { 326, -9 }, /* (285) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ + { 326, -4 }, /* (286) cmd ::= DROP TOPIC exists_opt topic_name */ + { 326, -7 }, /* (287) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ + { 326, -2 }, /* (288) cmd ::= DESC full_table_name */ + { 326, -2 }, /* (289) cmd ::= DESCRIBE full_table_name */ + { 326, -3 }, /* (290) cmd ::= RESET QUERY CACHE */ + { 326, -4 }, /* (291) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ + { 394, 0 }, /* (292) analyze_opt ::= */ + { 394, -1 }, /* (293) analyze_opt ::= ANALYZE */ + { 395, 0 }, /* (294) explain_options ::= */ + { 395, -3 }, /* (295) explain_options ::= explain_options VERBOSE NK_BOOL */ + { 395, -3 }, /* (296) explain_options ::= explain_options RATIO NK_FLOAT */ + { 326, -10 }, /* (297) cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ + { 326, -4 }, /* (298) cmd ::= DROP FUNCTION exists_opt function_name */ + { 396, 0 }, /* (299) agg_func_opt ::= */ + { 396, -1 }, /* (300) agg_func_opt ::= AGGREGATE */ + { 397, 0 }, /* (301) bufsize_opt ::= */ + { 397, -2 }, /* (302) bufsize_opt ::= BUFSIZE NK_INTEGER */ + { 326, -12 }, /* (303) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tags_def_opt subtable_opt AS query_or_subquery */ + { 326, -4 }, /* (304) cmd ::= DROP STREAM exists_opt stream_name */ + { 400, 0 }, /* (305) col_list_opt ::= */ + { 400, -3 }, /* (306) col_list_opt ::= NK_LP col_name_list NK_RP */ + { 399, 0 }, /* (307) stream_options ::= */ + { 399, -3 }, /* (308) stream_options ::= stream_options TRIGGER AT_ONCE */ + { 399, -3 }, /* (309) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ + { 399, -4 }, /* (310) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ + { 399, -3 }, /* (311) stream_options ::= stream_options WATERMARK duration_literal */ + { 399, -4 }, /* (312) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ + { 399, -3 }, /* (313) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ + { 401, 0 }, /* (314) subtable_opt ::= */ + { 401, -4 }, /* (315) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ + { 326, -3 }, /* (316) cmd ::= KILL CONNECTION NK_INTEGER */ + { 326, -3 }, /* (317) cmd ::= KILL QUERY NK_STRING */ + { 326, -3 }, /* (318) cmd ::= KILL TRANSACTION NK_INTEGER */ + { 326, -2 }, /* (319) cmd ::= BALANCE VGROUP */ + { 326, -4 }, /* (320) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + { 326, -4 }, /* (321) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ + { 326, -3 }, /* (322) cmd ::= SPLIT VGROUP NK_INTEGER */ + { 403, -2 }, /* (323) dnode_list ::= DNODE NK_INTEGER */ + { 403, -3 }, /* (324) dnode_list ::= dnode_list DNODE NK_INTEGER */ + { 326, -4 }, /* (325) cmd ::= DELETE FROM full_table_name where_clause_opt */ + { 326, -1 }, /* (326) cmd ::= query_or_subquery */ + { 326, -7 }, /* (327) cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ + { 326, -4 }, /* (328) cmd ::= INSERT INTO full_table_name query_or_subquery */ + { 329, -1 }, /* (329) literal ::= NK_INTEGER */ + { 329, -1 }, /* (330) literal ::= NK_FLOAT */ + { 329, -1 }, /* (331) literal ::= NK_STRING */ + { 329, -1 }, /* (332) literal ::= NK_BOOL */ + { 329, -2 }, /* (333) literal ::= TIMESTAMP NK_STRING */ + { 329, -1 }, /* (334) literal ::= duration_literal */ + { 329, -1 }, /* (335) literal ::= NULL */ + { 329, -1 }, /* (336) literal ::= NK_QUESTION */ + { 373, -1 }, /* (337) duration_literal ::= NK_VARIABLE */ + { 405, -1 }, /* (338) signed ::= NK_INTEGER */ + { 405, -2 }, /* (339) signed ::= NK_PLUS NK_INTEGER */ + { 405, -2 }, /* (340) signed ::= NK_MINUS NK_INTEGER */ + { 405, -1 }, /* (341) signed ::= NK_FLOAT */ + { 405, -2 }, /* (342) signed ::= NK_PLUS NK_FLOAT */ + { 405, -2 }, /* (343) signed ::= NK_MINUS NK_FLOAT */ + { 362, -1 }, /* (344) signed_literal ::= signed */ + { 362, -1 }, /* (345) signed_literal ::= NK_STRING */ + { 362, -1 }, /* (346) signed_literal ::= NK_BOOL */ + { 362, -2 }, /* (347) signed_literal ::= TIMESTAMP NK_STRING */ + { 362, -1 }, /* (348) signed_literal ::= duration_literal */ + { 362, -1 }, /* (349) signed_literal ::= NULL */ + { 362, -1 }, /* (350) signed_literal ::= literal_func */ + { 362, -1 }, /* (351) signed_literal ::= NK_QUESTION */ + { 407, -1 }, /* (352) literal_list ::= signed_literal */ + { 407, -3 }, /* (353) literal_list ::= literal_list NK_COMMA signed_literal */ + { 337, -1 }, /* (354) db_name ::= NK_ID */ + { 368, -1 }, /* (355) table_name ::= NK_ID */ + { 360, -1 }, /* (356) column_name ::= NK_ID */ + { 375, -1 }, /* (357) function_name ::= NK_ID */ + { 408, -1 }, /* (358) table_alias ::= NK_ID */ + { 383, -1 }, /* (359) column_alias ::= NK_ID */ + { 331, -1 }, /* (360) user_name ::= NK_ID */ + { 338, -1 }, /* (361) topic_name ::= NK_ID */ + { 398, -1 }, /* (362) stream_name ::= NK_ID */ + { 393, -1 }, /* (363) cgroup_name ::= NK_ID */ + { 386, -1 }, /* (364) index_name ::= NK_ID */ + { 409, -1 }, /* (365) expr_or_subquery ::= expression */ + { 402, -1 }, /* (366) expression ::= literal */ + { 402, -1 }, /* (367) expression ::= pseudo_column */ + { 402, -1 }, /* (368) expression ::= column_reference */ + { 402, -1 }, /* (369) expression ::= function_expression */ + { 402, -1 }, /* (370) expression ::= case_when_expression */ + { 402, -3 }, /* (371) expression ::= NK_LP expression NK_RP */ + { 402, -2 }, /* (372) expression ::= NK_PLUS expr_or_subquery */ + { 402, -2 }, /* (373) expression ::= NK_MINUS expr_or_subquery */ + { 402, -3 }, /* (374) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ + { 402, -3 }, /* (375) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ + { 402, -3 }, /* (376) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ + { 402, -3 }, /* (377) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ + { 402, -3 }, /* (378) expression ::= expr_or_subquery NK_REM expr_or_subquery */ + { 402, -3 }, /* (379) expression ::= column_reference NK_ARROW NK_STRING */ + { 402, -3 }, /* (380) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ + { 402, -3 }, /* (381) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ + { 365, -1 }, /* (382) expression_list ::= expr_or_subquery */ + { 365, -3 }, /* (383) expression_list ::= expression_list NK_COMMA expr_or_subquery */ + { 411, -1 }, /* (384) column_reference ::= column_name */ + { 411, -3 }, /* (385) column_reference ::= table_name NK_DOT column_name */ + { 410, -1 }, /* (386) pseudo_column ::= ROWTS */ + { 410, -1 }, /* (387) pseudo_column ::= TBNAME */ + { 410, -3 }, /* (388) pseudo_column ::= table_name NK_DOT TBNAME */ + { 410, -1 }, /* (389) pseudo_column ::= QSTART */ + { 410, -1 }, /* (390) pseudo_column ::= QEND */ + { 410, -1 }, /* (391) pseudo_column ::= QDURATION */ + { 410, -1 }, /* (392) pseudo_column ::= WSTART */ + { 410, -1 }, /* (393) pseudo_column ::= WEND */ + { 410, -1 }, /* (394) pseudo_column ::= WDURATION */ + { 410, -1 }, /* (395) pseudo_column ::= IROWTS */ + { 410, -1 }, /* (396) pseudo_column ::= ISFILLED */ + { 410, -1 }, /* (397) pseudo_column ::= QTAGS */ + { 412, -4 }, /* (398) function_expression ::= function_name NK_LP expression_list NK_RP */ + { 412, -4 }, /* (399) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ + { 412, -6 }, /* (400) function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ + { 412, -1 }, /* (401) function_expression ::= literal_func */ + { 406, -3 }, /* (402) literal_func ::= noarg_func NK_LP NK_RP */ + { 406, -1 }, /* (403) literal_func ::= NOW */ + { 416, -1 }, /* (404) noarg_func ::= NOW */ + { 416, -1 }, /* (405) noarg_func ::= TODAY */ + { 416, -1 }, /* (406) noarg_func ::= TIMEZONE */ + { 416, -1 }, /* (407) noarg_func ::= DATABASE */ + { 416, -1 }, /* (408) noarg_func ::= CLIENT_VERSION */ + { 416, -1 }, /* (409) noarg_func ::= SERVER_VERSION */ + { 416, -1 }, /* (410) noarg_func ::= SERVER_STATUS */ + { 416, -1 }, /* (411) noarg_func ::= CURRENT_USER */ + { 416, -1 }, /* (412) noarg_func ::= USER */ + { 414, -1 }, /* (413) star_func ::= COUNT */ + { 414, -1 }, /* (414) star_func ::= FIRST */ + { 414, -1 }, /* (415) star_func ::= LAST */ + { 414, -1 }, /* (416) star_func ::= LAST_ROW */ + { 415, -1 }, /* (417) star_func_para_list ::= NK_STAR */ + { 415, -1 }, /* (418) star_func_para_list ::= other_para_list */ + { 417, -1 }, /* (419) other_para_list ::= star_func_para */ + { 417, -3 }, /* (420) other_para_list ::= other_para_list NK_COMMA star_func_para */ + { 418, -1 }, /* (421) star_func_para ::= expr_or_subquery */ + { 418, -3 }, /* (422) star_func_para ::= table_name NK_DOT NK_STAR */ + { 413, -4 }, /* (423) case_when_expression ::= CASE when_then_list case_when_else_opt END */ + { 413, -5 }, /* (424) case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ + { 419, -1 }, /* (425) when_then_list ::= when_then_expr */ + { 419, -2 }, /* (426) when_then_list ::= when_then_list when_then_expr */ + { 422, -4 }, /* (427) when_then_expr ::= WHEN common_expression THEN common_expression */ + { 420, 0 }, /* (428) case_when_else_opt ::= */ + { 420, -2 }, /* (429) case_when_else_opt ::= ELSE common_expression */ + { 423, -3 }, /* (430) predicate ::= expr_or_subquery compare_op expr_or_subquery */ + { 423, -5 }, /* (431) predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ + { 423, -6 }, /* (432) predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ + { 423, -3 }, /* (433) predicate ::= expr_or_subquery IS NULL */ + { 423, -4 }, /* (434) predicate ::= expr_or_subquery IS NOT NULL */ + { 423, -3 }, /* (435) predicate ::= expr_or_subquery in_op in_predicate_value */ + { 424, -1 }, /* (436) compare_op ::= NK_LT */ + { 424, -1 }, /* (437) compare_op ::= NK_GT */ + { 424, -1 }, /* (438) compare_op ::= NK_LE */ + { 424, -1 }, /* (439) compare_op ::= NK_GE */ + { 424, -1 }, /* (440) compare_op ::= NK_NE */ + { 424, -1 }, /* (441) compare_op ::= NK_EQ */ + { 424, -1 }, /* (442) compare_op ::= LIKE */ + { 424, -2 }, /* (443) compare_op ::= NOT LIKE */ + { 424, -1 }, /* (444) compare_op ::= MATCH */ + { 424, -1 }, /* (445) compare_op ::= NMATCH */ + { 424, -1 }, /* (446) compare_op ::= CONTAINS */ + { 425, -1 }, /* (447) in_op ::= IN */ + { 425, -2 }, /* (448) in_op ::= NOT IN */ + { 426, -3 }, /* (449) in_predicate_value ::= NK_LP literal_list NK_RP */ + { 427, -1 }, /* (450) boolean_value_expression ::= boolean_primary */ + { 427, -2 }, /* (451) boolean_value_expression ::= NOT boolean_primary */ + { 427, -3 }, /* (452) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + { 427, -3 }, /* (453) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + { 428, -1 }, /* (454) boolean_primary ::= predicate */ + { 428, -3 }, /* (455) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + { 421, -1 }, /* (456) common_expression ::= expr_or_subquery */ + { 421, -1 }, /* (457) common_expression ::= boolean_value_expression */ + { 429, 0 }, /* (458) from_clause_opt ::= */ + { 429, -2 }, /* (459) from_clause_opt ::= FROM table_reference_list */ + { 430, -1 }, /* (460) table_reference_list ::= table_reference */ + { 430, -3 }, /* (461) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 431, -1 }, /* (462) table_reference ::= table_primary */ + { 431, -1 }, /* (463) table_reference ::= joined_table */ + { 432, -2 }, /* (464) table_primary ::= table_name alias_opt */ + { 432, -4 }, /* (465) table_primary ::= db_name NK_DOT table_name alias_opt */ + { 432, -2 }, /* (466) table_primary ::= subquery alias_opt */ + { 432, -1 }, /* (467) table_primary ::= parenthesized_joined_table */ + { 434, 0 }, /* (468) alias_opt ::= */ + { 434, -1 }, /* (469) alias_opt ::= table_alias */ + { 434, -2 }, /* (470) alias_opt ::= AS table_alias */ + { 436, -3 }, /* (471) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 436, -3 }, /* (472) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 433, -6 }, /* (473) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 437, 0 }, /* (474) join_type ::= */ + { 437, -1 }, /* (475) join_type ::= INNER */ + { 439, -12 }, /* (476) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 440, 0 }, /* (477) set_quantifier_opt ::= */ + { 440, -1 }, /* (478) set_quantifier_opt ::= DISTINCT */ + { 440, -1 }, /* (479) set_quantifier_opt ::= ALL */ + { 441, -1 }, /* (480) select_list ::= select_item */ + { 441, -3 }, /* (481) select_list ::= select_list NK_COMMA select_item */ + { 449, -1 }, /* (482) select_item ::= NK_STAR */ + { 449, -1 }, /* (483) select_item ::= common_expression */ + { 449, -2 }, /* (484) select_item ::= common_expression column_alias */ + { 449, -3 }, /* (485) select_item ::= common_expression AS column_alias */ + { 449, -3 }, /* (486) select_item ::= table_name NK_DOT NK_STAR */ + { 404, 0 }, /* (487) where_clause_opt ::= */ + { 404, -2 }, /* (488) where_clause_opt ::= WHERE search_condition */ + { 442, 0 }, /* (489) partition_by_clause_opt ::= */ + { 442, -3 }, /* (490) partition_by_clause_opt ::= PARTITION BY partition_list */ + { 450, -1 }, /* (491) partition_list ::= partition_item */ + { 450, -3 }, /* (492) partition_list ::= partition_list NK_COMMA partition_item */ + { 451, -1 }, /* (493) partition_item ::= expr_or_subquery */ + { 451, -2 }, /* (494) partition_item ::= expr_or_subquery column_alias */ + { 451, -3 }, /* (495) partition_item ::= expr_or_subquery AS column_alias */ + { 446, 0 }, /* (496) twindow_clause_opt ::= */ + { 446, -6 }, /* (497) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + { 446, -4 }, /* (498) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ + { 446, -6 }, /* (499) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 446, -8 }, /* (500) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 446, -7 }, /* (501) twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ + { 388, 0 }, /* (502) sliding_opt ::= */ + { 388, -4 }, /* (503) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 445, 0 }, /* (504) fill_opt ::= */ + { 445, -4 }, /* (505) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 445, -6 }, /* (506) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + { 452, -1 }, /* (507) fill_mode ::= NONE */ + { 452, -1 }, /* (508) fill_mode ::= PREV */ + { 452, -1 }, /* (509) fill_mode ::= NULL */ + { 452, -1 }, /* (510) fill_mode ::= LINEAR */ + { 452, -1 }, /* (511) fill_mode ::= NEXT */ + { 447, 0 }, /* (512) group_by_clause_opt ::= */ + { 447, -3 }, /* (513) group_by_clause_opt ::= GROUP BY group_by_list */ + { 453, -1 }, /* (514) group_by_list ::= expr_or_subquery */ + { 453, -3 }, /* (515) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ + { 448, 0 }, /* (516) having_clause_opt ::= */ + { 448, -2 }, /* (517) having_clause_opt ::= HAVING search_condition */ + { 443, 0 }, /* (518) range_opt ::= */ + { 443, -6 }, /* (519) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ + { 444, 0 }, /* (520) every_opt ::= */ + { 444, -4 }, /* (521) every_opt ::= EVERY NK_LP duration_literal NK_RP */ + { 454, -4 }, /* (522) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 455, -1 }, /* (523) query_simple ::= query_specification */ + { 455, -1 }, /* (524) query_simple ::= union_query_expression */ + { 459, -4 }, /* (525) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ + { 459, -3 }, /* (526) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ + { 460, -1 }, /* (527) query_simple_or_subquery ::= query_simple */ + { 460, -1 }, /* (528) query_simple_or_subquery ::= subquery */ + { 392, -1 }, /* (529) query_or_subquery ::= query_expression */ + { 392, -1 }, /* (530) query_or_subquery ::= subquery */ + { 456, 0 }, /* (531) order_by_clause_opt ::= */ + { 456, -3 }, /* (532) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 457, 0 }, /* (533) slimit_clause_opt ::= */ + { 457, -2 }, /* (534) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 457, -4 }, /* (535) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 457, -4 }, /* (536) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 458, 0 }, /* (537) limit_clause_opt ::= */ + { 458, -2 }, /* (538) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 458, -4 }, /* (539) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 458, -4 }, /* (540) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 435, -3 }, /* (541) subquery ::= NK_LP query_expression NK_RP */ + { 435, -3 }, /* (542) subquery ::= NK_LP subquery NK_RP */ + { 438, -1 }, /* (543) search_condition ::= common_expression */ + { 461, -1 }, /* (544) sort_specification_list ::= sort_specification */ + { 461, -3 }, /* (545) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 462, -3 }, /* (546) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ + { 463, 0 }, /* (547) ordering_specification_opt ::= */ + { 463, -1 }, /* (548) ordering_specification_opt ::= ASC */ + { 463, -1 }, /* (549) ordering_specification_opt ::= DESC */ + { 464, 0 }, /* (550) null_ordering_opt ::= */ + { 464, -2 }, /* (551) null_ordering_opt ::= NULLS FIRST */ + { 464, -2 }, /* (552) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -3629,11 +3663,11 @@ static YYACTIONTYPE yy_reduce( YYMINORTYPE yylhsminor; case 0: /* cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,323,&yymsp[0].minor); + yy_destructor(yypParser,327,&yymsp[0].minor); break; case 1: /* cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,324,&yymsp[0].minor); + yy_destructor(yypParser,328,&yymsp[0].minor); break; case 2: /* account_options ::= */ { } @@ -3647,20 +3681,20 @@ static YYACTIONTYPE yy_reduce( case 9: /* account_options ::= account_options USERS literal */ yytestcase(yyruleno==9); case 10: /* account_options ::= account_options CONNS literal */ yytestcase(yyruleno==10); case 11: /* account_options ::= account_options STATE literal */ yytestcase(yyruleno==11); -{ yy_destructor(yypParser,323,&yymsp[-2].minor); +{ yy_destructor(yypParser,327,&yymsp[-2].minor); { } - yy_destructor(yypParser,325,&yymsp[0].minor); + yy_destructor(yypParser,329,&yymsp[0].minor); } break; case 12: /* alter_account_options ::= alter_account_option */ -{ yy_destructor(yypParser,326,&yymsp[0].minor); +{ yy_destructor(yypParser,330,&yymsp[0].minor); { } } break; case 13: /* alter_account_options ::= alter_account_options alter_account_option */ -{ yy_destructor(yypParser,324,&yymsp[-1].minor); +{ yy_destructor(yypParser,328,&yymsp[-1].minor); { } - yy_destructor(yypParser,326,&yymsp[0].minor); + yy_destructor(yypParser,330,&yymsp[0].minor); } break; case 14: /* alter_account_option ::= PASS literal */ @@ -3674,80 +3708,81 @@ static YYACTIONTYPE yy_reduce( case 22: /* alter_account_option ::= CONNS literal */ yytestcase(yyruleno==22); case 23: /* alter_account_option ::= STATE literal */ yytestcase(yyruleno==23); { } - yy_destructor(yypParser,325,&yymsp[0].minor); + yy_destructor(yypParser,329,&yymsp[0].minor); break; case 24: /* cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ -{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-3].minor.yy317, &yymsp[-1].minor.yy0, yymsp[0].minor.yy449); } +{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-3].minor.yy179, &yymsp[-1].minor.yy0, yymsp[0].minor.yy113); } break; case 25: /* cmd ::= ALTER USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy317, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy179, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } break; case 26: /* cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy317, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy179, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } break; case 27: /* cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy317, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy179, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } break; case 28: /* cmd ::= DROP USER user_name */ -{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy317); } +{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy179); } break; case 29: /* sysinfo_opt ::= */ -{ yymsp[1].minor.yy449 = 1; } +{ yymsp[1].minor.yy113 = 1; } break; case 30: /* sysinfo_opt ::= SYSINFO NK_INTEGER */ -{ yymsp[-1].minor.yy449 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } +{ yymsp[-1].minor.yy113 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } break; case 31: /* cmd ::= GRANT privileges ON priv_level TO user_name */ -{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-4].minor.yy531, &yymsp[-2].minor.yy317, &yymsp[0].minor.yy317); } +{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-4].minor.yy159, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy179); } break; case 32: /* cmd ::= REVOKE privileges ON priv_level FROM user_name */ -{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-4].minor.yy531, &yymsp[-2].minor.yy317, &yymsp[0].minor.yy317); } +{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-4].minor.yy159, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy179); } break; case 33: /* privileges ::= ALL */ -{ yymsp[0].minor.yy531 = PRIVILEGE_TYPE_ALL; } +{ yymsp[0].minor.yy159 = PRIVILEGE_TYPE_ALL; } break; case 34: /* privileges ::= priv_type_list */ case 36: /* priv_type_list ::= priv_type */ yytestcase(yyruleno==36); -{ yylhsminor.yy531 = yymsp[0].minor.yy531; } - yymsp[0].minor.yy531 = yylhsminor.yy531; +{ yylhsminor.yy159 = yymsp[0].minor.yy159; } + yymsp[0].minor.yy159 = yylhsminor.yy159; break; case 35: /* privileges ::= SUBSCRIBE */ -{ yymsp[0].minor.yy531 = PRIVILEGE_TYPE_SUBSCRIBE; } +{ yymsp[0].minor.yy159 = PRIVILEGE_TYPE_SUBSCRIBE; } break; case 37: /* priv_type_list ::= priv_type_list NK_COMMA priv_type */ -{ yylhsminor.yy531 = yymsp[-2].minor.yy531 | yymsp[0].minor.yy531; } - yymsp[-2].minor.yy531 = yylhsminor.yy531; +{ yylhsminor.yy159 = yymsp[-2].minor.yy159 | yymsp[0].minor.yy159; } + yymsp[-2].minor.yy159 = yylhsminor.yy159; break; case 38: /* priv_type ::= READ */ -{ yymsp[0].minor.yy531 = PRIVILEGE_TYPE_READ; } +{ yymsp[0].minor.yy159 = PRIVILEGE_TYPE_READ; } break; case 39: /* priv_type ::= WRITE */ -{ yymsp[0].minor.yy531 = PRIVILEGE_TYPE_WRITE; } +{ yymsp[0].minor.yy159 = PRIVILEGE_TYPE_WRITE; } break; case 40: /* priv_level ::= NK_STAR NK_DOT NK_STAR */ -{ yylhsminor.yy317 = yymsp[-2].minor.yy0; } - yymsp[-2].minor.yy317 = yylhsminor.yy317; +{ yylhsminor.yy179 = yymsp[-2].minor.yy0; } + yymsp[-2].minor.yy179 = yylhsminor.yy179; break; case 41: /* priv_level ::= db_name NK_DOT NK_STAR */ -{ yylhsminor.yy317 = yymsp[-2].minor.yy317; } - yymsp[-2].minor.yy317 = yylhsminor.yy317; +{ yylhsminor.yy179 = yymsp[-2].minor.yy179; } + yymsp[-2].minor.yy179 = yylhsminor.yy179; break; case 42: /* priv_level ::= topic_name */ - case 458: /* alias_opt ::= table_alias */ yytestcase(yyruleno==458); -{ yylhsminor.yy317 = yymsp[0].minor.yy317; } - yymsp[0].minor.yy317 = yylhsminor.yy317; + case 272: /* sma_func_name ::= function_name */ yytestcase(yyruleno==272); + case 469: /* alias_opt ::= table_alias */ yytestcase(yyruleno==469); +{ yylhsminor.yy179 = yymsp[0].minor.yy179; } + yymsp[0].minor.yy179 = yylhsminor.yy179; break; case 43: /* cmd ::= CREATE DNODE dnode_endpoint */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy317, NULL); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy179, NULL); } break; case 44: /* cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy317, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy0); } break; case 45: /* cmd ::= DROP DNODE NK_INTEGER force_opt */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy335); } +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy767); } break; case 46: /* cmd ::= DROP DNODE dnode_endpoint force_opt */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy317, yymsp[0].minor.yy335); } +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy179, yymsp[0].minor.yy767); } break; case 47: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, NULL); } @@ -3764,46 +3799,50 @@ static YYACTIONTYPE yy_reduce( case 51: /* dnode_endpoint ::= NK_STRING */ case 52: /* dnode_endpoint ::= NK_ID */ yytestcase(yyruleno==52); case 53: /* dnode_endpoint ::= NK_IPTOKEN */ yytestcase(yyruleno==53); - case 344: /* db_name ::= NK_ID */ yytestcase(yyruleno==344); - case 345: /* table_name ::= NK_ID */ yytestcase(yyruleno==345); - case 346: /* column_name ::= NK_ID */ yytestcase(yyruleno==346); - case 347: /* function_name ::= NK_ID */ yytestcase(yyruleno==347); - case 348: /* table_alias ::= NK_ID */ yytestcase(yyruleno==348); - case 349: /* column_alias ::= NK_ID */ yytestcase(yyruleno==349); - case 350: /* user_name ::= NK_ID */ yytestcase(yyruleno==350); - case 351: /* topic_name ::= NK_ID */ yytestcase(yyruleno==351); - case 352: /* stream_name ::= NK_ID */ yytestcase(yyruleno==352); - case 353: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==353); - case 354: /* index_name ::= NK_ID */ yytestcase(yyruleno==354); - case 393: /* noarg_func ::= NOW */ yytestcase(yyruleno==393); - case 394: /* noarg_func ::= TODAY */ yytestcase(yyruleno==394); - case 395: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==395); - case 396: /* noarg_func ::= DATABASE */ yytestcase(yyruleno==396); - case 397: /* noarg_func ::= CLIENT_VERSION */ yytestcase(yyruleno==397); - case 398: /* noarg_func ::= SERVER_VERSION */ yytestcase(yyruleno==398); - case 399: /* noarg_func ::= SERVER_STATUS */ yytestcase(yyruleno==399); - case 400: /* noarg_func ::= CURRENT_USER */ yytestcase(yyruleno==400); - case 401: /* noarg_func ::= USER */ yytestcase(yyruleno==401); - case 402: /* star_func ::= COUNT */ yytestcase(yyruleno==402); - case 403: /* star_func ::= FIRST */ yytestcase(yyruleno==403); - case 404: /* star_func ::= LAST */ yytestcase(yyruleno==404); - case 405: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==405); -{ yylhsminor.yy317 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy317 = yylhsminor.yy317; + case 273: /* sma_func_name ::= COUNT */ yytestcase(yyruleno==273); + case 274: /* sma_func_name ::= FIRST */ yytestcase(yyruleno==274); + case 275: /* sma_func_name ::= LAST */ yytestcase(yyruleno==275); + case 276: /* sma_func_name ::= LAST_ROW */ yytestcase(yyruleno==276); + case 354: /* db_name ::= NK_ID */ yytestcase(yyruleno==354); + case 355: /* table_name ::= NK_ID */ yytestcase(yyruleno==355); + case 356: /* column_name ::= NK_ID */ yytestcase(yyruleno==356); + case 357: /* function_name ::= NK_ID */ yytestcase(yyruleno==357); + case 358: /* table_alias ::= NK_ID */ yytestcase(yyruleno==358); + case 359: /* column_alias ::= NK_ID */ yytestcase(yyruleno==359); + case 360: /* user_name ::= NK_ID */ yytestcase(yyruleno==360); + case 361: /* topic_name ::= NK_ID */ yytestcase(yyruleno==361); + case 362: /* stream_name ::= NK_ID */ yytestcase(yyruleno==362); + case 363: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==363); + case 364: /* index_name ::= NK_ID */ yytestcase(yyruleno==364); + case 404: /* noarg_func ::= NOW */ yytestcase(yyruleno==404); + case 405: /* noarg_func ::= TODAY */ yytestcase(yyruleno==405); + case 406: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==406); + case 407: /* noarg_func ::= DATABASE */ yytestcase(yyruleno==407); + case 408: /* noarg_func ::= CLIENT_VERSION */ yytestcase(yyruleno==408); + case 409: /* noarg_func ::= SERVER_VERSION */ yytestcase(yyruleno==409); + case 410: /* noarg_func ::= SERVER_STATUS */ yytestcase(yyruleno==410); + case 411: /* noarg_func ::= CURRENT_USER */ yytestcase(yyruleno==411); + case 412: /* noarg_func ::= USER */ yytestcase(yyruleno==412); + case 413: /* star_func ::= COUNT */ yytestcase(yyruleno==413); + case 414: /* star_func ::= FIRST */ yytestcase(yyruleno==414); + case 415: /* star_func ::= LAST */ yytestcase(yyruleno==415); + case 416: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==416); +{ yylhsminor.yy179 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy179 = yylhsminor.yy179; break; case 54: /* force_opt ::= */ case 73: /* not_exists_opt ::= */ yytestcase(yyruleno==73); case 75: /* exists_opt ::= */ yytestcase(yyruleno==75); - case 284: /* analyze_opt ::= */ yytestcase(yyruleno==284); - case 291: /* agg_func_opt ::= */ yytestcase(yyruleno==291); - case 466: /* set_quantifier_opt ::= */ yytestcase(yyruleno==466); -{ yymsp[1].minor.yy335 = false; } + case 292: /* analyze_opt ::= */ yytestcase(yyruleno==292); + case 299: /* agg_func_opt ::= */ yytestcase(yyruleno==299); + case 477: /* set_quantifier_opt ::= */ yytestcase(yyruleno==477); +{ yymsp[1].minor.yy767 = false; } break; case 55: /* force_opt ::= FORCE */ - case 285: /* analyze_opt ::= ANALYZE */ yytestcase(yyruleno==285); - case 292: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==292); - case 467: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==467); -{ yymsp[0].minor.yy335 = true; } + case 293: /* analyze_opt ::= ANALYZE */ yytestcase(yyruleno==293); + case 300: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==300); + case 478: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==478); +{ yymsp[0].minor.yy767 = true; } break; case 56: /* cmd ::= ALTER LOCAL NK_STRING */ { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[0].minor.yy0, NULL); } @@ -3836,196 +3875,196 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_MNODE_STMT, &yymsp[0].minor.yy0); } break; case 66: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ -{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy335, &yymsp[-1].minor.yy317, yymsp[0].minor.yy74); } +{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy767, &yymsp[-1].minor.yy179, yymsp[0].minor.yy602); } break; case 67: /* cmd ::= DROP DATABASE exists_opt db_name */ -{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy335, &yymsp[0].minor.yy317); } +{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy767, &yymsp[0].minor.yy179); } break; case 68: /* cmd ::= USE db_name */ -{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy317); } +{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy179); } break; case 69: /* cmd ::= ALTER DATABASE db_name alter_db_options */ -{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy317, yymsp[0].minor.yy74); } +{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy179, yymsp[0].minor.yy602); } break; case 70: /* cmd ::= FLUSH DATABASE db_name */ -{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy317); } +{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy179); } break; case 71: /* cmd ::= TRIM DATABASE db_name speed_opt */ -{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy317, yymsp[0].minor.yy856); } +{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy179, yymsp[0].minor.yy820); } break; case 72: /* not_exists_opt ::= IF NOT EXISTS */ -{ yymsp[-2].minor.yy335 = true; } +{ yymsp[-2].minor.yy767 = true; } break; case 74: /* exists_opt ::= IF EXISTS */ -{ yymsp[-1].minor.yy335 = true; } +{ yymsp[-1].minor.yy767 = true; } break; case 76: /* db_options ::= */ -{ yymsp[1].minor.yy74 = createDefaultDatabaseOptions(pCxt); } +{ yymsp[1].minor.yy602 = createDefaultDatabaseOptions(pCxt); } break; case 77: /* db_options ::= db_options BUFFER NK_INTEGER */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 78: /* db_options ::= db_options CACHEMODEL NK_STRING */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 79: /* db_options ::= db_options CACHESIZE NK_INTEGER */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 80: /* db_options ::= db_options COMP NK_INTEGER */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_COMP, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_COMP, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 81: /* db_options ::= db_options DURATION NK_INTEGER */ case 82: /* db_options ::= db_options DURATION NK_VARIABLE */ yytestcase(yyruleno==82); -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 83: /* db_options ::= db_options MAXROWS NK_INTEGER */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 84: /* db_options ::= db_options MINROWS NK_INTEGER */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 85: /* db_options ::= db_options KEEP integer_list */ case 86: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==86); -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_KEEP, yymsp[0].minor.yy874); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_KEEP, yymsp[0].minor.yy874); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 87: /* db_options ::= db_options PAGES NK_INTEGER */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 88: /* db_options ::= db_options PAGESIZE NK_INTEGER */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 89: /* db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 90: /* db_options ::= db_options PRECISION NK_STRING */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 91: /* db_options ::= db_options REPLICA NK_INTEGER */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 92: /* db_options ::= db_options VGROUPS NK_INTEGER */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 93: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 94: /* db_options ::= db_options RETENTIONS retention_list */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_RETENTIONS, yymsp[0].minor.yy874); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_RETENTIONS, yymsp[0].minor.yy874); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 95: /* db_options ::= db_options SCHEMALESS NK_INTEGER */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 96: /* db_options ::= db_options WAL_LEVEL NK_INTEGER */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_WAL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_WAL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 97: /* db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 98: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 99: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-3].minor.yy74, DB_OPTION_WAL_RETENTION_PERIOD, &t); + yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-3].minor.yy602, DB_OPTION_WAL_RETENTION_PERIOD, &t); } - yymsp[-3].minor.yy74 = yylhsminor.yy74; + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; case 100: /* db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 101: /* db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-3].minor.yy74, DB_OPTION_WAL_RETENTION_SIZE, &t); + yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-3].minor.yy602, DB_OPTION_WAL_RETENTION_SIZE, &t); } - yymsp[-3].minor.yy74 = yylhsminor.yy74; + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; case 102: /* db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 103: /* db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 104: /* db_options ::= db_options STT_TRIGGER NK_INTEGER */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 105: /* db_options ::= db_options TABLE_PREFIX NK_INTEGER */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_TABLE_PREFIX, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_TABLE_PREFIX, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 106: /* db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ -{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_TABLE_SUFFIX, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_TABLE_SUFFIX, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 107: /* alter_db_options ::= alter_db_option */ -{ yylhsminor.yy74 = createAlterDatabaseOptions(pCxt); yylhsminor.yy74 = setAlterDatabaseOption(pCxt, yylhsminor.yy74, &yymsp[0].minor.yy767); } - yymsp[0].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = createAlterDatabaseOptions(pCxt); yylhsminor.yy602 = setAlterDatabaseOption(pCxt, yylhsminor.yy602, &yymsp[0].minor.yy845); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; case 108: /* alter_db_options ::= alter_db_options alter_db_option */ -{ yylhsminor.yy74 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy74, &yymsp[0].minor.yy767); } - yymsp[-1].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy602, &yymsp[0].minor.yy845); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; case 109: /* alter_db_option ::= BUFFER NK_INTEGER */ -{ yymsp[-1].minor.yy767.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy767.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy845.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy845.val = yymsp[0].minor.yy0; } break; case 110: /* alter_db_option ::= CACHEMODEL NK_STRING */ -{ yymsp[-1].minor.yy767.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy767.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy845.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy845.val = yymsp[0].minor.yy0; } break; case 111: /* alter_db_option ::= CACHESIZE NK_INTEGER */ -{ yymsp[-1].minor.yy767.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy767.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy845.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy845.val = yymsp[0].minor.yy0; } break; case 112: /* alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ -{ yymsp[-1].minor.yy767.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy767.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy845.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy845.val = yymsp[0].minor.yy0; } break; case 113: /* alter_db_option ::= KEEP integer_list */ case 114: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==114); -{ yymsp[-1].minor.yy767.type = DB_OPTION_KEEP; yymsp[-1].minor.yy767.pList = yymsp[0].minor.yy874; } +{ yymsp[-1].minor.yy845.type = DB_OPTION_KEEP; yymsp[-1].minor.yy845.pList = yymsp[0].minor.yy874; } break; case 115: /* alter_db_option ::= PAGES NK_INTEGER */ -{ yymsp[-1].minor.yy767.type = DB_OPTION_PAGES; yymsp[-1].minor.yy767.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy845.type = DB_OPTION_PAGES; yymsp[-1].minor.yy845.val = yymsp[0].minor.yy0; } break; case 116: /* alter_db_option ::= REPLICA NK_INTEGER */ -{ yymsp[-1].minor.yy767.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy767.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy845.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy845.val = yymsp[0].minor.yy0; } break; case 117: /* alter_db_option ::= WAL_LEVEL NK_INTEGER */ -{ yymsp[-1].minor.yy767.type = DB_OPTION_WAL; yymsp[-1].minor.yy767.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy845.type = DB_OPTION_WAL; yymsp[-1].minor.yy845.val = yymsp[0].minor.yy0; } break; case 118: /* alter_db_option ::= STT_TRIGGER NK_INTEGER */ -{ yymsp[-1].minor.yy767.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy767.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy845.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy845.val = yymsp[0].minor.yy0; } break; case 119: /* integer_list ::= NK_INTEGER */ { yylhsminor.yy874 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } yymsp[0].minor.yy874 = yylhsminor.yy874; break; case 120: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ - case 314: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==314); + case 324: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==324); { yylhsminor.yy874 = addNodeToList(pCxt, yymsp[-2].minor.yy874, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } yymsp[-2].minor.yy874 = yylhsminor.yy874; break; @@ -4043,46 +4082,46 @@ static YYACTIONTYPE yy_reduce( case 155: /* column_def_list ::= column_def */ yytestcase(yyruleno==155); case 199: /* rollup_func_list ::= rollup_func_name */ yytestcase(yyruleno==199); case 204: /* col_name_list ::= col_name */ yytestcase(yyruleno==204); - case 253: /* tag_list_opt ::= tag_item */ yytestcase(yyruleno==253); - case 266: /* func_list ::= func */ yytestcase(yyruleno==266); - case 342: /* literal_list ::= signed_literal */ yytestcase(yyruleno==342); - case 408: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==408); - case 414: /* when_then_list ::= when_then_expr */ yytestcase(yyruleno==414); - case 469: /* select_list ::= select_item */ yytestcase(yyruleno==469); - case 480: /* partition_list ::= partition_item */ yytestcase(yyruleno==480); - case 532: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==532); -{ yylhsminor.yy874 = createNodeList(pCxt, yymsp[0].minor.yy74); } + case 255: /* tag_list_opt ::= tag_item */ yytestcase(yyruleno==255); + case 269: /* func_list ::= func */ yytestcase(yyruleno==269); + case 352: /* literal_list ::= signed_literal */ yytestcase(yyruleno==352); + case 419: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==419); + case 425: /* when_then_list ::= when_then_expr */ yytestcase(yyruleno==425); + case 480: /* select_list ::= select_item */ yytestcase(yyruleno==480); + case 491: /* partition_list ::= partition_item */ yytestcase(yyruleno==491); + case 544: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==544); +{ yylhsminor.yy874 = createNodeList(pCxt, yymsp[0].minor.yy602); } yymsp[0].minor.yy874 = yylhsminor.yy874; break; case 124: /* retention_list ::= retention_list NK_COMMA retention */ case 156: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==156); case 200: /* rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ yytestcase(yyruleno==200); case 205: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==205); - case 254: /* tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ yytestcase(yyruleno==254); - case 267: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==267); - case 343: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==343); - case 409: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==409); - case 470: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==470); - case 481: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==481); - case 533: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==533); -{ yylhsminor.yy874 = addNodeToList(pCxt, yymsp[-2].minor.yy874, yymsp[0].minor.yy74); } + case 256: /* tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ yytestcase(yyruleno==256); + case 270: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==270); + case 353: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==353); + case 420: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==420); + case 481: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==481); + case 492: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==492); + case 545: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==545); +{ yylhsminor.yy874 = addNodeToList(pCxt, yymsp[-2].minor.yy874, yymsp[0].minor.yy602); } yymsp[-2].minor.yy874 = yylhsminor.yy874; break; case 125: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ -{ yylhsminor.yy74 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 126: /* speed_opt ::= */ - case 293: /* bufsize_opt ::= */ yytestcase(yyruleno==293); -{ yymsp[1].minor.yy856 = 0; } + case 301: /* bufsize_opt ::= */ yytestcase(yyruleno==301); +{ yymsp[1].minor.yy820 = 0; } break; case 127: /* speed_opt ::= MAX_SPEED NK_INTEGER */ - case 294: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==294); -{ yymsp[-1].minor.yy856 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } + case 302: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==302); +{ yymsp[-1].minor.yy820 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } break; case 128: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ case 130: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==130); -{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy335, yymsp[-5].minor.yy74, yymsp[-3].minor.yy874, yymsp[-1].minor.yy874, yymsp[0].minor.yy74); } +{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy767, yymsp[-5].minor.yy602, yymsp[-3].minor.yy874, yymsp[-1].minor.yy874, yymsp[0].minor.yy602); } break; case 129: /* cmd ::= CREATE TABLE multi_create_clause */ { pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy874); } @@ -4091,165 +4130,167 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy874); } break; case 132: /* cmd ::= DROP STABLE exists_opt full_table_name */ -{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy335, yymsp[0].minor.yy74); } +{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy767, yymsp[0].minor.yy602); } break; case 133: /* cmd ::= ALTER TABLE alter_table_clause */ - case 316: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==316); -{ pCxt->pRootNode = yymsp[0].minor.yy74; } + case 326: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==326); +{ pCxt->pRootNode = yymsp[0].minor.yy602; } break; case 134: /* cmd ::= ALTER STABLE alter_table_clause */ -{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy74); } +{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy602); } break; case 135: /* alter_table_clause ::= full_table_name alter_table_options */ -{ yylhsminor.yy74 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy74, yymsp[0].minor.yy74); } - yymsp[-1].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy602, yymsp[0].minor.yy602); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; case 136: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ -{ yylhsminor.yy74 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy74, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy317, yymsp[0].minor.yy898); } - yymsp[-4].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy602, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy179, yymsp[0].minor.yy394); } + yymsp[-4].minor.yy602 = yylhsminor.yy602; break; case 137: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ -{ yylhsminor.yy74 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy74, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy317); } - yymsp[-3].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy602, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy179); } + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; case 138: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ -{ yylhsminor.yy74 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy74, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy317, yymsp[0].minor.yy898); } - yymsp[-4].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy602, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy179, yymsp[0].minor.yy394); } + yymsp[-4].minor.yy602 = yylhsminor.yy602; break; case 139: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ -{ yylhsminor.yy74 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy74, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy317, &yymsp[0].minor.yy317); } - yymsp[-4].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy602, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy179, &yymsp[0].minor.yy179); } + yymsp[-4].minor.yy602 = yylhsminor.yy602; break; case 140: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ -{ yylhsminor.yy74 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy74, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy317, yymsp[0].minor.yy898); } - yymsp[-4].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy602, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy179, yymsp[0].minor.yy394); } + yymsp[-4].minor.yy602 = yylhsminor.yy602; break; case 141: /* alter_table_clause ::= full_table_name DROP TAG column_name */ -{ yylhsminor.yy74 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy74, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy317); } - yymsp[-3].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy602, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy179); } + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; case 142: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ -{ yylhsminor.yy74 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy74, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy317, yymsp[0].minor.yy898); } - yymsp[-4].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy602, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy179, yymsp[0].minor.yy394); } + yymsp[-4].minor.yy602 = yylhsminor.yy602; break; case 143: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ -{ yylhsminor.yy74 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy74, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy317, &yymsp[0].minor.yy317); } - yymsp[-4].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy602, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy179, &yymsp[0].minor.yy179); } + yymsp[-4].minor.yy602 = yylhsminor.yy602; break; case 144: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ -{ yylhsminor.yy74 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy74, &yymsp[-2].minor.yy317, yymsp[0].minor.yy74); } - yymsp[-5].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy602, &yymsp[-2].minor.yy179, yymsp[0].minor.yy602); } + yymsp[-5].minor.yy602 = yylhsminor.yy602; break; case 146: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ case 149: /* multi_drop_clause ::= multi_drop_clause drop_table_clause */ yytestcase(yyruleno==149); - case 415: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==415); -{ yylhsminor.yy874 = addNodeToList(pCxt, yymsp[-1].minor.yy874, yymsp[0].minor.yy74); } + case 426: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==426); +{ yylhsminor.yy874 = addNodeToList(pCxt, yymsp[-1].minor.yy874, yymsp[0].minor.yy602); } yymsp[-1].minor.yy874 = yylhsminor.yy874; break; case 147: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ -{ yylhsminor.yy74 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy335, yymsp[-8].minor.yy74, yymsp[-6].minor.yy74, yymsp[-5].minor.yy874, yymsp[-2].minor.yy874, yymsp[0].minor.yy74); } - yymsp[-9].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy767, yymsp[-8].minor.yy602, yymsp[-6].minor.yy602, yymsp[-5].minor.yy874, yymsp[-2].minor.yy874, yymsp[0].minor.yy602); } + yymsp[-9].minor.yy602 = yylhsminor.yy602; break; case 150: /* drop_table_clause ::= exists_opt full_table_name */ -{ yylhsminor.yy74 = createDropTableClause(pCxt, yymsp[-1].minor.yy335, yymsp[0].minor.yy74); } - yymsp[-1].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = createDropTableClause(pCxt, yymsp[-1].minor.yy767, yymsp[0].minor.yy602); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; case 151: /* specific_cols_opt ::= */ case 182: /* tags_def_opt ::= */ yytestcase(yyruleno==182); - case 252: /* tag_list_opt ::= */ yytestcase(yyruleno==252); - case 478: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==478); - case 500: /* group_by_clause_opt ::= */ yytestcase(yyruleno==500); - case 519: /* order_by_clause_opt ::= */ yytestcase(yyruleno==519); + case 254: /* tag_list_opt ::= */ yytestcase(yyruleno==254); + case 305: /* col_list_opt ::= */ yytestcase(yyruleno==305); + case 489: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==489); + case 512: /* group_by_clause_opt ::= */ yytestcase(yyruleno==512); + case 531: /* order_by_clause_opt ::= */ yytestcase(yyruleno==531); { yymsp[1].minor.yy874 = NULL; } break; case 152: /* specific_cols_opt ::= NK_LP col_name_list NK_RP */ + case 306: /* col_list_opt ::= NK_LP col_name_list NK_RP */ yytestcase(yyruleno==306); { yymsp[-2].minor.yy874 = yymsp[-1].minor.yy874; } break; case 153: /* full_table_name ::= table_name */ -{ yylhsminor.yy74 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy317, NULL); } - yymsp[0].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy179, NULL); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; case 154: /* full_table_name ::= db_name NK_DOT table_name */ -{ yylhsminor.yy74 = createRealTableNode(pCxt, &yymsp[-2].minor.yy317, &yymsp[0].minor.yy317, NULL); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = createRealTableNode(pCxt, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy179, NULL); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 157: /* column_def ::= column_name type_name */ -{ yylhsminor.yy74 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy317, yymsp[0].minor.yy898, NULL); } - yymsp[-1].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy179, yymsp[0].minor.yy394, NULL); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; case 158: /* column_def ::= column_name type_name COMMENT NK_STRING */ -{ yylhsminor.yy74 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy317, yymsp[-2].minor.yy898, &yymsp[0].minor.yy0); } - yymsp[-3].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy179, yymsp[-2].minor.yy394, &yymsp[0].minor.yy0); } + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; case 159: /* type_name ::= BOOL */ -{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_BOOL); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_BOOL); } break; case 160: /* type_name ::= TINYINT */ -{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_TINYINT); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_TINYINT); } break; case 161: /* type_name ::= SMALLINT */ -{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_SMALLINT); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_SMALLINT); } break; case 162: /* type_name ::= INT */ case 163: /* type_name ::= INTEGER */ yytestcase(yyruleno==163); -{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_INT); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_INT); } break; case 164: /* type_name ::= BIGINT */ -{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_BIGINT); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_BIGINT); } break; case 165: /* type_name ::= FLOAT */ -{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_FLOAT); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_FLOAT); } break; case 166: /* type_name ::= DOUBLE */ -{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_DOUBLE); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_DOUBLE); } break; case 167: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy898 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy394 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } break; case 168: /* type_name ::= TIMESTAMP */ -{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } break; case 169: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy898 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy394 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } break; case 170: /* type_name ::= TINYINT UNSIGNED */ -{ yymsp[-1].minor.yy898 = createDataType(TSDB_DATA_TYPE_UTINYINT); } +{ yymsp[-1].minor.yy394 = createDataType(TSDB_DATA_TYPE_UTINYINT); } break; case 171: /* type_name ::= SMALLINT UNSIGNED */ -{ yymsp[-1].minor.yy898 = createDataType(TSDB_DATA_TYPE_USMALLINT); } +{ yymsp[-1].minor.yy394 = createDataType(TSDB_DATA_TYPE_USMALLINT); } break; case 172: /* type_name ::= INT UNSIGNED */ -{ yymsp[-1].minor.yy898 = createDataType(TSDB_DATA_TYPE_UINT); } +{ yymsp[-1].minor.yy394 = createDataType(TSDB_DATA_TYPE_UINT); } break; case 173: /* type_name ::= BIGINT UNSIGNED */ -{ yymsp[-1].minor.yy898 = createDataType(TSDB_DATA_TYPE_UBIGINT); } +{ yymsp[-1].minor.yy394 = createDataType(TSDB_DATA_TYPE_UBIGINT); } break; case 174: /* type_name ::= JSON */ -{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_JSON); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_JSON); } break; case 175: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy898 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy394 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } break; case 176: /* type_name ::= MEDIUMBLOB */ -{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } break; case 177: /* type_name ::= BLOB */ -{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_BLOB); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_BLOB); } break; case 178: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy898 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy394 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } break; case 179: /* type_name ::= DECIMAL */ -{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 180: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy898 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +{ yymsp[-3].minor.yy394 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 181: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ -{ yymsp[-5].minor.yy898 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +{ yymsp[-5].minor.yy394 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 183: /* tags_def_opt ::= tags_def */ - case 407: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==407); + case 418: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==418); { yylhsminor.yy874 = yymsp[0].minor.yy874; } yymsp[0].minor.yy874 = yylhsminor.yy874; break; @@ -4257,74 +4298,74 @@ static YYACTIONTYPE yy_reduce( { yymsp[-3].minor.yy874 = yymsp[-1].minor.yy874; } break; case 185: /* table_options ::= */ -{ yymsp[1].minor.yy74 = createDefaultTableOptions(pCxt); } +{ yymsp[1].minor.yy602 = createDefaultTableOptions(pCxt); } break; case 186: /* table_options ::= table_options COMMENT NK_STRING */ -{ yylhsminor.yy74 = setTableOption(pCxt, yymsp[-2].minor.yy74, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setTableOption(pCxt, yymsp[-2].minor.yy602, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 187: /* table_options ::= table_options MAX_DELAY duration_list */ -{ yylhsminor.yy74 = setTableOption(pCxt, yymsp[-2].minor.yy74, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy874); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setTableOption(pCxt, yymsp[-2].minor.yy602, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy874); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 188: /* table_options ::= table_options WATERMARK duration_list */ -{ yylhsminor.yy74 = setTableOption(pCxt, yymsp[-2].minor.yy74, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy874); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setTableOption(pCxt, yymsp[-2].minor.yy602, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy874); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 189: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ -{ yylhsminor.yy74 = setTableOption(pCxt, yymsp[-4].minor.yy74, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy874); } - yymsp[-4].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setTableOption(pCxt, yymsp[-4].minor.yy602, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy874); } + yymsp[-4].minor.yy602 = yylhsminor.yy602; break; case 190: /* table_options ::= table_options TTL NK_INTEGER */ -{ yylhsminor.yy74 = setTableOption(pCxt, yymsp[-2].minor.yy74, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setTableOption(pCxt, yymsp[-2].minor.yy602, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 191: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ -{ yylhsminor.yy74 = setTableOption(pCxt, yymsp[-4].minor.yy74, TABLE_OPTION_SMA, yymsp[-1].minor.yy874); } - yymsp[-4].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setTableOption(pCxt, yymsp[-4].minor.yy602, TABLE_OPTION_SMA, yymsp[-1].minor.yy874); } + yymsp[-4].minor.yy602 = yylhsminor.yy602; break; case 192: /* table_options ::= table_options DELETE_MARK duration_list */ -{ yylhsminor.yy74 = setTableOption(pCxt, yymsp[-2].minor.yy74, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy874); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setTableOption(pCxt, yymsp[-2].minor.yy602, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy874); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 193: /* alter_table_options ::= alter_table_option */ -{ yylhsminor.yy74 = createAlterTableOptions(pCxt); yylhsminor.yy74 = setTableOption(pCxt, yylhsminor.yy74, yymsp[0].minor.yy767.type, &yymsp[0].minor.yy767.val); } - yymsp[0].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = createAlterTableOptions(pCxt); yylhsminor.yy602 = setTableOption(pCxt, yylhsminor.yy602, yymsp[0].minor.yy845.type, &yymsp[0].minor.yy845.val); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; case 194: /* alter_table_options ::= alter_table_options alter_table_option */ -{ yylhsminor.yy74 = setTableOption(pCxt, yymsp[-1].minor.yy74, yymsp[0].minor.yy767.type, &yymsp[0].minor.yy767.val); } - yymsp[-1].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = setTableOption(pCxt, yymsp[-1].minor.yy602, yymsp[0].minor.yy845.type, &yymsp[0].minor.yy845.val); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; case 195: /* alter_table_option ::= COMMENT NK_STRING */ -{ yymsp[-1].minor.yy767.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy767.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy845.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy845.val = yymsp[0].minor.yy0; } break; case 196: /* alter_table_option ::= TTL NK_INTEGER */ -{ yymsp[-1].minor.yy767.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy767.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy845.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy845.val = yymsp[0].minor.yy0; } break; case 197: /* duration_list ::= duration_literal */ - case 372: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==372); -{ yylhsminor.yy874 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy74)); } + case 382: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==382); +{ yylhsminor.yy874 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy602)); } yymsp[0].minor.yy874 = yylhsminor.yy874; break; case 198: /* duration_list ::= duration_list NK_COMMA duration_literal */ - case 373: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==373); -{ yylhsminor.yy874 = addNodeToList(pCxt, yymsp[-2].minor.yy874, releaseRawExprNode(pCxt, yymsp[0].minor.yy74)); } + case 383: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==383); +{ yylhsminor.yy874 = addNodeToList(pCxt, yymsp[-2].minor.yy874, releaseRawExprNode(pCxt, yymsp[0].minor.yy602)); } yymsp[-2].minor.yy874 = yylhsminor.yy874; break; case 201: /* rollup_func_name ::= function_name */ -{ yylhsminor.yy74 = createFunctionNode(pCxt, &yymsp[0].minor.yy317, NULL); } - yymsp[0].minor.yy74 = yylhsminor.yy74; +{ yylhsminor.yy602 = createFunctionNode(pCxt, &yymsp[0].minor.yy179, NULL); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; case 202: /* rollup_func_name ::= FIRST */ case 203: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==203); - case 256: /* tag_item ::= QTAGS */ yytestcase(yyruleno==256); -{ yylhsminor.yy74 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } - yymsp[0].minor.yy74 = yylhsminor.yy74; + case 258: /* tag_item ::= QTAGS */ yytestcase(yyruleno==258); +{ yylhsminor.yy602 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; case 206: /* col_name ::= column_name */ - case 257: /* tag_item ::= column_name */ yytestcase(yyruleno==257); -{ yylhsminor.yy74 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy317); } - yymsp[0].minor.yy74 = yylhsminor.yy74; + case 259: /* tag_item ::= column_name */ yytestcase(yyruleno==259); +{ yylhsminor.yy602 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy179); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; case 207: /* cmd ::= SHOW DNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT); } @@ -4339,13 +4380,13 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT); } break; case 211: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy74, yymsp[0].minor.yy74, OP_TYPE_LIKE); } +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy602, yymsp[0].minor.yy602, OP_TYPE_LIKE); } break; case 212: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy74, yymsp[0].minor.yy74, OP_TYPE_LIKE); } +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy602, yymsp[0].minor.yy602, OP_TYPE_LIKE); } break; case 213: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy74, NULL, OP_TYPE_LIKE); } +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy602, NULL, OP_TYPE_LIKE); } break; case 214: /* cmd ::= SHOW MNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT); } @@ -4357,7 +4398,7 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT); } break; case 217: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy74, yymsp[-1].minor.yy74, OP_TYPE_EQUAL); } +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy602, yymsp[-1].minor.yy602, OP_TYPE_EQUAL); } break; case 218: /* cmd ::= SHOW STREAMS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT); } @@ -4376,13 +4417,13 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCES_STMT); } break; case 224: /* cmd ::= SHOW CREATE DATABASE db_name */ -{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy317); } +{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy179); } break; case 225: /* cmd ::= SHOW CREATE TABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy74); } +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy602); } break; case 226: /* cmd ::= SHOW CREATE STABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy74); } +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy602); } break; case 227: /* cmd ::= SHOW QUERIES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT); } @@ -4401,7 +4442,7 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT); } break; case 233: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy74); } +{ pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy602); } break; case 234: /* cmd ::= SHOW BNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_BNODES_STMT); } @@ -4416,7 +4457,7 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TRANSACTIONS_STMT); } break; case 238: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ -{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy74); } +{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy602); } break; case 239: /* cmd ::= SHOW CONSUMERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONSUMERS_STMT); } @@ -4425,10 +4466,10 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT); } break; case 241: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy74, yymsp[-1].minor.yy74, OP_TYPE_EQUAL); } +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy602, yymsp[-1].minor.yy602, OP_TYPE_EQUAL); } break; case 242: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy74, yymsp[0].minor.yy74, yymsp[-3].minor.yy874); } +{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy602, yymsp[0].minor.yy602, yymsp[-3].minor.yy874); } break; case 243: /* cmd ::= SHOW VNODES NK_INTEGER */ { pCxt->pRootNode = createShowVnodesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0), NULL); } @@ -4436,743 +4477,756 @@ static YYACTIONTYPE yy_reduce( case 244: /* cmd ::= SHOW VNODES NK_STRING */ { pCxt->pRootNode = createShowVnodesStmt(pCxt, NULL, createValueNode(pCxt, TSDB_DATA_TYPE_VARCHAR, &yymsp[0].minor.yy0)); } break; - case 245: /* db_name_cond_opt ::= */ - case 250: /* from_db_opt ::= */ yytestcase(yyruleno==250); -{ yymsp[1].minor.yy74 = createDefaultDatabaseCondValue(pCxt); } + case 245: /* cmd ::= SHOW db_name_cond_opt ALIVE */ +{ pCxt->pRootNode = createShowAliveStmt(pCxt, yymsp[-1].minor.yy602, QUERY_NODE_SHOW_DB_ALIVE_STMT); } break; - case 246: /* db_name_cond_opt ::= db_name NK_DOT */ -{ yylhsminor.yy74 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy317); } - yymsp[-1].minor.yy74 = yylhsminor.yy74; + case 246: /* cmd ::= SHOW CLUSTER ALIVE */ +{ pCxt->pRootNode = createShowAliveStmt(pCxt, NULL, QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT); } break; - case 247: /* like_pattern_opt ::= */ - case 304: /* subtable_opt ::= */ yytestcase(yyruleno==304); - case 417: /* case_when_else_opt ::= */ yytestcase(yyruleno==417); - case 447: /* from_clause_opt ::= */ yytestcase(yyruleno==447); - case 476: /* where_clause_opt ::= */ yytestcase(yyruleno==476); - case 485: /* twindow_clause_opt ::= */ yytestcase(yyruleno==485); - case 490: /* sliding_opt ::= */ yytestcase(yyruleno==490); - case 492: /* fill_opt ::= */ yytestcase(yyruleno==492); - case 504: /* having_clause_opt ::= */ yytestcase(yyruleno==504); - case 506: /* range_opt ::= */ yytestcase(yyruleno==506); - case 508: /* every_opt ::= */ yytestcase(yyruleno==508); - case 521: /* slimit_clause_opt ::= */ yytestcase(yyruleno==521); - case 525: /* limit_clause_opt ::= */ yytestcase(yyruleno==525); -{ yymsp[1].minor.yy74 = NULL; } + case 247: /* db_name_cond_opt ::= */ + case 252: /* from_db_opt ::= */ yytestcase(yyruleno==252); +{ yymsp[1].minor.yy602 = createDefaultDatabaseCondValue(pCxt); } break; - case 248: /* like_pattern_opt ::= LIKE NK_STRING */ -{ yymsp[-1].minor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + case 248: /* db_name_cond_opt ::= db_name NK_DOT */ +{ yylhsminor.yy602 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy179); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; - case 249: /* table_name_cond ::= table_name */ -{ yylhsminor.yy74 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy317); } - yymsp[0].minor.yy74 = yylhsminor.yy74; + case 249: /* like_pattern_opt ::= */ + case 314: /* subtable_opt ::= */ yytestcase(yyruleno==314); + case 428: /* case_when_else_opt ::= */ yytestcase(yyruleno==428); + case 458: /* from_clause_opt ::= */ yytestcase(yyruleno==458); + case 487: /* where_clause_opt ::= */ yytestcase(yyruleno==487); + case 496: /* twindow_clause_opt ::= */ yytestcase(yyruleno==496); + case 502: /* sliding_opt ::= */ yytestcase(yyruleno==502); + case 504: /* fill_opt ::= */ yytestcase(yyruleno==504); + case 516: /* having_clause_opt ::= */ yytestcase(yyruleno==516); + case 518: /* range_opt ::= */ yytestcase(yyruleno==518); + case 520: /* every_opt ::= */ yytestcase(yyruleno==520); + case 533: /* slimit_clause_opt ::= */ yytestcase(yyruleno==533); + case 537: /* limit_clause_opt ::= */ yytestcase(yyruleno==537); +{ yymsp[1].minor.yy602 = NULL; } break; - case 251: /* from_db_opt ::= FROM db_name */ -{ yymsp[-1].minor.yy74 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy317); } + case 250: /* like_pattern_opt ::= LIKE NK_STRING */ +{ yymsp[-1].minor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } break; - case 255: /* tag_item ::= TBNAME */ -{ yylhsminor.yy74 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } - yymsp[0].minor.yy74 = yylhsminor.yy74; + case 251: /* table_name_cond ::= table_name */ +{ yylhsminor.yy602 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy179); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; - case 258: /* tag_item ::= column_name column_alias */ -{ yylhsminor.yy74 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy317), &yymsp[0].minor.yy317); } - yymsp[-1].minor.yy74 = yylhsminor.yy74; + case 253: /* from_db_opt ::= FROM db_name */ +{ yymsp[-1].minor.yy602 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy179); } break; - case 259: /* tag_item ::= column_name AS column_alias */ -{ yylhsminor.yy74 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy317), &yymsp[0].minor.yy317); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + case 257: /* tag_item ::= TBNAME */ +{ yylhsminor.yy602 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; - case 260: /* cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy335, yymsp[-3].minor.yy74, yymsp[-1].minor.yy74, NULL, yymsp[0].minor.yy74); } + case 260: /* tag_item ::= column_name column_alias */ +{ yylhsminor.yy602 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy179), &yymsp[0].minor.yy179); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; - case 261: /* cmd ::= DROP INDEX exists_opt full_index_name */ -{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy335, yymsp[0].minor.yy74); } + case 261: /* tag_item ::= column_name AS column_alias */ +{ yylhsminor.yy602 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy179), &yymsp[0].minor.yy179); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 262: /* full_index_name ::= index_name */ -{ yylhsminor.yy74 = createRealTableNodeForIndexName(pCxt, NULL, &yymsp[0].minor.yy317); } - yymsp[0].minor.yy74 = yylhsminor.yy74; + case 262: /* cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy767, yymsp[-3].minor.yy602, yymsp[-1].minor.yy602, NULL, yymsp[0].minor.yy602); } break; - case 263: /* full_index_name ::= db_name NK_DOT index_name */ -{ yylhsminor.yy74 = createRealTableNodeForIndexName(pCxt, &yymsp[-2].minor.yy317, &yymsp[0].minor.yy317); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + case 263: /* cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_NORMAL, yymsp[-6].minor.yy767, yymsp[-5].minor.yy602, yymsp[-3].minor.yy602, yymsp[-1].minor.yy874, NULL); } break; - case 264: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ -{ yymsp[-9].minor.yy74 = createIndexOption(pCxt, yymsp[-7].minor.yy874, releaseRawExprNode(pCxt, yymsp[-3].minor.yy74), NULL, yymsp[-1].minor.yy74, yymsp[0].minor.yy74); } + case 264: /* cmd ::= DROP INDEX exists_opt full_index_name */ +{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy767, yymsp[0].minor.yy602); } break; - case 265: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ -{ yymsp[-11].minor.yy74 = createIndexOption(pCxt, yymsp[-9].minor.yy874, releaseRawExprNode(pCxt, yymsp[-5].minor.yy74), releaseRawExprNode(pCxt, yymsp[-3].minor.yy74), yymsp[-1].minor.yy74, yymsp[0].minor.yy74); } + case 265: /* full_index_name ::= index_name */ +{ yylhsminor.yy602 = createRealTableNodeForIndexName(pCxt, NULL, &yymsp[0].minor.yy179); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; - case 268: /* func ::= function_name NK_LP expression_list NK_RP */ -{ yylhsminor.yy74 = createFunctionNode(pCxt, &yymsp[-3].minor.yy317, yymsp[-1].minor.yy874); } - yymsp[-3].minor.yy74 = yylhsminor.yy74; + case 266: /* full_index_name ::= db_name NK_DOT index_name */ +{ yylhsminor.yy602 = createRealTableNodeForIndexName(pCxt, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy179); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 269: /* sma_stream_opt ::= */ - case 297: /* stream_options ::= */ yytestcase(yyruleno==297); -{ yymsp[1].minor.yy74 = createStreamOptions(pCxt); } + case 267: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ +{ yymsp[-9].minor.yy602 = createIndexOption(pCxt, yymsp[-7].minor.yy874, releaseRawExprNode(pCxt, yymsp[-3].minor.yy602), NULL, yymsp[-1].minor.yy602, yymsp[0].minor.yy602); } break; - case 270: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ - case 301: /* stream_options ::= stream_options WATERMARK duration_literal */ yytestcase(yyruleno==301); -{ ((SStreamOptions*)yymsp[-2].minor.yy74)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy74); yylhsminor.yy74 = yymsp[-2].minor.yy74; } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + case 268: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ +{ yymsp[-11].minor.yy602 = createIndexOption(pCxt, yymsp[-9].minor.yy874, releaseRawExprNode(pCxt, yymsp[-5].minor.yy602), releaseRawExprNode(pCxt, yymsp[-3].minor.yy602), yymsp[-1].minor.yy602, yymsp[0].minor.yy602); } break; - case 271: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy74)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy74); yylhsminor.yy74 = yymsp[-2].minor.yy74; } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + case 271: /* func ::= sma_func_name NK_LP expression_list NK_RP */ +{ yylhsminor.yy602 = createFunctionNode(pCxt, &yymsp[-3].minor.yy179, yymsp[-1].minor.yy874); } + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; - case 272: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy74)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy74); yylhsminor.yy74 = yymsp[-2].minor.yy74; } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + case 277: /* sma_stream_opt ::= */ + case 307: /* stream_options ::= */ yytestcase(yyruleno==307); +{ yymsp[1].minor.yy602 = createStreamOptions(pCxt); } break; - case 273: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ -{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy335, &yymsp[-2].minor.yy317, yymsp[0].minor.yy74); } + case 278: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ + case 311: /* stream_options ::= stream_options WATERMARK duration_literal */ yytestcase(yyruleno==311); +{ ((SStreamOptions*)yymsp[-2].minor.yy602)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy602); yylhsminor.yy602 = yymsp[-2].minor.yy602; } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 274: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy335, &yymsp[-3].minor.yy317, &yymsp[0].minor.yy317, false); } + case 279: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy602)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy602); yylhsminor.yy602 = yymsp[-2].minor.yy602; } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 275: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy335, &yymsp[-5].minor.yy317, &yymsp[0].minor.yy317, true); } + case 280: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy602)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy602); yylhsminor.yy602 = yymsp[-2].minor.yy602; } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 276: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-4].minor.yy335, &yymsp[-3].minor.yy317, yymsp[0].minor.yy74, false); } + case 281: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ +{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy767, &yymsp[-2].minor.yy179, yymsp[0].minor.yy602); } break; - case 277: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-6].minor.yy335, &yymsp[-5].minor.yy317, yymsp[0].minor.yy74, true); } + case 282: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy767, &yymsp[-3].minor.yy179, &yymsp[0].minor.yy179, false); } break; - case 278: /* cmd ::= DROP TOPIC exists_opt topic_name */ -{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy335, &yymsp[0].minor.yy317); } + case 283: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy767, &yymsp[-5].minor.yy179, &yymsp[0].minor.yy179, true); } break; - case 279: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ -{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy335, &yymsp[-2].minor.yy317, &yymsp[0].minor.yy317); } + case 284: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-4].minor.yy767, &yymsp[-3].minor.yy179, yymsp[0].minor.yy602, false); } break; - case 280: /* cmd ::= DESC full_table_name */ - case 281: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==281); -{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy74); } + case 285: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-6].minor.yy767, &yymsp[-5].minor.yy179, yymsp[0].minor.yy602, true); } break; - case 282: /* cmd ::= RESET QUERY CACHE */ + case 286: /* cmd ::= DROP TOPIC exists_opt topic_name */ +{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy767, &yymsp[0].minor.yy179); } + break; + case 287: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ +{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy767, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy179); } + break; + case 288: /* cmd ::= DESC full_table_name */ + case 289: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==289); +{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy602); } + break; + case 290: /* cmd ::= RESET QUERY CACHE */ { pCxt->pRootNode = createResetQueryCacheStmt(pCxt); } break; - case 283: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ -{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy335, yymsp[-1].minor.yy74, yymsp[0].minor.yy74); } + case 291: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ +{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy767, yymsp[-1].minor.yy602, yymsp[0].minor.yy602); } break; - case 286: /* explain_options ::= */ -{ yymsp[1].minor.yy74 = createDefaultExplainOptions(pCxt); } + case 294: /* explain_options ::= */ +{ yymsp[1].minor.yy602 = createDefaultExplainOptions(pCxt); } break; - case 287: /* explain_options ::= explain_options VERBOSE NK_BOOL */ -{ yylhsminor.yy74 = setExplainVerbose(pCxt, yymsp[-2].minor.yy74, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + case 295: /* explain_options ::= explain_options VERBOSE NK_BOOL */ +{ yylhsminor.yy602 = setExplainVerbose(pCxt, yymsp[-2].minor.yy602, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 288: /* explain_options ::= explain_options RATIO NK_FLOAT */ -{ yylhsminor.yy74 = setExplainRatio(pCxt, yymsp[-2].minor.yy74, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + case 296: /* explain_options ::= explain_options RATIO NK_FLOAT */ +{ yylhsminor.yy602 = setExplainRatio(pCxt, yymsp[-2].minor.yy602, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 289: /* cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ -{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy335, yymsp[-8].minor.yy335, &yymsp[-5].minor.yy317, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy898, yymsp[0].minor.yy856); } + case 297: /* cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ +{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy767, yymsp[-8].minor.yy767, &yymsp[-5].minor.yy179, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy394, yymsp[0].minor.yy820); } break; - case 290: /* cmd ::= DROP FUNCTION exists_opt function_name */ -{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy335, &yymsp[0].minor.yy317); } + case 298: /* cmd ::= DROP FUNCTION exists_opt function_name */ +{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy767, &yymsp[0].minor.yy179); } break; - case 295: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name tags_def_opt subtable_opt AS query_or_subquery */ -{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-8].minor.yy335, &yymsp[-7].minor.yy317, yymsp[-4].minor.yy74, yymsp[-6].minor.yy74, yymsp[-3].minor.yy874, yymsp[-2].minor.yy74, yymsp[0].minor.yy74); } + case 303: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tags_def_opt subtable_opt AS query_or_subquery */ +{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-9].minor.yy767, &yymsp[-8].minor.yy179, yymsp[-5].minor.yy602, yymsp[-7].minor.yy602, yymsp[-3].minor.yy874, yymsp[-2].minor.yy602, yymsp[0].minor.yy602, yymsp[-4].minor.yy874); } break; - case 296: /* cmd ::= DROP STREAM exists_opt stream_name */ -{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy335, &yymsp[0].minor.yy317); } + case 304: /* cmd ::= DROP STREAM exists_opt stream_name */ +{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy767, &yymsp[0].minor.yy179); } break; - case 298: /* stream_options ::= stream_options TRIGGER AT_ONCE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy74)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy74 = yymsp[-2].minor.yy74; } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + case 308: /* stream_options ::= stream_options TRIGGER AT_ONCE */ +{ ((SStreamOptions*)yymsp[-2].minor.yy602)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy602 = yymsp[-2].minor.yy602; } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 299: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy74)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy74 = yymsp[-2].minor.yy74; } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + case 309: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ +{ ((SStreamOptions*)yymsp[-2].minor.yy602)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy602 = yymsp[-2].minor.yy602; } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 300: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ -{ ((SStreamOptions*)yymsp[-3].minor.yy74)->triggerType = STREAM_TRIGGER_MAX_DELAY; ((SStreamOptions*)yymsp[-3].minor.yy74)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy74); yylhsminor.yy74 = yymsp[-3].minor.yy74; } - yymsp[-3].minor.yy74 = yylhsminor.yy74; + case 310: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ +{ ((SStreamOptions*)yymsp[-3].minor.yy602)->triggerType = STREAM_TRIGGER_MAX_DELAY; ((SStreamOptions*)yymsp[-3].minor.yy602)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy602); yylhsminor.yy602 = yymsp[-3].minor.yy602; } + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; - case 302: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ -{ ((SStreamOptions*)yymsp[-3].minor.yy74)->ignoreExpired = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy74 = yymsp[-3].minor.yy74; } - yymsp[-3].minor.yy74 = yylhsminor.yy74; + case 312: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ +{ ((SStreamOptions*)yymsp[-3].minor.yy602)->ignoreExpired = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy602 = yymsp[-3].minor.yy602; } + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; - case 303: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ -{ ((SStreamOptions*)yymsp[-2].minor.yy74)->fillHistory = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy74 = yymsp[-2].minor.yy74; } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + case 313: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ +{ ((SStreamOptions*)yymsp[-2].minor.yy602)->fillHistory = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy602 = yymsp[-2].minor.yy602; } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 305: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ - case 491: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==491); - case 509: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==509); -{ yymsp[-3].minor.yy74 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy74); } + case 315: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ + case 503: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==503); + case 521: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==521); +{ yymsp[-3].minor.yy602 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy602); } break; - case 306: /* cmd ::= KILL CONNECTION NK_INTEGER */ + case 316: /* cmd ::= KILL CONNECTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &yymsp[0].minor.yy0); } break; - case 307: /* cmd ::= KILL QUERY NK_STRING */ + case 317: /* cmd ::= KILL QUERY NK_STRING */ { pCxt->pRootNode = createKillQueryStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 308: /* cmd ::= KILL TRANSACTION NK_INTEGER */ + case 318: /* cmd ::= KILL TRANSACTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_TRANSACTION_STMT, &yymsp[0].minor.yy0); } break; - case 309: /* cmd ::= BALANCE VGROUP */ + case 319: /* cmd ::= BALANCE VGROUP */ { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); } break; - case 310: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + case 320: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 311: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ + case 321: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ { pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy874); } break; - case 312: /* cmd ::= SPLIT VGROUP NK_INTEGER */ + case 322: /* cmd ::= SPLIT VGROUP NK_INTEGER */ { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 313: /* dnode_list ::= DNODE NK_INTEGER */ + case 323: /* dnode_list ::= DNODE NK_INTEGER */ { yymsp[-1].minor.yy874 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } break; - case 315: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ -{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy74, yymsp[0].minor.yy74); } - break; - case 317: /* cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ -{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-4].minor.yy74, yymsp[-2].minor.yy874, yymsp[0].minor.yy74); } - break; - case 318: /* cmd ::= INSERT INTO full_table_name query_or_subquery */ -{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-1].minor.yy74, NULL, yymsp[0].minor.yy74); } - break; - case 319: /* literal ::= NK_INTEGER */ -{ yylhsminor.yy74 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy74 = yylhsminor.yy74; - break; - case 320: /* literal ::= NK_FLOAT */ -{ yylhsminor.yy74 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy74 = yylhsminor.yy74; - break; - case 321: /* literal ::= NK_STRING */ -{ yylhsminor.yy74 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy74 = yylhsminor.yy74; - break; - case 322: /* literal ::= NK_BOOL */ -{ yylhsminor.yy74 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy74 = yylhsminor.yy74; - break; - case 323: /* literal ::= TIMESTAMP NK_STRING */ -{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } - yymsp[-1].minor.yy74 = yylhsminor.yy74; - break; - case 324: /* literal ::= duration_literal */ - case 334: /* signed_literal ::= signed */ yytestcase(yyruleno==334); - case 355: /* expr_or_subquery ::= expression */ yytestcase(yyruleno==355); - case 356: /* expression ::= literal */ yytestcase(yyruleno==356); - case 357: /* expression ::= pseudo_column */ yytestcase(yyruleno==357); - case 358: /* expression ::= column_reference */ yytestcase(yyruleno==358); - case 359: /* expression ::= function_expression */ yytestcase(yyruleno==359); - case 360: /* expression ::= case_when_expression */ yytestcase(yyruleno==360); - case 390: /* function_expression ::= literal_func */ yytestcase(yyruleno==390); - case 439: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==439); - case 443: /* boolean_primary ::= predicate */ yytestcase(yyruleno==443); - case 445: /* common_expression ::= expr_or_subquery */ yytestcase(yyruleno==445); - case 446: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==446); - case 449: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==449); - case 451: /* table_reference ::= table_primary */ yytestcase(yyruleno==451); - case 452: /* table_reference ::= joined_table */ yytestcase(yyruleno==452); - case 456: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==456); - case 511: /* query_simple ::= query_specification */ yytestcase(yyruleno==511); - case 512: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==512); - case 515: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==515); - case 517: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==517); -{ yylhsminor.yy74 = yymsp[0].minor.yy74; } - yymsp[0].minor.yy74 = yylhsminor.yy74; - break; - case 325: /* literal ::= NULL */ -{ yylhsminor.yy74 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy74 = yylhsminor.yy74; - break; - case 326: /* literal ::= NK_QUESTION */ -{ yylhsminor.yy74 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy74 = yylhsminor.yy74; - break; - case 327: /* duration_literal ::= NK_VARIABLE */ -{ yylhsminor.yy74 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy74 = yylhsminor.yy74; - break; - case 328: /* signed ::= NK_INTEGER */ -{ yylhsminor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy74 = yylhsminor.yy74; - break; - case 329: /* signed ::= NK_PLUS NK_INTEGER */ -{ yymsp[-1].minor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } - break; - case 330: /* signed ::= NK_MINUS NK_INTEGER */ + case 325: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ +{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy602, yymsp[0].minor.yy602); } + break; + case 327: /* cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ +{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-4].minor.yy602, yymsp[-2].minor.yy874, yymsp[0].minor.yy602); } + break; + case 328: /* cmd ::= INSERT INTO full_table_name query_or_subquery */ +{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-1].minor.yy602, NULL, yymsp[0].minor.yy602); } + break; + case 329: /* literal ::= NK_INTEGER */ +{ yylhsminor.yy602 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 330: /* literal ::= NK_FLOAT */ +{ yylhsminor.yy602 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 331: /* literal ::= NK_STRING */ +{ yylhsminor.yy602 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 332: /* literal ::= NK_BOOL */ +{ yylhsminor.yy602 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 333: /* literal ::= TIMESTAMP NK_STRING */ +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; + break; + case 334: /* literal ::= duration_literal */ + case 344: /* signed_literal ::= signed */ yytestcase(yyruleno==344); + case 365: /* expr_or_subquery ::= expression */ yytestcase(yyruleno==365); + case 366: /* expression ::= literal */ yytestcase(yyruleno==366); + case 367: /* expression ::= pseudo_column */ yytestcase(yyruleno==367); + case 368: /* expression ::= column_reference */ yytestcase(yyruleno==368); + case 369: /* expression ::= function_expression */ yytestcase(yyruleno==369); + case 370: /* expression ::= case_when_expression */ yytestcase(yyruleno==370); + case 401: /* function_expression ::= literal_func */ yytestcase(yyruleno==401); + case 450: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==450); + case 454: /* boolean_primary ::= predicate */ yytestcase(yyruleno==454); + case 456: /* common_expression ::= expr_or_subquery */ yytestcase(yyruleno==456); + case 457: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==457); + case 460: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==460); + case 462: /* table_reference ::= table_primary */ yytestcase(yyruleno==462); + case 463: /* table_reference ::= joined_table */ yytestcase(yyruleno==463); + case 467: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==467); + case 523: /* query_simple ::= query_specification */ yytestcase(yyruleno==523); + case 524: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==524); + case 527: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==527); + case 529: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==529); +{ yylhsminor.yy602 = yymsp[0].minor.yy602; } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 335: /* literal ::= NULL */ +{ yylhsminor.yy602 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 336: /* literal ::= NK_QUESTION */ +{ yylhsminor.yy602 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 337: /* duration_literal ::= NK_VARIABLE */ +{ yylhsminor.yy602 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 338: /* signed ::= NK_INTEGER */ +{ yylhsminor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 339: /* signed ::= NK_PLUS NK_INTEGER */ +{ yymsp[-1].minor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } + break; + case 340: /* signed ::= NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); + yylhsminor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } - yymsp[-1].minor.yy74 = yylhsminor.yy74; + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; - case 331: /* signed ::= NK_FLOAT */ -{ yylhsminor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy74 = yylhsminor.yy74; + case 341: /* signed ::= NK_FLOAT */ +{ yylhsminor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; - case 332: /* signed ::= NK_PLUS NK_FLOAT */ -{ yymsp[-1].minor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + case 342: /* signed ::= NK_PLUS NK_FLOAT */ +{ yymsp[-1].minor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } break; - case 333: /* signed ::= NK_MINUS NK_FLOAT */ + case 343: /* signed ::= NK_MINUS NK_FLOAT */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); + yylhsminor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } - yymsp[-1].minor.yy74 = yylhsminor.yy74; - break; - case 335: /* signed_literal ::= NK_STRING */ -{ yylhsminor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy74 = yylhsminor.yy74; - break; - case 336: /* signed_literal ::= NK_BOOL */ -{ yylhsminor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy74 = yylhsminor.yy74; - break; - case 337: /* signed_literal ::= TIMESTAMP NK_STRING */ -{ yymsp[-1].minor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } - break; - case 338: /* signed_literal ::= duration_literal */ - case 340: /* signed_literal ::= literal_func */ yytestcase(yyruleno==340); - case 410: /* star_func_para ::= expr_or_subquery */ yytestcase(yyruleno==410); - case 472: /* select_item ::= common_expression */ yytestcase(yyruleno==472); - case 482: /* partition_item ::= expr_or_subquery */ yytestcase(yyruleno==482); - case 516: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==516); - case 518: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==518); - case 531: /* search_condition ::= common_expression */ yytestcase(yyruleno==531); -{ yylhsminor.yy74 = releaseRawExprNode(pCxt, yymsp[0].minor.yy74); } - yymsp[0].minor.yy74 = yylhsminor.yy74; - break; - case 339: /* signed_literal ::= NULL */ -{ yylhsminor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy74 = yylhsminor.yy74; - break; - case 341: /* signed_literal ::= NK_QUESTION */ -{ yylhsminor.yy74 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy74 = yylhsminor.yy74; - break; - case 361: /* expression ::= NK_LP expression NK_RP */ - case 444: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==444); - case 530: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==530); -{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy74)); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; - break; - case 362: /* expression ::= NK_PLUS expr_or_subquery */ + yymsp[-1].minor.yy602 = yylhsminor.yy602; + break; + case 345: /* signed_literal ::= NK_STRING */ +{ yylhsminor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 346: /* signed_literal ::= NK_BOOL */ +{ yylhsminor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 347: /* signed_literal ::= TIMESTAMP NK_STRING */ +{ yymsp[-1].minor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } + break; + case 348: /* signed_literal ::= duration_literal */ + case 350: /* signed_literal ::= literal_func */ yytestcase(yyruleno==350); + case 421: /* star_func_para ::= expr_or_subquery */ yytestcase(yyruleno==421); + case 483: /* select_item ::= common_expression */ yytestcase(yyruleno==483); + case 493: /* partition_item ::= expr_or_subquery */ yytestcase(yyruleno==493); + case 528: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==528); + case 530: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==530); + case 543: /* search_condition ::= common_expression */ yytestcase(yyruleno==543); +{ yylhsminor.yy602 = releaseRawExprNode(pCxt, yymsp[0].minor.yy602); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 349: /* signed_literal ::= NULL */ +{ yylhsminor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 351: /* signed_literal ::= NK_QUESTION */ +{ yylhsminor.yy602 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 371: /* expression ::= NK_LP expression NK_RP */ + case 455: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==455); + case 542: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==542); +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy602)); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; + break; + case 372: /* expression ::= NK_PLUS expr_or_subquery */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); - yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy74)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy602)); } - yymsp[-1].minor.yy74 = yylhsminor.yy74; + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; - case 363: /* expression ::= NK_MINUS expr_or_subquery */ + case 373: /* expression ::= NK_MINUS expr_or_subquery */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); - yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy74), NULL)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy602), NULL)); } - yymsp[-1].minor.yy74 = yylhsminor.yy74; + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; - case 364: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ + case 374: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); - yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 365: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ + case 375: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); - yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 366: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ + case 376: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); - yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 367: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ + case 377: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); - yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 368: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ + case 378: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); - yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 369: /* expression ::= column_reference NK_ARROW NK_STRING */ + case 379: /* expression ::= column_reference NK_ARROW NK_STRING */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); - yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 370: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ + case 380: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); - yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 371: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ + case 381: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); - yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; - break; - case 374: /* column_reference ::= column_name */ -{ yylhsminor.yy74 = createRawExprNode(pCxt, &yymsp[0].minor.yy317, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy317)); } - yymsp[0].minor.yy74 = yylhsminor.yy74; - break; - case 375: /* column_reference ::= table_name NK_DOT column_name */ -{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy317, &yymsp[0].minor.yy317, createColumnNode(pCxt, &yymsp[-2].minor.yy317, &yymsp[0].minor.yy317)); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; - break; - case 376: /* pseudo_column ::= ROWTS */ - case 377: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==377); - case 379: /* pseudo_column ::= QSTART */ yytestcase(yyruleno==379); - case 380: /* pseudo_column ::= QEND */ yytestcase(yyruleno==380); - case 381: /* pseudo_column ::= QDURATION */ yytestcase(yyruleno==381); - case 382: /* pseudo_column ::= WSTART */ yytestcase(yyruleno==382); - case 383: /* pseudo_column ::= WEND */ yytestcase(yyruleno==383); - case 384: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==384); - case 385: /* pseudo_column ::= IROWTS */ yytestcase(yyruleno==385); - case 386: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==386); - case 392: /* literal_func ::= NOW */ yytestcase(yyruleno==392); -{ yylhsminor.yy74 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } - yymsp[0].minor.yy74 = yylhsminor.yy74; - break; - case 378: /* pseudo_column ::= table_name NK_DOT TBNAME */ -{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy317, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy317)))); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; - break; - case 387: /* function_expression ::= function_name NK_LP expression_list NK_RP */ - case 388: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==388); -{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy317, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy317, yymsp[-1].minor.yy874)); } - yymsp[-3].minor.yy74 = yylhsminor.yy74; - break; - case 389: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ -{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy74), yymsp[-1].minor.yy898)); } - yymsp[-5].minor.yy74 = yylhsminor.yy74; - break; - case 391: /* literal_func ::= noarg_func NK_LP NK_RP */ -{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy317, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy317, NULL)); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; - break; - case 406: /* star_func_para_list ::= NK_STAR */ + yymsp[-2].minor.yy602 = yylhsminor.yy602; + break; + case 384: /* column_reference ::= column_name */ +{ yylhsminor.yy602 = createRawExprNode(pCxt, &yymsp[0].minor.yy179, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy179)); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 385: /* column_reference ::= table_name NK_DOT column_name */ +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy179, createColumnNode(pCxt, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy179)); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; + break; + case 386: /* pseudo_column ::= ROWTS */ + case 387: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==387); + case 389: /* pseudo_column ::= QSTART */ yytestcase(yyruleno==389); + case 390: /* pseudo_column ::= QEND */ yytestcase(yyruleno==390); + case 391: /* pseudo_column ::= QDURATION */ yytestcase(yyruleno==391); + case 392: /* pseudo_column ::= WSTART */ yytestcase(yyruleno==392); + case 393: /* pseudo_column ::= WEND */ yytestcase(yyruleno==393); + case 394: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==394); + case 395: /* pseudo_column ::= IROWTS */ yytestcase(yyruleno==395); + case 396: /* pseudo_column ::= ISFILLED */ yytestcase(yyruleno==396); + case 397: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==397); + case 403: /* literal_func ::= NOW */ yytestcase(yyruleno==403); +{ yylhsminor.yy602 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 388: /* pseudo_column ::= table_name NK_DOT TBNAME */ +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy179)))); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; + break; + case 398: /* function_expression ::= function_name NK_LP expression_list NK_RP */ + case 399: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==399); +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy179, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy179, yymsp[-1].minor.yy874)); } + yymsp[-3].minor.yy602 = yylhsminor.yy602; + break; + case 400: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy602), yymsp[-1].minor.yy394)); } + yymsp[-5].minor.yy602 = yylhsminor.yy602; + break; + case 402: /* literal_func ::= noarg_func NK_LP NK_RP */ +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy179, NULL)); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; + break; + case 417: /* star_func_para_list ::= NK_STAR */ { yylhsminor.yy874 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } yymsp[0].minor.yy874 = yylhsminor.yy874; break; - case 411: /* star_func_para ::= table_name NK_DOT NK_STAR */ - case 475: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==475); -{ yylhsminor.yy74 = createColumnNode(pCxt, &yymsp[-2].minor.yy317, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + case 422: /* star_func_para ::= table_name NK_DOT NK_STAR */ + case 486: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==486); +{ yylhsminor.yy602 = createColumnNode(pCxt, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 412: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ -{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy874, yymsp[-1].minor.yy74)); } - yymsp[-3].minor.yy74 = yylhsminor.yy74; + case 423: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy874, yymsp[-1].minor.yy602)); } + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; - case 413: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ -{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy74), yymsp[-2].minor.yy874, yymsp[-1].minor.yy74)); } - yymsp[-4].minor.yy74 = yylhsminor.yy74; + case 424: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy602), yymsp[-2].minor.yy874, yymsp[-1].minor.yy602)); } + yymsp[-4].minor.yy602 = yylhsminor.yy602; break; - case 416: /* when_then_expr ::= WHEN common_expression THEN common_expression */ -{ yymsp[-3].minor.yy74 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74)); } + case 427: /* when_then_expr ::= WHEN common_expression THEN common_expression */ +{ yymsp[-3].minor.yy602 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602)); } break; - case 418: /* case_when_else_opt ::= ELSE common_expression */ -{ yymsp[-1].minor.yy74 = releaseRawExprNode(pCxt, yymsp[0].minor.yy74); } + case 429: /* case_when_else_opt ::= ELSE common_expression */ +{ yymsp[-1].minor.yy602 = releaseRawExprNode(pCxt, yymsp[0].minor.yy602); } break; - case 419: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ - case 424: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==424); + case 430: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ + case 435: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==435); { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); - yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy20, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy290, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 420: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ + case 431: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy74); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); - yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy74), releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy602), releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-4].minor.yy74 = yylhsminor.yy74; + yymsp[-4].minor.yy602 = yylhsminor.yy602; break; - case 421: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ + case 432: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy74); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); - yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy74), releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy602), releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-5].minor.yy74 = yylhsminor.yy74; + yymsp[-5].minor.yy602 = yylhsminor.yy602; break; - case 422: /* predicate ::= expr_or_subquery IS NULL */ + case 433: /* predicate ::= expr_or_subquery IS NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); - yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), NULL)); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 423: /* predicate ::= expr_or_subquery IS NOT NULL */ + case 434: /* predicate ::= expr_or_subquery IS NOT NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy74); - yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy74), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy602), NULL)); } - yymsp[-3].minor.yy74 = yylhsminor.yy74; + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; - case 425: /* compare_op ::= NK_LT */ -{ yymsp[0].minor.yy20 = OP_TYPE_LOWER_THAN; } + case 436: /* compare_op ::= NK_LT */ +{ yymsp[0].minor.yy290 = OP_TYPE_LOWER_THAN; } break; - case 426: /* compare_op ::= NK_GT */ -{ yymsp[0].minor.yy20 = OP_TYPE_GREATER_THAN; } + case 437: /* compare_op ::= NK_GT */ +{ yymsp[0].minor.yy290 = OP_TYPE_GREATER_THAN; } break; - case 427: /* compare_op ::= NK_LE */ -{ yymsp[0].minor.yy20 = OP_TYPE_LOWER_EQUAL; } + case 438: /* compare_op ::= NK_LE */ +{ yymsp[0].minor.yy290 = OP_TYPE_LOWER_EQUAL; } break; - case 428: /* compare_op ::= NK_GE */ -{ yymsp[0].minor.yy20 = OP_TYPE_GREATER_EQUAL; } + case 439: /* compare_op ::= NK_GE */ +{ yymsp[0].minor.yy290 = OP_TYPE_GREATER_EQUAL; } break; - case 429: /* compare_op ::= NK_NE */ -{ yymsp[0].minor.yy20 = OP_TYPE_NOT_EQUAL; } + case 440: /* compare_op ::= NK_NE */ +{ yymsp[0].minor.yy290 = OP_TYPE_NOT_EQUAL; } break; - case 430: /* compare_op ::= NK_EQ */ -{ yymsp[0].minor.yy20 = OP_TYPE_EQUAL; } + case 441: /* compare_op ::= NK_EQ */ +{ yymsp[0].minor.yy290 = OP_TYPE_EQUAL; } break; - case 431: /* compare_op ::= LIKE */ -{ yymsp[0].minor.yy20 = OP_TYPE_LIKE; } + case 442: /* compare_op ::= LIKE */ +{ yymsp[0].minor.yy290 = OP_TYPE_LIKE; } break; - case 432: /* compare_op ::= NOT LIKE */ -{ yymsp[-1].minor.yy20 = OP_TYPE_NOT_LIKE; } + case 443: /* compare_op ::= NOT LIKE */ +{ yymsp[-1].minor.yy290 = OP_TYPE_NOT_LIKE; } break; - case 433: /* compare_op ::= MATCH */ -{ yymsp[0].minor.yy20 = OP_TYPE_MATCH; } + case 444: /* compare_op ::= MATCH */ +{ yymsp[0].minor.yy290 = OP_TYPE_MATCH; } break; - case 434: /* compare_op ::= NMATCH */ -{ yymsp[0].minor.yy20 = OP_TYPE_NMATCH; } + case 445: /* compare_op ::= NMATCH */ +{ yymsp[0].minor.yy290 = OP_TYPE_NMATCH; } break; - case 435: /* compare_op ::= CONTAINS */ -{ yymsp[0].minor.yy20 = OP_TYPE_JSON_CONTAINS; } + case 446: /* compare_op ::= CONTAINS */ +{ yymsp[0].minor.yy290 = OP_TYPE_JSON_CONTAINS; } break; - case 436: /* in_op ::= IN */ -{ yymsp[0].minor.yy20 = OP_TYPE_IN; } + case 447: /* in_op ::= IN */ +{ yymsp[0].minor.yy290 = OP_TYPE_IN; } break; - case 437: /* in_op ::= NOT IN */ -{ yymsp[-1].minor.yy20 = OP_TYPE_NOT_IN; } + case 448: /* in_op ::= NOT IN */ +{ yymsp[-1].minor.yy290 = OP_TYPE_NOT_IN; } break; - case 438: /* in_predicate_value ::= NK_LP literal_list NK_RP */ -{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy874)); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + case 449: /* in_predicate_value ::= NK_LP literal_list NK_RP */ +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy874)); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 440: /* boolean_value_expression ::= NOT boolean_primary */ + case 451: /* boolean_value_expression ::= NOT boolean_primary */ { - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); - yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy74), NULL)); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy602), NULL)); } - yymsp[-1].minor.yy74 = yylhsminor.yy74; + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; - case 441: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + case 452: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); - yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 442: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + case 453: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); - yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 448: /* from_clause_opt ::= FROM table_reference_list */ - case 477: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==477); - case 505: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==505); -{ yymsp[-1].minor.yy74 = yymsp[0].minor.yy74; } + case 459: /* from_clause_opt ::= FROM table_reference_list */ + case 488: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==488); + case 517: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==517); +{ yymsp[-1].minor.yy602 = yymsp[0].minor.yy602; } break; - case 450: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -{ yylhsminor.yy74 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy74, yymsp[0].minor.yy74, NULL); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + case 461: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ yylhsminor.yy602 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy602, yymsp[0].minor.yy602, NULL); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 453: /* table_primary ::= table_name alias_opt */ -{ yylhsminor.yy74 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy317, &yymsp[0].minor.yy317); } - yymsp[-1].minor.yy74 = yylhsminor.yy74; + case 464: /* table_primary ::= table_name alias_opt */ +{ yylhsminor.yy602 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy179, &yymsp[0].minor.yy179); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; - case 454: /* table_primary ::= db_name NK_DOT table_name alias_opt */ -{ yylhsminor.yy74 = createRealTableNode(pCxt, &yymsp[-3].minor.yy317, &yymsp[-1].minor.yy317, &yymsp[0].minor.yy317); } - yymsp[-3].minor.yy74 = yylhsminor.yy74; + case 465: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ yylhsminor.yy602 = createRealTableNode(pCxt, &yymsp[-3].minor.yy179, &yymsp[-1].minor.yy179, &yymsp[0].minor.yy179); } + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; - case 455: /* table_primary ::= subquery alias_opt */ -{ yylhsminor.yy74 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy74), &yymsp[0].minor.yy317); } - yymsp[-1].minor.yy74 = yylhsminor.yy74; + case 466: /* table_primary ::= subquery alias_opt */ +{ yylhsminor.yy602 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy602), &yymsp[0].minor.yy179); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; - case 457: /* alias_opt ::= */ -{ yymsp[1].minor.yy317 = nil_token; } + case 468: /* alias_opt ::= */ +{ yymsp[1].minor.yy179 = nil_token; } break; - case 459: /* alias_opt ::= AS table_alias */ -{ yymsp[-1].minor.yy317 = yymsp[0].minor.yy317; } + case 470: /* alias_opt ::= AS table_alias */ +{ yymsp[-1].minor.yy179 = yymsp[0].minor.yy179; } break; - case 460: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - case 461: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==461); -{ yymsp[-2].minor.yy74 = yymsp[-1].minor.yy74; } + case 471: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + case 472: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==472); +{ yymsp[-2].minor.yy602 = yymsp[-1].minor.yy602; } break; - case 462: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -{ yylhsminor.yy74 = createJoinTableNode(pCxt, yymsp[-4].minor.yy630, yymsp[-5].minor.yy74, yymsp[-2].minor.yy74, yymsp[0].minor.yy74); } - yymsp[-5].minor.yy74 = yylhsminor.yy74; + case 473: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ yylhsminor.yy602 = createJoinTableNode(pCxt, yymsp[-4].minor.yy42, yymsp[-5].minor.yy602, yymsp[-2].minor.yy602, yymsp[0].minor.yy602); } + yymsp[-5].minor.yy602 = yylhsminor.yy602; break; - case 463: /* join_type ::= */ -{ yymsp[1].minor.yy630 = JOIN_TYPE_INNER; } + case 474: /* join_type ::= */ +{ yymsp[1].minor.yy42 = JOIN_TYPE_INNER; } break; - case 464: /* join_type ::= INNER */ -{ yymsp[0].minor.yy630 = JOIN_TYPE_INNER; } + case 475: /* join_type ::= INNER */ +{ yymsp[0].minor.yy42 = JOIN_TYPE_INNER; } break; - case 465: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 476: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { - yymsp[-11].minor.yy74 = createSelectStmt(pCxt, yymsp[-10].minor.yy335, yymsp[-9].minor.yy874, yymsp[-8].minor.yy74); - yymsp[-11].minor.yy74 = addWhereClause(pCxt, yymsp[-11].minor.yy74, yymsp[-7].minor.yy74); - yymsp[-11].minor.yy74 = addPartitionByClause(pCxt, yymsp[-11].minor.yy74, yymsp[-6].minor.yy874); - yymsp[-11].minor.yy74 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy74, yymsp[-2].minor.yy74); - yymsp[-11].minor.yy74 = addGroupByClause(pCxt, yymsp[-11].minor.yy74, yymsp[-1].minor.yy874); - yymsp[-11].minor.yy74 = addHavingClause(pCxt, yymsp[-11].minor.yy74, yymsp[0].minor.yy74); - yymsp[-11].minor.yy74 = addRangeClause(pCxt, yymsp[-11].minor.yy74, yymsp[-5].minor.yy74); - yymsp[-11].minor.yy74 = addEveryClause(pCxt, yymsp[-11].minor.yy74, yymsp[-4].minor.yy74); - yymsp[-11].minor.yy74 = addFillClause(pCxt, yymsp[-11].minor.yy74, yymsp[-3].minor.yy74); + yymsp[-11].minor.yy602 = createSelectStmt(pCxt, yymsp[-10].minor.yy767, yymsp[-9].minor.yy874, yymsp[-8].minor.yy602); + yymsp[-11].minor.yy602 = addWhereClause(pCxt, yymsp[-11].minor.yy602, yymsp[-7].minor.yy602); + yymsp[-11].minor.yy602 = addPartitionByClause(pCxt, yymsp[-11].minor.yy602, yymsp[-6].minor.yy874); + yymsp[-11].minor.yy602 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy602, yymsp[-2].minor.yy602); + yymsp[-11].minor.yy602 = addGroupByClause(pCxt, yymsp[-11].minor.yy602, yymsp[-1].minor.yy874); + yymsp[-11].minor.yy602 = addHavingClause(pCxt, yymsp[-11].minor.yy602, yymsp[0].minor.yy602); + yymsp[-11].minor.yy602 = addRangeClause(pCxt, yymsp[-11].minor.yy602, yymsp[-5].minor.yy602); + yymsp[-11].minor.yy602 = addEveryClause(pCxt, yymsp[-11].minor.yy602, yymsp[-4].minor.yy602); + yymsp[-11].minor.yy602 = addFillClause(pCxt, yymsp[-11].minor.yy602, yymsp[-3].minor.yy602); } break; - case 468: /* set_quantifier_opt ::= ALL */ -{ yymsp[0].minor.yy335 = false; } + case 479: /* set_quantifier_opt ::= ALL */ +{ yymsp[0].minor.yy767 = false; } break; - case 471: /* select_item ::= NK_STAR */ -{ yylhsminor.yy74 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy74 = yylhsminor.yy74; + case 482: /* select_item ::= NK_STAR */ +{ yylhsminor.yy602 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; - case 473: /* select_item ::= common_expression column_alias */ - case 483: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==483); -{ yylhsminor.yy74 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy74), &yymsp[0].minor.yy317); } - yymsp[-1].minor.yy74 = yylhsminor.yy74; + case 484: /* select_item ::= common_expression column_alias */ + case 494: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==494); +{ yylhsminor.yy602 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy602), &yymsp[0].minor.yy179); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; - case 474: /* select_item ::= common_expression AS column_alias */ - case 484: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==484); -{ yylhsminor.yy74 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), &yymsp[0].minor.yy317); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + case 485: /* select_item ::= common_expression AS column_alias */ + case 495: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==495); +{ yylhsminor.yy602 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), &yymsp[0].minor.yy179); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 479: /* partition_by_clause_opt ::= PARTITION BY partition_list */ - case 501: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==501); - case 520: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==520); + case 490: /* partition_by_clause_opt ::= PARTITION BY partition_list */ + case 513: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==513); + case 532: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==532); { yymsp[-2].minor.yy874 = yymsp[0].minor.yy874; } break; - case 486: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ -{ yymsp[-5].minor.yy74 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy74), releaseRawExprNode(pCxt, yymsp[-1].minor.yy74)); } + case 497: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ +{ yymsp[-5].minor.yy602 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy602), releaseRawExprNode(pCxt, yymsp[-1].minor.yy602)); } + break; + case 498: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ +{ yymsp[-3].minor.yy602 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy602)); } break; - case 487: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ -{ yymsp[-3].minor.yy74 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy74)); } + case 499: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-5].minor.yy602 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy602), NULL, yymsp[-1].minor.yy602, yymsp[0].minor.yy602); } break; - case 488: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-5].minor.yy74 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy74), NULL, yymsp[-1].minor.yy74, yymsp[0].minor.yy74); } + case 500: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-7].minor.yy602 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy602), releaseRawExprNode(pCxt, yymsp[-3].minor.yy602), yymsp[-1].minor.yy602, yymsp[0].minor.yy602); } break; - case 489: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-7].minor.yy74 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy74), releaseRawExprNode(pCxt, yymsp[-3].minor.yy74), yymsp[-1].minor.yy74, yymsp[0].minor.yy74); } + case 501: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ +{ yymsp[-6].minor.yy602 = createEventWindowNode(pCxt, yymsp[-3].minor.yy602, yymsp[0].minor.yy602); } break; - case 493: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -{ yymsp[-3].minor.yy74 = createFillNode(pCxt, yymsp[-1].minor.yy828, NULL); } + case 505: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ yymsp[-3].minor.yy602 = createFillNode(pCxt, yymsp[-1].minor.yy324, NULL); } break; - case 494: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ -{ yymsp[-5].minor.yy74 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy874)); } + case 506: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ +{ yymsp[-5].minor.yy602 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy874)); } break; - case 495: /* fill_mode ::= NONE */ -{ yymsp[0].minor.yy828 = FILL_MODE_NONE; } + case 507: /* fill_mode ::= NONE */ +{ yymsp[0].minor.yy324 = FILL_MODE_NONE; } break; - case 496: /* fill_mode ::= PREV */ -{ yymsp[0].minor.yy828 = FILL_MODE_PREV; } + case 508: /* fill_mode ::= PREV */ +{ yymsp[0].minor.yy324 = FILL_MODE_PREV; } break; - case 497: /* fill_mode ::= NULL */ -{ yymsp[0].minor.yy828 = FILL_MODE_NULL; } + case 509: /* fill_mode ::= NULL */ +{ yymsp[0].minor.yy324 = FILL_MODE_NULL; } break; - case 498: /* fill_mode ::= LINEAR */ -{ yymsp[0].minor.yy828 = FILL_MODE_LINEAR; } + case 510: /* fill_mode ::= LINEAR */ +{ yymsp[0].minor.yy324 = FILL_MODE_LINEAR; } break; - case 499: /* fill_mode ::= NEXT */ -{ yymsp[0].minor.yy828 = FILL_MODE_NEXT; } + case 511: /* fill_mode ::= NEXT */ +{ yymsp[0].minor.yy324 = FILL_MODE_NEXT; } break; - case 502: /* group_by_list ::= expr_or_subquery */ -{ yylhsminor.yy874 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); } + case 514: /* group_by_list ::= expr_or_subquery */ +{ yylhsminor.yy874 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } yymsp[0].minor.yy874 = yylhsminor.yy874; break; - case 503: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ -{ yylhsminor.yy874 = addNodeToList(pCxt, yymsp[-2].minor.yy874, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); } + case 515: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ +{ yylhsminor.yy874 = addNodeToList(pCxt, yymsp[-2].minor.yy874, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } yymsp[-2].minor.yy874 = yylhsminor.yy874; break; - case 507: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ -{ yymsp[-5].minor.yy74 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy74), releaseRawExprNode(pCxt, yymsp[-1].minor.yy74)); } + case 519: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ +{ yymsp[-5].minor.yy602 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy602), releaseRawExprNode(pCxt, yymsp[-1].minor.yy602)); } break; - case 510: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 522: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ { - yylhsminor.yy74 = addOrderByClause(pCxt, yymsp[-3].minor.yy74, yymsp[-2].minor.yy874); - yylhsminor.yy74 = addSlimitClause(pCxt, yylhsminor.yy74, yymsp[-1].minor.yy74); - yylhsminor.yy74 = addLimitClause(pCxt, yylhsminor.yy74, yymsp[0].minor.yy74); + yylhsminor.yy602 = addOrderByClause(pCxt, yymsp[-3].minor.yy602, yymsp[-2].minor.yy874); + yylhsminor.yy602 = addSlimitClause(pCxt, yylhsminor.yy602, yymsp[-1].minor.yy602); + yylhsminor.yy602 = addLimitClause(pCxt, yylhsminor.yy602, yymsp[0].minor.yy602); } - yymsp[-3].minor.yy74 = yylhsminor.yy74; + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; - case 513: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ -{ yylhsminor.yy74 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy74, yymsp[0].minor.yy74); } - yymsp[-3].minor.yy74 = yylhsminor.yy74; + case 525: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ +{ yylhsminor.yy602 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy602, yymsp[0].minor.yy602); } + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; - case 514: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ -{ yylhsminor.yy74 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy74, yymsp[0].minor.yy74); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + case 526: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ +{ yylhsminor.yy602 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy602, yymsp[0].minor.yy602); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 522: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 526: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==526); -{ yymsp[-1].minor.yy74 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + case 534: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 538: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==538); +{ yymsp[-1].minor.yy602 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 523: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 527: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==527); -{ yymsp[-3].minor.yy74 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 535: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 539: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==539); +{ yymsp[-3].minor.yy602 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 524: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 528: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==528); -{ yymsp[-3].minor.yy74 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 536: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 540: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==540); +{ yymsp[-3].minor.yy602 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 529: /* subquery ::= NK_LP query_expression NK_RP */ -{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy74); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + case 541: /* subquery ::= NK_LP query_expression NK_RP */ +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy602); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 534: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ -{ yylhsminor.yy74 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), yymsp[-1].minor.yy326, yymsp[0].minor.yy109); } - yymsp[-2].minor.yy74 = yylhsminor.yy74; + case 546: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ +{ yylhsminor.yy602 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), yymsp[-1].minor.yy878, yymsp[0].minor.yy487); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 535: /* ordering_specification_opt ::= */ -{ yymsp[1].minor.yy326 = ORDER_ASC; } + case 547: /* ordering_specification_opt ::= */ +{ yymsp[1].minor.yy878 = ORDER_ASC; } break; - case 536: /* ordering_specification_opt ::= ASC */ -{ yymsp[0].minor.yy326 = ORDER_ASC; } + case 548: /* ordering_specification_opt ::= ASC */ +{ yymsp[0].minor.yy878 = ORDER_ASC; } break; - case 537: /* ordering_specification_opt ::= DESC */ -{ yymsp[0].minor.yy326 = ORDER_DESC; } + case 549: /* ordering_specification_opt ::= DESC */ +{ yymsp[0].minor.yy878 = ORDER_DESC; } break; - case 538: /* null_ordering_opt ::= */ -{ yymsp[1].minor.yy109 = NULL_ORDER_DEFAULT; } + case 550: /* null_ordering_opt ::= */ +{ yymsp[1].minor.yy487 = NULL_ORDER_DEFAULT; } break; - case 539: /* null_ordering_opt ::= NULLS FIRST */ -{ yymsp[-1].minor.yy109 = NULL_ORDER_FIRST; } + case 551: /* null_ordering_opt ::= NULLS FIRST */ +{ yymsp[-1].minor.yy487 = NULL_ORDER_FIRST; } break; - case 540: /* null_ordering_opt ::= NULLS LAST */ -{ yymsp[-1].minor.yy109 = NULL_ORDER_LAST; } + case 552: /* null_ordering_opt ::= NULLS LAST */ +{ yymsp[-1].minor.yy487 = NULL_ORDER_LAST; } break; default: break; diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 9cc55a7cd55a0060635e9a9044ac076ce5a77119..0d46b63278722bbeaaff35d2aca72953a81b5994 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -33,7 +33,6 @@ std::unique_ptr g_mockCatalogService; class TableBuilder : public ITableBuilder { public: virtual TableBuilder& addColumn(const string& name, int8_t type, int32_t bytes) { - assert(colId_ <= schema()->tableInfo.numOfTags + schema()->tableInfo.numOfColumns); SSchema* col = schema()->schema + (colId_ - 1); col->type = type; col->colId = colId_++; @@ -427,7 +426,7 @@ class MockCatalogServiceImpl { int32_t copyTableSchemaMeta(const string& db, const string& tbname, std::unique_ptr* dst) const { STableMeta* src = getTableSchemaMeta(db, tbname); if (nullptr == src) { - return TSDB_CODE_TSC_INVALID_TABLE_NAME; + return TSDB_CODE_PAR_TABLE_NOT_EXIST; } int32_t len = sizeof(STableMeta) + sizeof(SSchema) * (src->tableInfo.numOfTags + src->tableInfo.numOfColumns); dst->reset((STableMeta*)taosMemoryCalloc(1, len)); diff --git a/source/libs/parser/test/parAlterToBalanceTest.cpp b/source/libs/parser/test/parAlterToBalanceTest.cpp index ba1ba9fc831ec8661cc7a8252a1772d5bbae6317..a0de93782ddeffa351cd1700c5447c712a1da967 100644 --- a/source/libs/parser/test/parAlterToBalanceTest.cpp +++ b/source/libs/parser/test/parAlterToBalanceTest.cpp @@ -21,6 +21,25 @@ namespace ParserTest { class ParserInitialATest : public ParserDdlTest {}; +/* + * ALTER ACCOUNT account_name alter_account_options + * + * alter_account_options: + * alter_account_option ... + * + * alter_account_option: { + * PASS value + * | PPS value + * | TSERIES value + * | STORAGE value + * | STREAMS value + * | QTIME value + * | DBS value + * | USERS value + * | CONNS value + * | STATE value + * } + */ TEST_F(ParserInitialATest, alterAccount) { useDb("root", "test"); @@ -48,6 +67,7 @@ TEST_F(ParserInitialATest, alterDnode) { setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_ALTER_DNODE_STMT); + ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_CONFIG_DNODE); SMCfgDnodeReq req = {0}; ASSERT_EQ(tDeserializeSMCfgDnodeReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS); ASSERT_EQ(req.dnodeId, expect.dnodeId); @@ -86,9 +106,8 @@ TEST_F(ParserInitialATest, alterDnode) { * | KEEP {int_value | duration_value} -- rang [1, 365000], default 3650, unit day * | PAGES int_value -- rang [64, INT32_MAX], default 256, unit page * | REPLICA int_value -- todo: enum 1, 3, default 1, unit replica - * | STRICT {'off' | 'on'} -- todo: default 'off' * | WAL_LEVEL int_value -- enum 1, 2, default 1 - * | SST_TRIGGER int_value -- rang [1, 16], default 8 + * | STT_TRIGGER int_value -- rang [1, 16], default 8 * } */ TEST_F(ParserInitialATest, alterDatabase) { @@ -130,10 +149,11 @@ TEST_F(ParserInitialATest, alterDatabase) { auto setAlterDbStrict = [&](int8_t strict) { expect.strict = strict; }; auto setAlterDbCacheModel = [&](int8_t cacheModel) { expect.cacheLast = cacheModel; }; auto setAlterDbReplica = [&](int8_t replications) { expect.replications = replications; }; - auto setAlterDbSstTrigger = [&](int8_t sstTrigger) { expect.sstTrigger = sstTrigger; }; + auto setAlterDbSttTrigger = [&](int8_t sstTrigger) { expect.sstTrigger = sstTrigger; }; setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_ALTER_DATABASE_STMT); + ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_ALTER_DB); SAlterDbReq req = {0}; ASSERT_EQ(tDeserializeSAlterDbReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS); ASSERT_EQ(std::string(req.db), std::string(expect.db)); @@ -161,11 +181,12 @@ TEST_F(ParserInitialATest, alterDatabase) { setAlterDbFsync(200); setAlterDbWal(1); setAlterDbCacheModel(TSDB_CACHE_MODEL_LAST_ROW); - setAlterDbSstTrigger(16); + setAlterDbSttTrigger(16); setAlterDbBuffer(16); setAlterDbPages(128); + setAlterDbReplica(3); run("ALTER DATABASE test BUFFER 16 CACHEMODEL 'last_row' CACHESIZE 32 WAL_FSYNC_PERIOD 200 KEEP 10 PAGES 128 " - "WAL_LEVEL 1 STT_TRIGGER 16"); + "REPLICA 3 WAL_LEVEL 1 STT_TRIGGER 16"); clearAlterDbReq(); initAlterDb("test"); @@ -240,6 +261,22 @@ TEST_F(ParserInitialATest, alterDatabase) { setAlterDbWal(2); run("ALTER DATABASE test WAL_LEVEL 2"); clearAlterDbReq(); + + initAlterDb("test"); + setAlterDbReplica(1); + run("ALTER DATABASE test REPLICA 1"); + setAlterDbReplica(3); + run("ALTER DATABASE test REPLICA 3"); + clearAlterDbReq(); + + initAlterDb("test"); + setAlterDbSttTrigger(1); + run("ALTER DATABASE test STT_TRIGGER 1"); + setAlterDbSttTrigger(4); + run("ALTER DATABASE test STT_TRIGGER 4"); + setAlterDbSttTrigger(16); + run("ALTER DATABASE test STT_TRIGGER 16"); + clearAlterDbReq(); } TEST_F(ParserInitialATest, alterDatabaseSemanticCheck) { @@ -260,6 +297,7 @@ TEST_F(ParserInitialATest, alterDatabaseSemanticCheck) { run("ALTER DATABASE test PAGES 63", TSDB_CODE_PAR_INVALID_DB_OPTION); run("ALTER DATABASE test WAL_LEVEL 0", TSDB_CODE_PAR_INVALID_DB_OPTION); run("ALTER DATABASE test WAL_LEVEL 3", TSDB_CODE_PAR_INVALID_DB_OPTION); + run("ALTER DATABASE test REPLICA 2", TSDB_CODE_PAR_INVALID_DB_OPTION); run("ALTER DATABASE test STT_TRIGGER 0", TSDB_CODE_PAR_INVALID_DB_OPTION); run("ALTER DATABASE test STT_TRIGGER 17", TSDB_CODE_PAR_INVALID_DB_OPTION); // Regardless of the specific sentence @@ -267,7 +305,7 @@ TEST_F(ParserInitialATest, alterDatabaseSemanticCheck) { } /* - * ALTER LOCAL dnode_id 'config' ['value'] + * ALTER LOCAL 'config' ['value'] */ TEST_F(ParserInitialATest, alterLocal) { useDb("root", "test"); @@ -311,19 +349,19 @@ TEST_F(ParserInitialATest, alterLocal) { * | ADD COLUMN col_name column_type * | DROP COLUMN col_name * | MODIFY COLUMN col_name column_type - * | RENAME COLUMN old_col_name new_col_name -- normal table - * | ADD TAG tag_name tag_type -- super table - * | DROP TAG tag_name -- super table - * | MODIFY TAG tag_name tag_type -- super table - * | RENAME TAG old_tag_name new_tag_name -- super table - * | SET TAG tag_name = new_tag_value -- child table + * | RENAME COLUMN old_col_name new_col_name -- only normal table + * | ADD TAG tag_name tag_type -- only super table + * | DROP TAG tag_name -- only super table + * | MODIFY TAG tag_name tag_type -- only super table + * | RENAME TAG old_tag_name new_tag_name -- only super table + * | SET TAG tag_name = new_tag_value -- only child table * } * * alter_table_options: * alter_table_option ... * * alter_table_option: { - * TTL int_value -- child/normal table + * TTL int_value -- only child/normal table * | COMMENT 'string_value' * } */ @@ -379,6 +417,7 @@ TEST_F(ParserInitialATest, alterSTable) { setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_ALTER_SUPER_TABLE_STMT); + ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_ALTER_STB); SMAlterStbReq req = {0}; ASSERT_EQ(tDeserializeSMAlterStbReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS); ASSERT_EQ(std::string(req.name), std::string(expect.name)); @@ -444,136 +483,255 @@ TEST_F(ParserInitialATest, alterSTableSemanticCheck) { run("ALTER STABLE st1 TTL 10", TSDB_CODE_PAR_INVALID_ALTER_TABLE); } +/* + * ALTER TABLE [db_name.]tb_name alter_table_clause + * + * alter_table_clause: { + * alter_table_options + * | ADD COLUMN col_name column_type + * | DROP COLUMN col_name + * | MODIFY COLUMN col_name column_type + * | RENAME COLUMN old_col_name new_col_name -- only normal table + * | ADD TAG tag_name tag_type -- only super table + * | DROP TAG tag_name -- only super table + * | MODIFY TAG tag_name tag_type -- only super table + * | RENAME TAG old_tag_name new_tag_name -- only super table + * | SET TAG tag_name = new_tag_value -- only child table + * } + * + * alter_table_options: + * alter_table_option ... + * + * alter_table_option: { + * TTL int_value -- only child/normal table + * | COMMENT 'string_value' + * } + */ TEST_F(ParserInitialATest, alterTable) { useDb("root", "test"); - SVAlterTbReq expect = {0}; - - auto clearAlterTbReq = [&]() { - free(expect.tbName); - free(expect.colName); - free(expect.colNewName); - free(expect.tagName); - memset(&expect, 0, sizeof(SVAlterTbReq)); - }; - - auto setAlterTableCol = [&](const char* pTbname, int8_t alterType, const char* pColName, int8_t dataType = 0, - int32_t dataBytes = 0, const char* pNewColName = nullptr) { - expect.tbName = strdup(pTbname); - expect.action = alterType; - expect.colName = strdup(pColName); - - switch (alterType) { - case TSDB_ALTER_TABLE_ADD_COLUMN: - expect.type = dataType; - expect.flags = COL_SMA_ON; - expect.bytes = dataBytes > 0 ? dataBytes : (dataType > 0 ? tDataTypes[dataType].bytes : 0); - break; - case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: - expect.colModBytes = dataBytes; - break; - case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: - expect.colNewName = strdup(pNewColName); - break; - default: - break; - } - }; - - auto setAlterTableTag = [&](const char* pTbname, const char* pTagName, uint8_t* pNewVal, uint32_t bytes) { - expect.tbName = strdup(pTbname); - expect.action = TSDB_ALTER_TABLE_UPDATE_TAG_VAL; - expect.tagName = strdup(pTagName); - - expect.isNull = (nullptr == pNewVal); - expect.nTagVal = bytes; - expect.pTagVal = pNewVal; - }; - - auto setAlterTableOptions = [&](const char* pTbname, int32_t ttl, char* pComment = nullptr) { - expect.tbName = strdup(pTbname); - expect.action = TSDB_ALTER_TABLE_UPDATE_OPTIONS; - if (-1 != ttl) { - expect.updateTTL = true; - expect.newTTL = ttl; - } - if (nullptr != pComment) { - expect.newCommentLen = strlen(pComment); - expect.newComment = pComment; - } - }; - - setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { - ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_VNODE_MODIF_STMT); - SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot; - - ASSERT_EQ(pStmt->sqlNodeType, QUERY_NODE_ALTER_TABLE_STMT); - ASSERT_NE(pStmt->pDataBlocks, nullptr); - ASSERT_EQ(taosArrayGetSize(pStmt->pDataBlocks), 1); - SVgDataBlocks* pVgData = (SVgDataBlocks*)taosArrayGetP(pStmt->pDataBlocks, 0); - void* pBuf = POINTER_SHIFT(pVgData->pData, sizeof(SMsgHead)); - SVAlterTbReq req = {0}; - SDecoder coder = {0}; - tDecoderInit(&coder, (uint8_t*)pBuf, pVgData->size); - ASSERT_EQ(tDecodeSVAlterTbReq(&coder, &req), TSDB_CODE_SUCCESS); - - ASSERT_EQ(std::string(req.tbName), std::string(expect.tbName)); - ASSERT_EQ(req.action, expect.action); - if (nullptr != expect.colName) { - ASSERT_EQ(std::string(req.colName), std::string(expect.colName)); - } - ASSERT_EQ(req.type, expect.type); - ASSERT_EQ(req.flags, expect.flags); - ASSERT_EQ(req.bytes, expect.bytes); - ASSERT_EQ(req.colModBytes, expect.colModBytes); - if (nullptr != expect.colNewName) { - ASSERT_EQ(std::string(req.colNewName), std::string(expect.colNewName)); - } - if (nullptr != expect.tagName) { - ASSERT_EQ(std::string(req.tagName), std::string(expect.tagName)); - } - ASSERT_EQ(req.isNull, expect.isNull); - ASSERT_EQ(req.nTagVal, expect.nTagVal); - ASSERT_EQ(memcmp(req.pTagVal, expect.pTagVal, expect.nTagVal), 0); - ASSERT_EQ(req.updateTTL, expect.updateTTL); - ASSERT_EQ(req.newTTL, expect.newTTL); - if (nullptr != expect.newComment) { - ASSERT_EQ(std::string(req.newComment), std::string(expect.newComment)); - ASSERT_EQ(req.newCommentLen, strlen(req.newComment)); - ASSERT_EQ(expect.newCommentLen, strlen(expect.newComment)); - } - - tDecoderClear(&coder); - }); - - setAlterTableOptions("t1", 10, nullptr); - run("ALTER TABLE t1 TTL 10"); - clearAlterTbReq(); - - setAlterTableOptions("t1", -1, (char*)"test"); - run("ALTER TABLE t1 COMMENT 'test'"); - clearAlterTbReq(); - - setAlterTableCol("t1", TSDB_ALTER_TABLE_ADD_COLUMN, "cc1", TSDB_DATA_TYPE_BIGINT); - run("ALTER TABLE t1 ADD COLUMN cc1 BIGINT"); - clearAlterTbReq(); - - setAlterTableCol("t1", TSDB_ALTER_TABLE_DROP_COLUMN, "c1"); - run("ALTER TABLE t1 DROP COLUMN c1"); - clearAlterTbReq(); - - setAlterTableCol("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, "c2", TSDB_DATA_TYPE_VARCHAR, 30 + VARSTR_HEADER_SIZE); - run("ALTER TABLE t1 MODIFY COLUMN c2 VARCHAR(30)"); - clearAlterTbReq(); - - setAlterTableCol("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, "c1", 0, 0, "cc1"); - run("ALTER TABLE t1 RENAME COLUMN c1 cc1"); - clearAlterTbReq(); - - int32_t val = 10; - setAlterTableTag("st1s1", "tag1", (uint8_t*)&val, sizeof(val)); - run("ALTER TABLE st1s1 SET TAG tag1=10"); - clearAlterTbReq(); + // normal/child table + { + SVAlterTbReq expect = {0}; + + auto clearAlterTbReq = [&]() { + free(expect.tbName); + free(expect.colName); + free(expect.colNewName); + free(expect.tagName); + memset(&expect, 0, sizeof(SVAlterTbReq)); + }; + + auto setAlterTableCol = [&](const char* pTbname, int8_t alterType, const char* pColName, int8_t dataType = 0, + int32_t dataBytes = 0, const char* pNewColName = nullptr) { + expect.tbName = strdup(pTbname); + expect.action = alterType; + expect.colName = strdup(pColName); + + switch (alterType) { + case TSDB_ALTER_TABLE_ADD_COLUMN: + expect.type = dataType; + expect.flags = COL_SMA_ON; + expect.bytes = dataBytes > 0 ? dataBytes : (dataType > 0 ? tDataTypes[dataType].bytes : 0); + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: + expect.colModBytes = dataBytes; + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: + expect.colNewName = strdup(pNewColName); + break; + default: + break; + } + }; + + auto setAlterTableTag = [&](const char* pTbname, const char* pTagName, uint8_t* pNewVal, uint32_t bytes) { + expect.tbName = strdup(pTbname); + expect.action = TSDB_ALTER_TABLE_UPDATE_TAG_VAL; + expect.tagName = strdup(pTagName); + + expect.isNull = (nullptr == pNewVal); + expect.nTagVal = bytes; + expect.pTagVal = pNewVal; + }; + + auto setAlterTableOptions = [&](const char* pTbname, int32_t ttl, char* pComment = nullptr) { + expect.tbName = strdup(pTbname); + expect.action = TSDB_ALTER_TABLE_UPDATE_OPTIONS; + if (-1 != ttl) { + expect.updateTTL = true; + expect.newTTL = ttl; + } + if (nullptr != pComment) { + expect.newCommentLen = strlen(pComment); + expect.newComment = pComment; + } + }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_VNODE_MODIFY_STMT); + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; + + ASSERT_EQ(pStmt->sqlNodeType, QUERY_NODE_ALTER_TABLE_STMT); + ASSERT_NE(pStmt->pDataBlocks, nullptr); + ASSERT_EQ(taosArrayGetSize(pStmt->pDataBlocks), 1); + SVgDataBlocks* pVgData = (SVgDataBlocks*)taosArrayGetP(pStmt->pDataBlocks, 0); + void* pBuf = POINTER_SHIFT(pVgData->pData, sizeof(SMsgHead)); + SVAlterTbReq req = {0}; + SDecoder coder = {0}; + tDecoderInit(&coder, (uint8_t*)pBuf, pVgData->size); + ASSERT_EQ(tDecodeSVAlterTbReq(&coder, &req), TSDB_CODE_SUCCESS); + + ASSERT_EQ(std::string(req.tbName), std::string(expect.tbName)); + ASSERT_EQ(req.action, expect.action); + if (nullptr != expect.colName) { + ASSERT_EQ(std::string(req.colName), std::string(expect.colName)); + } + ASSERT_EQ(req.type, expect.type); + ASSERT_EQ(req.flags, expect.flags); + ASSERT_EQ(req.bytes, expect.bytes); + ASSERT_EQ(req.colModBytes, expect.colModBytes); + if (nullptr != expect.colNewName) { + ASSERT_EQ(std::string(req.colNewName), std::string(expect.colNewName)); + } + if (nullptr != expect.tagName) { + ASSERT_EQ(std::string(req.tagName), std::string(expect.tagName)); + } + ASSERT_EQ(req.isNull, expect.isNull); + ASSERT_EQ(req.nTagVal, expect.nTagVal); + ASSERT_EQ(memcmp(req.pTagVal, expect.pTagVal, expect.nTagVal), 0); + ASSERT_EQ(req.updateTTL, expect.updateTTL); + ASSERT_EQ(req.newTTL, expect.newTTL); + if (nullptr != expect.newComment) { + ASSERT_EQ(std::string(req.newComment), std::string(expect.newComment)); + ASSERT_EQ(req.newCommentLen, strlen(req.newComment)); + ASSERT_EQ(expect.newCommentLen, strlen(expect.newComment)); + } + + tDecoderClear(&coder); + }); + + setAlterTableOptions("t1", 10, nullptr); + run("ALTER TABLE t1 TTL 10"); + clearAlterTbReq(); + + setAlterTableOptions("t1", -1, (char*)"test"); + run("ALTER TABLE t1 COMMENT 'test'"); + clearAlterTbReq(); + + setAlterTableCol("t1", TSDB_ALTER_TABLE_ADD_COLUMN, "cc1", TSDB_DATA_TYPE_BIGINT); + run("ALTER TABLE t1 ADD COLUMN cc1 BIGINT"); + clearAlterTbReq(); + + setAlterTableCol("t1", TSDB_ALTER_TABLE_DROP_COLUMN, "c1"); + run("ALTER TABLE t1 DROP COLUMN c1"); + clearAlterTbReq(); + + setAlterTableCol("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, "c2", TSDB_DATA_TYPE_VARCHAR, 30 + VARSTR_HEADER_SIZE); + run("ALTER TABLE t1 MODIFY COLUMN c2 VARCHAR(30)"); + clearAlterTbReq(); + + setAlterTableCol("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, "c1", 0, 0, "cc1"); + run("ALTER TABLE t1 RENAME COLUMN c1 cc1"); + clearAlterTbReq(); + + int32_t val = 10; + setAlterTableTag("st1s1", "tag1", (uint8_t*)&val, sizeof(val)); + run("ALTER TABLE st1s1 SET TAG tag1=10"); + clearAlterTbReq(); + } + + // super table + { + SMAlterStbReq expect = {0}; + + auto clearAlterStbReq = [&]() { + tFreeSMAltertbReq(&expect); + memset(&expect, 0, sizeof(SMAlterStbReq)); + }; + + auto setAlterStbReq = [&](const char* pTbname, int8_t alterType, int32_t numOfFields = 0, + const char* pField1Name = nullptr, int8_t field1Type = 0, int32_t field1Bytes = 0, + const char* pField2Name = nullptr, const char* pComment = nullptr) { + int32_t len = snprintf(expect.name, sizeof(expect.name), "0.test.%s", pTbname); + expect.name[len] = '\0'; + expect.alterType = alterType; + if (nullptr != pComment) { + expect.comment = strdup(pComment); + expect.commentLen = strlen(pComment); + } + + expect.numOfFields = numOfFields; + if (NULL == expect.pFields) { + expect.pFields = taosArrayInit(2, sizeof(TAOS_FIELD)); + TAOS_FIELD field = {0}; + taosArrayPush(expect.pFields, &field); + taosArrayPush(expect.pFields, &field); + } + + TAOS_FIELD* pField = (TAOS_FIELD*)taosArrayGet(expect.pFields, 0); + if (NULL != pField1Name) { + strcpy(pField->name, pField1Name); + pField->name[strlen(pField1Name)] = '\0'; + } else { + memset(pField, 0, sizeof(TAOS_FIELD)); + } + pField->type = field1Type; + pField->bytes = field1Bytes > 0 ? field1Bytes : (field1Type > 0 ? tDataTypes[field1Type].bytes : 0); + + pField = (TAOS_FIELD*)taosArrayGet(expect.pFields, 1); + if (NULL != pField2Name) { + strcpy(pField->name, pField2Name); + pField->name[strlen(pField2Name)] = '\0'; + } else { + memset(pField, 0, sizeof(TAOS_FIELD)); + } + pField->type = 0; + pField->bytes = 0; + }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_ALTER_TABLE_STMT); + ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_ALTER_STB); + SMAlterStbReq req = {0}; + ASSERT_EQ(tDeserializeSMAlterStbReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS); + ASSERT_EQ(std::string(req.name), std::string(expect.name)); + ASSERT_EQ(req.alterType, expect.alterType); + ASSERT_EQ(req.numOfFields, expect.numOfFields); + if (expect.numOfFields > 0) { + TAOS_FIELD* pField = (TAOS_FIELD*)taosArrayGet(req.pFields, 0); + TAOS_FIELD* pExpectField = (TAOS_FIELD*)taosArrayGet(expect.pFields, 0); + ASSERT_EQ(std::string(pField->name), std::string(pExpectField->name)); + ASSERT_EQ(pField->type, pExpectField->type); + ASSERT_EQ(pField->bytes, pExpectField->bytes); + } + if (expect.numOfFields > 1) { + TAOS_FIELD* pField = (TAOS_FIELD*)taosArrayGet(req.pFields, 1); + TAOS_FIELD* pExpectField = (TAOS_FIELD*)taosArrayGet(expect.pFields, 1); + ASSERT_EQ(std::string(pField->name), std::string(pExpectField->name)); + ASSERT_EQ(pField->type, pExpectField->type); + ASSERT_EQ(pField->bytes, pExpectField->bytes); + } + tFreeSMAltertbReq(&req); + }); + + setAlterStbReq("st1", TSDB_ALTER_TABLE_ADD_TAG, 1, "tag11", TSDB_DATA_TYPE_BIGINT); + run("ALTER TABLE st1 ADD TAG tag11 BIGINT"); + clearAlterStbReq(); + + setAlterStbReq("st1", TSDB_ALTER_TABLE_DROP_TAG, 1, "tag1"); + run("ALTER TABLE st1 DROP TAG tag1"); + clearAlterStbReq(); + + setAlterStbReq("st1", TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, 1, "tag2", TSDB_DATA_TYPE_VARCHAR, + 30 + VARSTR_HEADER_SIZE); + run("ALTER TABLE st1 MODIFY TAG tag2 VARCHAR(30)"); + clearAlterStbReq(); + + setAlterStbReq("st1", TSDB_ALTER_TABLE_UPDATE_TAG_NAME, 2, "tag1", 0, 0, "tag11"); + run("ALTER TABLE st1 RENAME TAG tag1 tag11"); + clearAlterStbReq(); + } } TEST_F(ParserInitialATest, alterTableSemanticCheck) { @@ -588,7 +746,7 @@ TEST_F(ParserInitialATest, alterTableSemanticCheck) { } /* - * ALTER USER user_name PASS str_value + * ALTER USER user_name alter_user_clause * * alter_user_clause: { * PASS str_value @@ -618,6 +776,7 @@ TEST_F(ParserInitialATest, alterUser) { setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_ALTER_USER_STMT); + ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_ALTER_USER); SAlterUserReq req = {0}; ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSAlterUserReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); diff --git a/source/libs/parser/test/parExplainToSyncdbTest.cpp b/source/libs/parser/test/parExplainToSyncdbTest.cpp index f88d6d316b13606ba95226e669a59bcc588bf566..f23685b3058b911f23baf8c37c6608ab989de5c7 100644 --- a/source/libs/parser/test/parExplainToSyncdbTest.cpp +++ b/source/libs/parser/test/parExplainToSyncdbTest.cpp @@ -52,8 +52,8 @@ TEST_F(ParserExplainToSyncdbTest, grant) { ASSERT_EQ(string(req.objname), string(expect.objname)); }); - setAlterUserReq(TSDB_ALTER_USER_ADD_ALL_DB, "wxy", "0.test"); - run("GRANT ALL ON test.* TO wxy"); + setAlterUserReq(TSDB_ALTER_USER_ADD_ALL_DB, "wxy", "0.*"); + run("GRANT ALL ON *.* TO wxy"); setAlterUserReq(TSDB_ALTER_USER_ADD_READ_DB, "wxy", "0.test"); run("GRANT READ ON test.* TO wxy"); @@ -138,10 +138,38 @@ TEST_F(ParserExplainToSyncdbTest, redistributeVgroup) { TEST_F(ParserExplainToSyncdbTest, revoke) { useDb("root", "test"); - run("REVOKE ALL ON test.* FROM wxy"); + SAlterUserReq expect = {0}; + + auto setAlterUserReq = [&](int8_t alterType, const string& user, const string& obj) { + expect.alterType = alterType; + snprintf(expect.user, sizeof(expect.user), "%s", user.c_str()); + snprintf(expect.objname, sizeof(expect.objname), "%s", obj.c_str()); + }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_REVOKE_STMT); + ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_ALTER_USER); + SAlterUserReq req = {0}; + ASSERT_EQ(tDeserializeSAlterUserReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS); + ASSERT_EQ(req.alterType, expect.alterType); + ASSERT_EQ(string(req.user), string(expect.user)); + ASSERT_EQ(string(req.objname), string(expect.objname)); + }); + + setAlterUserReq(TSDB_ALTER_USER_REMOVE_ALL_DB, "wxy", "0.*"); + run("REVOKE ALL ON *.* FROM wxy"); + + setAlterUserReq(TSDB_ALTER_USER_REMOVE_READ_DB, "wxy", "0.test"); run("REVOKE READ ON test.* FROM wxy"); + + setAlterUserReq(TSDB_ALTER_USER_REMOVE_WRITE_DB, "wxy", "0.test"); run("REVOKE WRITE ON test.* FROM wxy"); + + setAlterUserReq(TSDB_ALTER_USER_REMOVE_ALL_DB, "wxy", "0.test"); run("REVOKE READ, WRITE ON test.* FROM wxy"); + + setAlterUserReq(TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC, "wxy", "0.tp1"); + run("REVOKE SUBSCRIBE ON tp1 FROM wxy"); } // todo syncdb diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index 17d02c1cce7084c99d025d89b36c69e223083069..8286ae72e19b767fdddf6a882e370261ebfc4ccb 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -21,12 +21,35 @@ namespace ParserTest { class ParserInitialCTest : public ParserDdlTest {}; +/* + * CREATE ACCOUNT account_name PASS value [create_account_options] + * + * create_account_options: + * create_account_option ... + * + * create_account_option: { + * PPS value + * | TSERIES value + * | STORAGE value + * | STREAMS value + * | QTIME value + * | DBS value + * | USERS value + * | CONNS value + * | STATE value + * } + */ TEST_F(ParserInitialCTest, createAccount) { useDb("root", "test"); run("CREATE ACCOUNT ac_wxy PASS '123456'", TSDB_CODE_PAR_EXPRIE_STATEMENT, PARSER_STAGE_PARSE); } +/* + * CREATE BNODE ON DNODE dnode_id + * the server does not support it temporarily + */ + /* * CREATE DATABASE [IF NOT EXISTS] db_name [database_options] * @@ -68,7 +91,7 @@ TEST_F(ParserInitialCTest, createDatabase) { memset(&expect, 0, sizeof(SCreateDbReq)); }; - auto setCreateDbReqFunc = [&](const char* pDbname, int8_t igExists = 0) { + auto setCreateDbReq = [&](const char* pDbname, int8_t igExists = 0) { int32_t len = snprintf(expect.db, sizeof(expect.db), "0.%s", pDbname); expect.db[len] = '\0'; expect.ignoreExist = igExists; @@ -102,28 +125,28 @@ TEST_F(ParserInitialCTest, createDatabase) { expect.tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE; }; - auto setDbBufferFunc = [&](int32_t buffer) { expect.buffer = buffer; }; - auto setDbCachelastFunc = [&](int8_t cachelast) { expect.cacheLast = cachelast; }; + auto setDbBuffer = [&](int32_t buffer) { expect.buffer = buffer; }; + auto setDbCachelast = [&](int8_t cachelast) { expect.cacheLast = cachelast; }; auto setDbCachelastSize = [&](int8_t cachelastSize) { expect.cacheLastSize = cachelastSize; }; - auto setDbCompressionFunc = [&](int8_t compressionLevel) { expect.compression = compressionLevel; }; - auto setDbDaysFunc = [&](int32_t daysPerFile) { expect.daysPerFile = daysPerFile; }; - auto setDbFsyncFunc = [&](int32_t fsyncPeriod) { expect.walFsyncPeriod = fsyncPeriod; }; - auto setDbMaxRowsFunc = [&](int32_t maxRowsPerBlock) { expect.maxRows = maxRowsPerBlock; }; - auto setDbMinRowsFunc = [&](int32_t minRowsPerBlock) { expect.minRows = minRowsPerBlock; }; - auto setDbKeepFunc = [&](int32_t keep0, int32_t keep1 = 0, int32_t keep2 = 0) { + auto setDbCompression = [&](int8_t compressionLevel) { expect.compression = compressionLevel; }; + auto setDbDays = [&](int32_t daysPerFile) { expect.daysPerFile = daysPerFile; }; + auto setDbFsync = [&](int32_t fsyncPeriod) { expect.walFsyncPeriod = fsyncPeriod; }; + auto setDbMaxRows = [&](int32_t maxRowsPerBlock) { expect.maxRows = maxRowsPerBlock; }; + auto setDbMinRows = [&](int32_t minRowsPerBlock) { expect.minRows = minRowsPerBlock; }; + auto setDbKeep = [&](int32_t keep0, int32_t keep1 = 0, int32_t keep2 = 0) { expect.daysToKeep0 = keep0; expect.daysToKeep1 = 0 == keep1 ? expect.daysToKeep0 : keep1; expect.daysToKeep2 = 0 == keep2 ? expect.daysToKeep1 : keep2; }; - auto setDbPagesFunc = [&](int32_t pages) { expect.pages = pages; }; - auto setDbPageSizeFunc = [&](int32_t pagesize) { expect.pageSize = pagesize; }; - auto setDbPrecisionFunc = [&](int8_t precision) { expect.precision = precision; }; - auto setDbReplicaFunc = [&](int8_t replica) { expect.replications = replica; }; - auto setDbStrictaFunc = [&](int8_t strict) { expect.strict = strict; }; - auto setDbWalLevelFunc = [&](int8_t walLevel) { expect.walLevel = walLevel; }; - auto setDbVgroupsFunc = [&](int32_t numOfVgroups) { expect.numOfVgroups = numOfVgroups; }; - auto setDbSingleStableFunc = [&](int8_t singleStable) { expect.numOfStables = singleStable; }; - auto addDbRetentionFunc = [&](int64_t freq, int64_t keep, int8_t freqUnit, int8_t keepUnit) { + auto setDbPages = [&](int32_t pages) { expect.pages = pages; }; + auto setDbPageSize = [&](int32_t pagesize) { expect.pageSize = pagesize; }; + auto setDbPrecision = [&](int8_t precision) { expect.precision = precision; }; + auto setDbReplica = [&](int8_t replica) { expect.replications = replica; }; + auto setDbStricta = [&](int8_t strict) { expect.strict = strict; }; + auto setDbWalLevel = [&](int8_t walLevel) { expect.walLevel = walLevel; }; + auto setDbVgroups = [&](int32_t numOfVgroups) { expect.numOfVgroups = numOfVgroups; }; + auto setDbSingleStable = [&](int8_t singleStable) { expect.numOfStables = singleStable; }; + auto addDbRetention = [&](int64_t freq, int64_t keep, int8_t freqUnit, int8_t keepUnit) { SRetention retention = {0}; retention.freq = freq; retention.keep = keep; @@ -135,7 +158,7 @@ TEST_F(ParserInitialCTest, createDatabase) { taosArrayPush(expect.pRetensions, &retention); ++expect.numOfRetensions; }; - auto setDbSchemalessFunc = [&](int8_t schemaless) { expect.schemaless = schemaless; }; + auto setDbSchemaless = [&](int8_t schemaless) { expect.schemaless = schemaless; }; auto setDbWalRetentionPeriod = [&](int32_t walRetentionPeriod) { expect.walRetentionPeriod = walRetentionPeriod; }; auto setDbWalRetentionSize = [&](int32_t walRetentionSize) { expect.walRetentionSize = walRetentionSize; }; auto setDbWalRollPeriod = [&](int32_t walRollPeriod) { expect.walRollPeriod = walRollPeriod; }; @@ -147,6 +170,7 @@ TEST_F(ParserInitialCTest, createDatabase) { setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_DATABASE_STMT); + ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_CREATE_DB); SCreateDbReq req = {0}; ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSCreateDbReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); @@ -195,32 +219,32 @@ TEST_F(ParserInitialCTest, createDatabase) { tFreeSCreateDbReq(&req); }); - setCreateDbReqFunc("wxy_db"); + setCreateDbReq("wxy_db"); run("CREATE DATABASE wxy_db"); clearCreateDbReq(); - setCreateDbReqFunc("wxy_db", 1); - setDbBufferFunc(64); - setDbCachelastFunc(2); + setCreateDbReq("wxy_db", 1); + setDbBuffer(64); + setDbCachelast(2); setDbCachelastSize(20); - setDbCompressionFunc(1); - setDbDaysFunc(100 * 1440); - setDbFsyncFunc(100); - setDbMaxRowsFunc(1000); - setDbMinRowsFunc(100); - setDbKeepFunc(1440 * 1440); - setDbPagesFunc(96); - setDbPageSizeFunc(8); - setDbPrecisionFunc(TSDB_TIME_PRECISION_NANO); - setDbReplicaFunc(3); - addDbRetentionFunc(15 * MILLISECOND_PER_SECOND, 7 * MILLISECOND_PER_DAY, TIME_UNIT_SECOND, TIME_UNIT_DAY); - addDbRetentionFunc(1 * MILLISECOND_PER_MINUTE, 21 * MILLISECOND_PER_DAY, TIME_UNIT_MINUTE, TIME_UNIT_DAY); - addDbRetentionFunc(15 * MILLISECOND_PER_MINUTE, 500 * MILLISECOND_PER_DAY, TIME_UNIT_MINUTE, TIME_UNIT_DAY); - // setDbStrictaFunc(1); - setDbWalLevelFunc(2); - setDbVgroupsFunc(100); - setDbSingleStableFunc(1); - setDbSchemalessFunc(1); + setDbCompression(1); + setDbDays(100 * 1440); + setDbFsync(100); + setDbMaxRows(1000); + setDbMinRows(100); + setDbKeep(1440 * 1440); + setDbPages(96); + setDbPageSize(8); + setDbPrecision(TSDB_TIME_PRECISION_NANO); + setDbReplica(3); + addDbRetention(15 * MILLISECOND_PER_SECOND, 7 * MILLISECOND_PER_DAY, TIME_UNIT_SECOND, TIME_UNIT_DAY); + addDbRetention(1 * MILLISECOND_PER_MINUTE, 21 * MILLISECOND_PER_DAY, TIME_UNIT_MINUTE, TIME_UNIT_DAY); + addDbRetention(15 * MILLISECOND_PER_MINUTE, 500 * MILLISECOND_PER_DAY, TIME_UNIT_MINUTE, TIME_UNIT_DAY); + // setDbStricta(1); + setDbWalLevel(2); + setDbVgroups(100); + setDbSingleStable(1); + setDbSchemaless(1); setDbWalRetentionPeriod(-1); setDbWalRetentionSize(-1); setDbWalRollPeriod(10); @@ -259,16 +283,16 @@ TEST_F(ParserInitialCTest, createDatabase) { "TSDB_PAGESIZE 32"); clearCreateDbReq(); - setCreateDbReqFunc("wxy_db", 1); - setDbDaysFunc(100); - setDbKeepFunc(1440, 300 * 60, 400 * 1440); + setCreateDbReq("wxy_db", 1); + setDbDays(100); + setDbKeep(1440, 300 * 60, 400 * 1440); run("CREATE DATABASE IF NOT EXISTS wxy_db " "DURATION 100m " "KEEP 1440m,300h,400d "); clearCreateDbReq(); - setCreateDbReqFunc("wxy_db", 1); - setDbReplicaFunc(3); + setCreateDbReq("wxy_db", 1); + setDbReplica(3); setDbWalRetentionPeriod(TSDB_REPS_DEF_DB_WAL_RET_PERIOD); setDbWalRetentionSize(TSDB_REPS_DEF_DB_WAL_RET_SIZE); setDbWalRollPeriod(TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD); @@ -287,6 +311,9 @@ TEST_F(ParserInitialCTest, createDatabaseSemanticCheck) { run("create database db2 retentions 15s:7d,5m:21d,10m:10d", TSDB_CODE_PAR_INVALID_DB_OPTION); } +/* + * CREATE DNODE {dnode_endpoint | dnode_host_name PORT port_val} + */ TEST_F(ParserInitialCTest, createDnode) { useDb("root", "test"); @@ -294,7 +321,7 @@ TEST_F(ParserInitialCTest, createDnode) { auto clearCreateDnodeReq = [&]() { memset(&expect, 0, sizeof(SCreateDnodeReq)); }; - auto setCreateDnodeReqFunc = [&](const char* pFqdn, int32_t port = tsServerPort) { + auto setCreateDnodeReq = [&](const char* pFqdn, int32_t port = tsServerPort) { strcpy(expect.fqdn, pFqdn); expect.port = port; }; @@ -308,39 +335,41 @@ TEST_F(ParserInitialCTest, createDnode) { ASSERT_EQ(req.port, expect.port); }); - setCreateDnodeReqFunc("abc1", 7030); + setCreateDnodeReq("abc1", 7030); run("CREATE DNODE 'abc1' PORT 7030"); clearCreateDnodeReq(); - setCreateDnodeReqFunc("1.1.1.1", 8030); + setCreateDnodeReq("1.1.1.1", 8030); run("CREATE DNODE 1.1.1.1 PORT 8030"); clearCreateDnodeReq(); - setCreateDnodeReqFunc("host1", 9030); + setCreateDnodeReq("host1", 9030); run("CREATE DNODE host1 PORT 9030"); clearCreateDnodeReq(); - setCreateDnodeReqFunc("abc2", 7040); + setCreateDnodeReq("abc2", 7040); run("CREATE DNODE 'abc2:7040'"); clearCreateDnodeReq(); - setCreateDnodeReqFunc("1.1.1.2"); + setCreateDnodeReq("1.1.1.2"); run("CREATE DNODE 1.1.1.2"); clearCreateDnodeReq(); - setCreateDnodeReqFunc("host2"); + setCreateDnodeReq("host2"); run("CREATE DNODE host2"); clearCreateDnodeReq(); } -// CREATE [AGGREGATE] FUNCTION [IF NOT EXISTS] func_name AS library_path OUTPUTTYPE type_name [BUFSIZE value] +/* + * CREATE [AGGREGATE] FUNCTION [IF NOT EXISTS] func_name AS library_path OUTPUTTYPE type_name [BUFSIZE value] + */ TEST_F(ParserInitialCTest, createFunction) { useDb("root", "test"); SCreateFuncReq expect = {0}; - auto setCreateFuncReqFunc = [&](const char* pUdfName, int8_t outputType, int32_t outputBytes = 0, - int8_t funcType = TSDB_FUNC_TYPE_SCALAR, int8_t igExists = 0, int32_t bufSize = 0) { + auto setCreateFuncReq = [&](const char* pUdfName, int8_t outputType, int32_t outputBytes = 0, + int8_t funcType = TSDB_FUNC_TYPE_SCALAR, int8_t igExists = 0, int32_t bufSize = 0) { memset(&expect, 0, sizeof(SCreateFuncReq)); strcpy(expect.name, pUdfName); expect.igExists = igExists; @@ -365,13 +394,69 @@ TEST_F(ParserInitialCTest, createFunction) { ASSERT_EQ(req.bufSize, expect.bufSize); }); - setCreateFuncReqFunc("udf1", TSDB_DATA_TYPE_INT); + setCreateFuncReq("udf1", TSDB_DATA_TYPE_INT); // run("CREATE FUNCTION udf1 AS './build/lib/libudf1.so' OUTPUTTYPE INT"); - setCreateFuncReqFunc("udf2", TSDB_DATA_TYPE_DOUBLE, 0, TSDB_FUNC_TYPE_AGGREGATE, 1, 8); + setCreateFuncReq("udf2", TSDB_DATA_TYPE_DOUBLE, 0, TSDB_FUNC_TYPE_AGGREGATE, 1, 8); // run("CREATE AGGREGATE FUNCTION IF NOT EXISTS udf2 AS './build/lib/libudf2.so' OUTPUTTYPE DOUBLE BUFSIZE 8"); } +/* + * CREATE MNODE ON DNODE dnode_id + */ +TEST_F(ParserInitialCTest, createMnode) { + useDb("root", "test"); + + SMCreateMnodeReq expect = {0}; + + auto setCreateMnodeReq = [&](int32_t dnodeId) { expect.dnodeId = dnodeId; }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_MNODE_STMT); + SMCreateMnodeReq req = {0}; + ASSERT_TRUE(TSDB_CODE_SUCCESS == + tDeserializeSCreateDropMQSNodeReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); + + ASSERT_EQ(req.dnodeId, expect.dnodeId); + }); + + setCreateMnodeReq(1); + run("CREATE MNODE ON DNODE 1"); +} + +/* + * CREATE QNODE ON DNODE dnode_id + */ +TEST_F(ParserInitialCTest, createQnode) { + useDb("root", "test"); + + SMCreateQnodeReq expect = {0}; + + auto setCreateQnodeReq = [&](int32_t dnodeId) { expect.dnodeId = dnodeId; }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_QNODE_STMT); + SMCreateQnodeReq req = {0}; + ASSERT_TRUE(TSDB_CODE_SUCCESS == + tDeserializeSCreateDropMQSNodeReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); + + ASSERT_EQ(req.dnodeId, expect.dnodeId); + }); + + setCreateQnodeReq(1); + run("CREATE QNODE ON DNODE 1"); +} + +/* + * CREATE SMA INDEX index_name ON tb_name index_option + * + * index_option: + * FUNCTION(functions) INTERVAL(interval_val [, interval_offset]) [SLIDING(sliding_val)] + * [WATERMARK(watermark_val)] [MAX_DELAY(max_delay_val)] + * + * functions: + * function [, function] ... + */ TEST_F(ParserInitialCTest, createSmaIndex) { useDb("root", "test"); @@ -439,24 +524,39 @@ TEST_F(ParserInitialCTest, createSmaIndex) { "DELETE_MARK 1000s"); } -TEST_F(ParserInitialCTest, createMnode) { +/* + * CREATE SNODE ON DNODE dnode_id + */ +TEST_F(ParserInitialCTest, createSnode) { useDb("root", "test"); - run("CREATE MNODE ON DNODE 1"); -} + SMCreateSnodeReq expect = {0}; -TEST_F(ParserInitialCTest, createQnode) { - useDb("root", "test"); + auto setCreateSnodeReq = [&](int32_t dnodeId) { expect.dnodeId = dnodeId; }; - run("CREATE QNODE ON DNODE 1"); -} + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_SNODE_STMT); + SMCreateSnodeReq req = {0}; + ASSERT_TRUE(TSDB_CODE_SUCCESS == + tDeserializeSCreateDropMQSNodeReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); -TEST_F(ParserInitialCTest, createSnode) { - useDb("root", "test"); + ASSERT_EQ(req.dnodeId, expect.dnodeId); + }); + setCreateSnodeReq(1); run("CREATE SNODE ON DNODE 1"); } +/* + * CREATE STABLE [IF NOT EXISTS] stb_name (create_definition [, create_definitionn] ...) + * TAGS (create_definition [, create_definition] ...) [table_options] + * + * create_definition: + * col_name column_definition + * + * column_definition: + * type_name [COMMENT 'string_value'] + */ TEST_F(ParserInitialCTest, createStable) { useDb("root", "test"); @@ -467,7 +567,7 @@ TEST_F(ParserInitialCTest, createStable) { memset(&expect, 0, sizeof(SMCreateStbReq)); }; - auto setCreateStbReqFunc = + auto setCreateStbReq = [&](const char* pDbName, const char* pTbName, int8_t igExists = 0, int64_t delay1 = -1, int64_t delay2 = -1, int64_t watermark1 = TSDB_DEFAULT_ROLLUP_WATERMARK, int64_t watermark2 = TSDB_DEFAULT_ROLLUP_WATERMARK, int64_t deleteMark1 = TSDB_DEFAULT_ROLLUP_DELETE_MARK, int64_t deleteMark2 = TSDB_DEFAULT_ROLLUP_DELETE_MARK, @@ -488,8 +588,8 @@ TEST_F(ParserInitialCTest, createStable) { } }; - auto addFieldToCreateStbReqFunc = [&](bool col, const char* pFieldName, uint8_t type, int32_t bytes = 0, - int8_t flags = COL_SMA_ON) { + auto addFieldToCreateStbReq = [&](bool col, const char* pFieldName, uint8_t type, int32_t bytes = 0, + int8_t flags = COL_SMA_ON) { SField field = {0}; strcpy(field.name, pFieldName); field.type = type; @@ -565,46 +665,46 @@ TEST_F(ParserInitialCTest, createStable) { tFreeSMCreateStbReq(&req); }); - setCreateStbReqFunc("test", "t1"); - addFieldToCreateStbReqFunc(true, "ts", TSDB_DATA_TYPE_TIMESTAMP); - addFieldToCreateStbReqFunc(true, "c1", TSDB_DATA_TYPE_INT); - addFieldToCreateStbReqFunc(false, "id", TSDB_DATA_TYPE_INT); + setCreateStbReq("test", "t1"); + addFieldToCreateStbReq(true, "ts", TSDB_DATA_TYPE_TIMESTAMP); + addFieldToCreateStbReq(true, "c1", TSDB_DATA_TYPE_INT); + addFieldToCreateStbReq(false, "id", TSDB_DATA_TYPE_INT); run("CREATE STABLE t1(ts TIMESTAMP, c1 INT) TAGS(id INT)"); clearCreateStbReq(); - setCreateStbReqFunc("rollup_db", "t1", 1, 100 * MILLISECOND_PER_SECOND, 10 * MILLISECOND_PER_MINUTE, 10, - 1 * MILLISECOND_PER_MINUTE, 1000 * MILLISECOND_PER_SECOND, 200 * MILLISECOND_PER_MINUTE, 100, - "test create table"); - addFieldToCreateStbReqFunc(true, "ts", TSDB_DATA_TYPE_TIMESTAMP, 0, 0); - addFieldToCreateStbReqFunc(true, "c1", TSDB_DATA_TYPE_INT); - addFieldToCreateStbReqFunc(true, "c2", TSDB_DATA_TYPE_UINT); - addFieldToCreateStbReqFunc(true, "c3", TSDB_DATA_TYPE_BIGINT); - addFieldToCreateStbReqFunc(true, "c4", TSDB_DATA_TYPE_UBIGINT, 0, 0); - addFieldToCreateStbReqFunc(true, "c5", TSDB_DATA_TYPE_FLOAT, 0, 0); - addFieldToCreateStbReqFunc(true, "c6", TSDB_DATA_TYPE_DOUBLE, 0, 0); - addFieldToCreateStbReqFunc(true, "c7", TSDB_DATA_TYPE_BINARY, 20 + VARSTR_HEADER_SIZE, 0); - addFieldToCreateStbReqFunc(true, "c8", TSDB_DATA_TYPE_SMALLINT, 0, 0); - addFieldToCreateStbReqFunc(true, "c9", TSDB_DATA_TYPE_USMALLINT, 0, 0); - addFieldToCreateStbReqFunc(true, "c10", TSDB_DATA_TYPE_TINYINT, 0, 0); - addFieldToCreateStbReqFunc(true, "c11", TSDB_DATA_TYPE_UTINYINT, 0, 0); - addFieldToCreateStbReqFunc(true, "c12", TSDB_DATA_TYPE_BOOL, 0, 0); - addFieldToCreateStbReqFunc(true, "c13", TSDB_DATA_TYPE_NCHAR, 30 * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 0); - addFieldToCreateStbReqFunc(true, "c14", TSDB_DATA_TYPE_VARCHAR, 50 + VARSTR_HEADER_SIZE, 0); - addFieldToCreateStbReqFunc(false, "a1", TSDB_DATA_TYPE_TIMESTAMP); - addFieldToCreateStbReqFunc(false, "a2", TSDB_DATA_TYPE_INT); - addFieldToCreateStbReqFunc(false, "a3", TSDB_DATA_TYPE_UINT); - addFieldToCreateStbReqFunc(false, "a4", TSDB_DATA_TYPE_BIGINT); - addFieldToCreateStbReqFunc(false, "a5", TSDB_DATA_TYPE_UBIGINT); - addFieldToCreateStbReqFunc(false, "a6", TSDB_DATA_TYPE_FLOAT); - addFieldToCreateStbReqFunc(false, "a7", TSDB_DATA_TYPE_DOUBLE); - addFieldToCreateStbReqFunc(false, "a8", TSDB_DATA_TYPE_BINARY, 20 + VARSTR_HEADER_SIZE); - addFieldToCreateStbReqFunc(false, "a9", TSDB_DATA_TYPE_SMALLINT); - addFieldToCreateStbReqFunc(false, "a10", TSDB_DATA_TYPE_USMALLINT); - addFieldToCreateStbReqFunc(false, "a11", TSDB_DATA_TYPE_TINYINT); - addFieldToCreateStbReqFunc(false, "a12", TSDB_DATA_TYPE_UTINYINT); - addFieldToCreateStbReqFunc(false, "a13", TSDB_DATA_TYPE_BOOL); - addFieldToCreateStbReqFunc(false, "a14", TSDB_DATA_TYPE_NCHAR, 30 * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); - addFieldToCreateStbReqFunc(false, "a15", TSDB_DATA_TYPE_VARCHAR, 50 + VARSTR_HEADER_SIZE); + setCreateStbReq("rollup_db", "t1", 1, 100 * MILLISECOND_PER_SECOND, 10 * MILLISECOND_PER_MINUTE, 10, + 1 * MILLISECOND_PER_MINUTE, 1000 * MILLISECOND_PER_SECOND, 200 * MILLISECOND_PER_MINUTE, 100, + "test create table"); + addFieldToCreateStbReq(true, "ts", TSDB_DATA_TYPE_TIMESTAMP, 0, 0); + addFieldToCreateStbReq(true, "c1", TSDB_DATA_TYPE_INT); + addFieldToCreateStbReq(true, "c2", TSDB_DATA_TYPE_UINT); + addFieldToCreateStbReq(true, "c3", TSDB_DATA_TYPE_BIGINT); + addFieldToCreateStbReq(true, "c4", TSDB_DATA_TYPE_UBIGINT, 0, 0); + addFieldToCreateStbReq(true, "c5", TSDB_DATA_TYPE_FLOAT, 0, 0); + addFieldToCreateStbReq(true, "c6", TSDB_DATA_TYPE_DOUBLE, 0, 0); + addFieldToCreateStbReq(true, "c7", TSDB_DATA_TYPE_BINARY, 20 + VARSTR_HEADER_SIZE, 0); + addFieldToCreateStbReq(true, "c8", TSDB_DATA_TYPE_SMALLINT, 0, 0); + addFieldToCreateStbReq(true, "c9", TSDB_DATA_TYPE_USMALLINT, 0, 0); + addFieldToCreateStbReq(true, "c10", TSDB_DATA_TYPE_TINYINT, 0, 0); + addFieldToCreateStbReq(true, "c11", TSDB_DATA_TYPE_UTINYINT, 0, 0); + addFieldToCreateStbReq(true, "c12", TSDB_DATA_TYPE_BOOL, 0, 0); + addFieldToCreateStbReq(true, "c13", TSDB_DATA_TYPE_NCHAR, 30 * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 0); + addFieldToCreateStbReq(true, "c14", TSDB_DATA_TYPE_VARCHAR, 50 + VARSTR_HEADER_SIZE, 0); + addFieldToCreateStbReq(false, "a1", TSDB_DATA_TYPE_TIMESTAMP); + addFieldToCreateStbReq(false, "a2", TSDB_DATA_TYPE_INT); + addFieldToCreateStbReq(false, "a3", TSDB_DATA_TYPE_UINT); + addFieldToCreateStbReq(false, "a4", TSDB_DATA_TYPE_BIGINT); + addFieldToCreateStbReq(false, "a5", TSDB_DATA_TYPE_UBIGINT); + addFieldToCreateStbReq(false, "a6", TSDB_DATA_TYPE_FLOAT); + addFieldToCreateStbReq(false, "a7", TSDB_DATA_TYPE_DOUBLE); + addFieldToCreateStbReq(false, "a8", TSDB_DATA_TYPE_BINARY, 20 + VARSTR_HEADER_SIZE); + addFieldToCreateStbReq(false, "a9", TSDB_DATA_TYPE_SMALLINT); + addFieldToCreateStbReq(false, "a10", TSDB_DATA_TYPE_USMALLINT); + addFieldToCreateStbReq(false, "a11", TSDB_DATA_TYPE_TINYINT); + addFieldToCreateStbReq(false, "a12", TSDB_DATA_TYPE_UTINYINT); + addFieldToCreateStbReq(false, "a13", TSDB_DATA_TYPE_BOOL); + addFieldToCreateStbReq(false, "a14", TSDB_DATA_TYPE_NCHAR, 30 * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); + addFieldToCreateStbReq(false, "a15", TSDB_DATA_TYPE_VARCHAR, 50 + VARSTR_HEADER_SIZE); run("CREATE STABLE IF NOT EXISTS rollup_db.t1(" "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), " "c8 SMALLINT, c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " @@ -630,6 +730,20 @@ TEST_F(ParserInitialCTest, createStableSemanticCheck) { TSDB_CODE_PAR_INVALID_TABLE_OPTION); } +/* + * CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] + * INTO stb_name [TAGS (create_definition [, create_definition] ...)] [SUBTABLE (expr)] AS subquery + * + * stream_options: + * stream_option ... + * + * stream_option: { + * TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time] + * | WATERMARK time + * | IGNORE EXPIRED value + * | FILL_HISTORY value + * } + */ TEST_F(ParserInitialCTest, createStream) { useDb("root", "test"); @@ -641,14 +755,23 @@ TEST_F(ParserInitialCTest, createStream) { }; auto setCreateStreamReq = [&](const char* pStream, const char* pSrcDb, const char* pSql, const char* pDstStb, - int8_t igExists = 0, int8_t triggerType = STREAM_TRIGGER_AT_ONCE, int64_t maxDelay = 0, - int64_t watermark = 0, int8_t igExpired = STREAM_DEFAULT_IGNORE_EXPIRED, - int8_t fillHistory = STREAM_DEFAULT_FILL_HISTORY) { + int8_t createStb = STREAM_CREATE_STABLE_TRUE, int8_t igExists = 0) { snprintf(expect.name, sizeof(expect.name), "0.%s", pStream); snprintf(expect.sourceDB, sizeof(expect.sourceDB), "0.%s", pSrcDb); snprintf(expect.targetStbFullName, sizeof(expect.targetStbFullName), "0.test.%s", pDstStb); expect.igExists = igExists; expect.sql = strdup(pSql); + expect.createStb = createStb; + expect.triggerType = STREAM_TRIGGER_AT_ONCE; + expect.maxDelay = 0; + expect.watermark = 0; + expect.fillHistory = STREAM_DEFAULT_FILL_HISTORY; + expect.igExpired = STREAM_DEFAULT_IGNORE_EXPIRED; + }; + + auto setStreamOptions = [&](int8_t triggerType = STREAM_TRIGGER_AT_ONCE, int64_t maxDelay = 0, int64_t watermark = 0, + int8_t igExpired = STREAM_DEFAULT_IGNORE_EXPIRED, + int8_t fillHistory = STREAM_DEFAULT_FILL_HISTORY) { expect.triggerType = triggerType; expect.maxDelay = maxDelay; expect.watermark = watermark; @@ -699,19 +822,22 @@ TEST_F(ParserInitialCTest, createStream) { ASSERT_EQ(pField->flags, pExpectField->flags); } } + ASSERT_EQ(req.checkpointFreq, expect.checkpointFreq); + ASSERT_EQ(req.createStb, expect.createStb); tFreeSCMCreateStreamReq(&req); }); - setCreateStreamReq("s1", "test", "create stream s1 into st1 as select count(*) from t1 interval(10s)", "st1"); - run("CREATE STREAM s1 INTO st1 AS SELECT COUNT(*) FROM t1 INTERVAL(10S)"); + setCreateStreamReq("s1", "test", "create stream s1 into st3 as select count(*) from t1 interval(10s)", "st3"); + run("CREATE STREAM s1 INTO st3 AS SELECT COUNT(*) FROM t1 INTERVAL(10S)"); clearCreateStreamReq(); setCreateStreamReq( "s1", "test", - "create stream if not exists s1 trigger max_delay 20s watermark 10s ignore expired 0 fill_history 1 into st1 " + "create stream if not exists s1 trigger max_delay 20s watermark 10s ignore expired 0 fill_history 1 into st3 " "as select count(*) from t1 interval(10s)", - "st1", 1, STREAM_TRIGGER_MAX_DELAY, 20 * MILLISECOND_PER_SECOND, 10 * MILLISECOND_PER_SECOND, 0, 1); - run("CREATE STREAM IF NOT EXISTS s1 TRIGGER MAX_DELAY 20s WATERMARK 10s IGNORE EXPIRED 0 FILL_HISTORY 1 INTO st1 AS " + "st3", 1, 1); + setStreamOptions(STREAM_TRIGGER_MAX_DELAY, 20 * MILLISECOND_PER_SECOND, 10 * MILLISECOND_PER_SECOND, 0, 1); + run("CREATE STREAM IF NOT EXISTS s1 TRIGGER MAX_DELAY 20s WATERMARK 10s IGNORE EXPIRED 0 FILL_HISTORY 1 INTO st3 AS " "SELECT COUNT(*) " "FROM t1 INTERVAL(10S)"); clearCreateStreamReq(); @@ -725,6 +851,11 @@ TEST_F(ParserInitialCTest, createStream) { run("CREATE STREAM s1 INTO st3 TAGS(tname VARCHAR(10), id INT) SUBTABLE(CONCAT('new-', tname)) " "AS SELECT _WSTART wstart, COUNT(*) cnt FROM st1 PARTITION BY TBNAME tname, tag1 id INTERVAL(10S)"); clearCreateStreamReq(); + + setCreateStreamReq("s1", "test", "create stream s1 into st1 as select max(c1), c2 from t1 interval(10s)", "st1", + STREAM_CREATE_STABLE_FALSE); + run("CREATE STREAM s1 INTO st1 AS SELECT MAX(c1), c2 FROM t1 INTERVAL(10S)"); + clearCreateStreamReq(); } TEST_F(ParserInitialCTest, createStreamSemanticCheck) { @@ -734,9 +865,103 @@ TEST_F(ParserInitialCTest, createStreamSemanticCheck) { TSDB_CODE_PAR_STREAM_NOT_ALLOWED_FUNC); } +/* + * CREATE TABLE [IF NOT EXISTS] [db_name.]tb_name (create_definition [, create_definitionn] ...) [table_options] + * + * CREATE TABLE create_subtable_clause + * + * CREATE TABLE [IF NOT EXISTS] [db_name.]tb_name (create_definition [, create_definitionn] ...) + * [TAGS (create_definition [, create_definitionn] ...)] + * [table_options] + * + * create_subtable_clause: { + * create_subtable_clause [create_subtable_clause] ... + * | [IF NOT EXISTS] [db_name.]tb_name USING [db_name.]stb_name [(tag_name [, tag_name] ...)] + * TAGS (tag_value [, tag_value] ...) + * } + * + * create_definition: + * col_name column_definition + * + * column_definition: + * type_name [comment 'string_value'] + * + * table_options: + * table_option ... + * + * table_option: { + * COMMENT 'string_value' + * | WATERMARK duration[,duration] + * | MAX_DELAY duration[,duration] + * | ROLLUP(func_name [, func_name] ...) + * | SMA(col_name [, col_name] ...) + * | TTL value + * } + */ TEST_F(ParserInitialCTest, createTable) { useDb("root", "test"); + SVCreateTbBatchReq expect = {0}; + + auto addCreateTbReq = [&](const char* pName, bool ignoreExists = false, int32_t ttl = TSDB_DEFAULT_TABLE_TTL, + const char* pComment = nullptr) { + SVCreateTbReq req = {0}; + req.name = strdup(pName); + if (ignoreExists) { + req.flags |= TD_CREATE_IF_NOT_EXISTS; + } + req.ttl = ttl; + if (nullptr != pComment) { + req.comment = strdup(pComment); + req.commentLen = strlen(pComment); + } + ++expect.nReqs; + if (nullptr == expect.pArray) { + expect.pArray = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVCreateTbReq)); + } + taosArrayPush(expect.pArray, &req); + }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + return; // todo + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_VNODE_MODIFY_STMT); + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; + + ASSERT_EQ(pStmt->sqlNodeType, QUERY_NODE_CREATE_TABLE_STMT); + ASSERT_NE(pStmt->pDataBlocks, nullptr); + int32_t numOfBlocks = taosArrayGetSize(pStmt->pDataBlocks); + for (int32_t i = 0; i < numOfBlocks; ++i) { + SVgDataBlocks* pVgData = (SVgDataBlocks*)taosArrayGetP(pStmt->pDataBlocks, i); + void* pBuf = POINTER_SHIFT(pVgData->pData, sizeof(SMsgHead)); + SVCreateTbBatchReq req = {0}; + SDecoder coder = {0}; + tDecoderInit(&coder, (uint8_t*)pBuf, pVgData->size); + ASSERT_EQ(tDecodeSVCreateTbBatchReq(&coder, &req), TSDB_CODE_SUCCESS); + ASSERT_EQ(req.nReqs, expect.nReqs); + for (int32_t j = 0; j < req.nReqs; ++j) { + SVCreateTbReq* pReq = req.pReqs + j; + SVCreateTbReq* pExpect = (SVCreateTbReq*)taosArrayGet(expect.pArray, j); + ASSERT_EQ(pReq->flags, pExpect->flags); + ASSERT_EQ(std::string(pReq->name), std::string(pExpect->name)); + ASSERT_EQ(pReq->uid, pExpect->uid); + ASSERT_EQ(pReq->ctime, pExpect->ctime); + ASSERT_EQ(pReq->ttl, pExpect->ttl); + ASSERT_EQ(pReq->commentLen, pExpect->commentLen); + ASSERT_EQ(std::string(pReq->comment), std::string(pExpect->comment)); + ASSERT_EQ(pReq->type, pExpect->type); + if (TD_NORMAL_TABLE == pExpect->type) { + ASSERT_EQ(pReq->ntb.schemaRow.version, pExpect->ntb.schemaRow.version); + ASSERT_EQ(pReq->ntb.schemaRow.nCols, pExpect->ntb.schemaRow.nCols); + } else if (TD_CHILD_TABLE == pExpect->type) { + ASSERT_EQ(std::string(pReq->ctb.stbName), std::string(pExpect->ctb.stbName)); + ASSERT_EQ(pReq->ctb.tagNum, pExpect->ctb.tagNum); + ASSERT_EQ(pReq->ctb.suid, pExpect->ctb.suid); + } + } + tDecoderClear(&coder); + } + }); + run("CREATE TABLE t1(ts TIMESTAMP, c1 INT)"); run("CREATE TABLE IF NOT EXISTS test.t1(" @@ -761,7 +986,7 @@ TEST_F(ParserInitialCTest, createTable) { "IF NOT EXISTS test.t2 USING test.st1 (tag1, tag2) TAGS(2, 'abc') " "IF NOT EXISTS test.t3 USING test.st1 (tag1, tag2) TAGS(3, 'abc') "); - // run("CREATE TABLE IF NOT EXISTS t1 USING st1 TAGS(1, 'wxy', NOW + 1S)"); + run("CREATE TABLE IF NOT EXISTS t1 USING st1 TAGS(1, 'wxy', NOW + 1S)"); } TEST_F(ParserInitialCTest, createTableSemanticCheck) { @@ -779,6 +1004,11 @@ TEST_F(ParserInitialCTest, createTableSemanticCheck) { run(sql, TSDB_CODE_PAR_TOO_MANY_COLUMNS); } +/* + * CREATE TOPIC [IF NOT EXISTS] topic_name AS subquery + * + * CREATE TOPIC [IF NOT EXISTS] topic_name [WITH META] AS {DATABASE db_name | STABLE stb_name } + */ TEST_F(ParserInitialCTest, createTopic) { useDb("root", "test"); @@ -786,8 +1016,8 @@ TEST_F(ParserInitialCTest, createTopic) { auto clearCreateTopicReq = [&]() { memset(&expect, 0, sizeof(SCMCreateTopicReq)); }; - auto setCreateTopicReqFunc = [&](const char* pTopicName, int8_t igExists, const char* pSql, const char* pAst, - const char* pDbName = nullptr, const char* pTbname = nullptr, int8_t withMeta = 0) { + auto setCreateTopicReq = [&](const char* pTopicName, int8_t igExists, const char* pSql, const char* pAst, + const char* pDbName = nullptr, const char* pTbname = nullptr, int8_t withMeta = 0) { snprintf(expect.name, sizeof(expect.name), "0.%s", pTopicName); expect.igExists = igExists; expect.sql = (char*)pSql; @@ -831,31 +1061,34 @@ TEST_F(ParserInitialCTest, createTopic) { tFreeSCMCreateTopicReq(&req); }); - setCreateTopicReqFunc("tp1", 0, "create topic tp1 as select * from t1", "ast"); + setCreateTopicReq("tp1", 0, "create topic tp1 as select * from t1", "ast"); run("CREATE TOPIC tp1 AS SELECT * FROM t1"); clearCreateTopicReq(); - setCreateTopicReqFunc("tp1", 1, "create topic if not exists tp1 as select ts, ceil(c1) from t1", "ast"); + setCreateTopicReq("tp1", 1, "create topic if not exists tp1 as select ts, ceil(c1) from t1", "ast"); run("CREATE TOPIC IF NOT EXISTS tp1 AS SELECT ts, CEIL(c1) FROM t1"); clearCreateTopicReq(); - setCreateTopicReqFunc("tp1", 0, "create topic tp1 as database test", nullptr, "test"); + setCreateTopicReq("tp1", 0, "create topic tp1 as database test", nullptr, "test"); run("CREATE TOPIC tp1 AS DATABASE test"); clearCreateTopicReq(); - setCreateTopicReqFunc("tp1", 0, "create topic tp1 with meta as database test", nullptr, "test", nullptr, 1); + setCreateTopicReq("tp1", 0, "create topic tp1 with meta as database test", nullptr, "test", nullptr, 1); run("CREATE TOPIC tp1 WITH META AS DATABASE test"); clearCreateTopicReq(); - setCreateTopicReqFunc("tp1", 1, "create topic if not exists tp1 as stable st1", nullptr, "test", "st1"); + setCreateTopicReq("tp1", 1, "create topic if not exists tp1 as stable st1", nullptr, "test", "st1"); run("CREATE TOPIC IF NOT EXISTS tp1 AS STABLE st1"); clearCreateTopicReq(); - setCreateTopicReqFunc("tp1", 1, "create topic if not exists tp1 with meta as stable st1", nullptr, "test", "st1", 1); + setCreateTopicReq("tp1", 1, "create topic if not exists tp1 with meta as stable st1", nullptr, "test", "st1", 1); run("CREATE TOPIC IF NOT EXISTS tp1 WITH META AS STABLE st1"); clearCreateTopicReq(); } +/* + * CREATE USER use_name PASS password [SYSINFO value] + */ TEST_F(ParserInitialCTest, createUser) { useDb("root", "test"); diff --git a/source/libs/parser/test/parInitialDTest.cpp b/source/libs/parser/test/parInitialDTest.cpp index 3ded71242b961ceaf12abbdff44272c7fa6cb32c..3d773d351070fa55f0f658dd093d4f185fa5d4ef 100644 --- a/source/libs/parser/test/parInitialDTest.cpp +++ b/source/libs/parser/test/parInitialDTest.cpp @@ -99,7 +99,7 @@ TEST_F(ParserInitialDTest, dropDnode) { expect.force = force; }; - auto setDropDnodeReqByEndpoint = [&](const char* pFqdn, int32_t port, bool force = false) { + auto setDropDnodeReqByEndpoint = [&](const char* pFqdn, int32_t port = tsServerPort, bool force = false) { strcpy(expect.fqdn, pFqdn); expect.port = port; expect.force = force; @@ -131,6 +131,14 @@ TEST_F(ParserInitialDTest, dropDnode) { setDropDnodeReqByEndpoint("host2", 8030, true); run("DROP DNODE 'host2:8030' FORCE"); clearDropDnodeReq(); + + setDropDnodeReqByEndpoint("host1"); + run("DROP DNODE host1"); + clearDropDnodeReq(); + + setDropDnodeReqByEndpoint("host2", tsServerPort, true); + run("DROP DNODE host2 FORCE"); + clearDropDnodeReq(); } // todo DROP function @@ -174,7 +182,21 @@ TEST_F(ParserInitialDTest, dropMnode) { TEST_F(ParserInitialDTest, dropQnode) { useDb("root", "test"); - run("DROP qnode on dnode 1"); + SMDropQnodeReq expect = {0}; + + auto setDropQnodeReq = [&](int32_t dnodeId) { expect.dnodeId = dnodeId; }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_DROP_QNODE_STMT); + SMDropQnodeReq req = {0}; + ASSERT_TRUE(TSDB_CODE_SUCCESS == + tDeserializeSCreateDropMQSNodeReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); + + ASSERT_EQ(req.dnodeId, expect.dnodeId); + }); + + setDropQnodeReq(1); + run("DROP QNODE ON DNODE 1"); } TEST_F(ParserInitialDTest, dropSnode) { @@ -237,7 +259,20 @@ TEST_F(ParserInitialDTest, dropUser) { login("root"); useDb("root", "test"); - run("DROP user wxy"); + SDropUserReq expect = {0}; + + auto setDropUserReq = [&](const char* pUser) { sprintf(expect.user, "%s", pUser); }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_DROP_USER_STMT); + SDropUserReq req = {0}; + ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSDropUserReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); + + ASSERT_EQ(std::string(req.user), std::string(expect.user)); + }); + + setDropUserReq("wxy"); + run("DROP USER wxy"); } } // namespace ParserTest diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 084d99cae5102b62efaa82b45a8e86b4aa7fa48e..fef5bd654e04bee0618aa8d25fd2e2c6d46e1b39 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -815,6 +815,29 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode); } +static int32_t createWindowLogicNodeByEvent(SLogicPlanContext* pCxt, SEventWindowNode* pEvent, SSelectStmt* pSelect, + SLogicNode** pLogicNode) { + SWindowLogicNode* pWindow = (SWindowLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_WINDOW); + if (NULL == pWindow) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pWindow->winType = WINDOW_TYPE_EVENT; + pWindow->node.groupAction = getGroupAction(pCxt, pSelect); + pWindow->node.requireDataOrder = + pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : getRequireDataOrder(true, pSelect); + pWindow->node.resultDataOrder = + pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_GLOBAL : pWindow->node.requireDataOrder; + pWindow->pStartCond = nodesCloneNode(pEvent->pStartCond); + pWindow->pEndCond = nodesCloneNode(pEvent->pEndCond); + pWindow->pTspk = nodesCloneNode(pEvent->pCol); + if (NULL == pWindow->pStartCond || NULL == pWindow->pEndCond || NULL == pWindow->pTspk) { + nodesDestroyNode((SNode*)pWindow); + return TSDB_CODE_OUT_OF_MEMORY; + } + return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode); +} + static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { if (NULL == pSelect->pWindow) { return TSDB_CODE_SUCCESS; @@ -827,6 +850,8 @@ static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele return createWindowLogicNodeBySession(pCxt, (SSessionWindowNode*)pSelect->pWindow, pSelect, pLogicNode); case QUERY_NODE_INTERVAL_WINDOW: return createWindowLogicNodeByInterval(pCxt, (SIntervalWindowNode*)pSelect->pWindow, pSelect, pLogicNode); + case QUERY_NODE_EVENT_WINDOW: + return createWindowLogicNodeByEvent(pCxt, (SEventWindowNode*)pSelect->pWindow, pSelect, pLogicNode); default: break; } @@ -1363,7 +1388,7 @@ static int32_t createSetOperatorLogicNode(SLogicPlanContext* pCxt, SSetOperator* static int32_t getMsgType(ENodeType sqlType) { switch (sqlType) { case QUERY_NODE_CREATE_TABLE_STMT: - case QUERY_NODE_CREATE_MULTI_TABLE_STMT: + case QUERY_NODE_CREATE_MULTI_TABLES_STMT: return TDMT_VND_CREATE_TABLE; case QUERY_NODE_DROP_TABLE_STMT: return TDMT_VND_DROP_TABLE; @@ -1377,7 +1402,7 @@ static int32_t getMsgType(ENodeType sqlType) { return TDMT_VND_SUBMIT; } -static int32_t createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifOpStmt* pStmt, SLogicNode** pLogicNode) { +static int32_t createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifyOpStmt* pStmt, SLogicNode** pLogicNode) { SVnodeModifyLogicNode* pModif = (SVnodeModifyLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY); if (NULL == pModif) { return TSDB_CODE_OUT_OF_MEMORY; @@ -1561,8 +1586,8 @@ static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogi switch (nodeType(pStmt)) { case QUERY_NODE_SELECT_STMT: return createSelectLogicNode(pCxt, (SSelectStmt*)pStmt, pLogicNode); - case QUERY_NODE_VNODE_MODIF_STMT: - return createVnodeModifLogicNode(pCxt, (SVnodeModifOpStmt*)pStmt, pLogicNode); + case QUERY_NODE_VNODE_MODIFY_STMT: + return createVnodeModifLogicNode(pCxt, (SVnodeModifyOpStmt*)pStmt, pLogicNode); case QUERY_NODE_EXPLAIN_STMT: return createQueryLogicNode(pCxt, ((SExplainStmt*)pStmt)->pQuery, pLogicNode); case QUERY_NODE_SET_OPERATOR: diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 03e0275d7d9a5fc7874dd1a54e4fc24973c81776..120a8537417562a4002268ba4f897a3811b0eaa4 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1284,18 +1284,14 @@ static int32_t smaIndexOptFindSmaFunc(SNode* pQueryFunc, SNodeList* pSmaFuncs) { return -1; } -static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, SNodeList** pOutput, - int32_t* pWStrartIndex) { +static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, + SNodeList** pOutput) { SNodeList* pCols = NULL; SNode* pFunc = NULL; int32_t code = TSDB_CODE_SUCCESS; int32_t index = 0; int32_t smaFuncIndex = -1; - *pWStrartIndex = -1; FOREACH(pFunc, pFuncs) { - if (FUNCTION_TYPE_WSTART == ((SFunctionNode*)pFunc)->funcType) { - *pWStrartIndex = index; - } smaFuncIndex = smaIndexOptFindSmaFunc(pFunc, pSmaFuncs); if (smaFuncIndex < 0) { break; @@ -1317,8 +1313,7 @@ static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNo return code; } -static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList** pCols, - int32_t* pWStrartIndex) { +static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList** pCols) { SWindowLogicNode* pWindow = (SWindowLogicNode*)pScan->node.pParent; if (!smaIndexOptEqualInterval(pScan, pWindow, pIndex)) { return TSDB_CODE_SUCCESS; @@ -1326,14 +1321,14 @@ static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo SNodeList* pSmaFuncs = NULL; int32_t code = nodesStringToList(pIndex->expr, &pSmaFuncs); if (TSDB_CODE_SUCCESS == code) { - code = smaIndexOptCreateSmaCols(pWindow->pFuncs, pIndex->dstTbUid, pSmaFuncs, pCols, pWStrartIndex); + code = smaIndexOptCreateSmaCols(pWindow->pFuncs, pIndex->dstTbUid, pSmaFuncs, pCols); } nodesDestroyList(pSmaFuncs); return code; } static int32_t smaIndexOptApplyIndex(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, - SNodeList* pSmaCols, int32_t wstrartIndex) { + SNodeList* pSmaCols) { SLogicNode* pSmaScan = NULL; int32_t code = smaIndexOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); if (TSDB_CODE_SUCCESS == code) { @@ -1350,10 +1345,9 @@ static int32_t smaIndexOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogi for (int32_t i = 0; i < nindexes; ++i) { STableIndexInfo* pIndex = taosArrayGet(pScan->pSmaIndexes, i); SNodeList* pSmaCols = NULL; - int32_t wstrartIndex = -1; - code = smaIndexOptCouldApplyIndex(pScan, pIndex, &pSmaCols, &wstrartIndex); + code = smaIndexOptCouldApplyIndex(pScan, pIndex, &pSmaCols); if (TSDB_CODE_SUCCESS == code && NULL != pSmaCols) { - code = smaIndexOptApplyIndex(pLogicSubplan, pScan, pIndex, pSmaCols, wstrartIndex); + code = smaIndexOptApplyIndex(pLogicSubplan, pScan, pIndex, pSmaCols); taosArrayDestroyEx(pScan->pSmaIndexes, smaIndexOptDestroySmaIndex); pScan->pSmaIndexes = NULL; pCxt->optimized = true; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index df10fe8ee33e29f1153cecadb58228e18dcb66c9..7d6238193d7e6411cffd9bd8b092afa01e651b7a 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1298,6 +1298,33 @@ static int32_t createStateWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pC return code; } +static int32_t createEventWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, + SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { + SEventWinodwPhysiNode* pEvent = (SEventWinodwPhysiNode*)makePhysiNode( + pCxt, (SLogicNode*)pWindowLogicNode, + (pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT : QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT)); + if (NULL == pEvent) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); + int32_t code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pWindowLogicNode->pStartCond, &pEvent->pStartCond); + if (TSDB_CODE_SUCCESS == code) { + code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pWindowLogicNode->pEndCond, &pEvent->pEndCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = createWindowPhysiNodeFinalize(pCxt, pChildren, &pEvent->window, pWindowLogicNode); + } + + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pEvent; + } else { + nodesDestroyNode((SNode*)pEvent); + } + + return code; +} + static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { switch (pWindowLogicNode->winType) { @@ -1307,6 +1334,8 @@ static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildr return createSessionWindowPhysiNode(pCxt, pChildren, pWindowLogicNode, pPhyNode); case WINDOW_TYPE_STATE: return createStateWindowPhysiNode(pCxt, pChildren, pWindowLogicNode, pPhyNode); + case WINDOW_TYPE_EVENT: + return createEventWindowPhysiNode(pCxt, pChildren, pWindowLogicNode, pPhyNode); default: break; } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index a7eac2c853659c3526a3c0527d64734a50e95115..6208fc172aa4fd9a55b259f794e0d9e266bfabea 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -739,6 +739,18 @@ static int32_t stbSplSplitState(SSplitContext* pCxt, SStableSplitInfo* pInfo) { } } +static int32_t stbSplSplitEventForStream(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + return TSDB_CODE_PLAN_INTERNAL_ERROR; +} + +static int32_t stbSplSplitEvent(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + if (pCxt->pPlanCxt->streamQuery) { + return stbSplSplitEventForStream(pCxt, pInfo); + } else { + return stbSplSplitSessionOrStateForBatch(pCxt, pInfo); + } +} + static bool stbSplIsPartTableWinodw(SWindowLogicNode* pWindow) { return stbSplHasPartTbname(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pWindow->node.pChildren, 0))); } @@ -751,6 +763,8 @@ static int32_t stbSplSplitWindowForCrossTable(SSplitContext* pCxt, SStableSplitI return stbSplSplitSession(pCxt, pInfo); case WINDOW_TYPE_STATE: return stbSplSplitState(pCxt, pInfo); + case WINDOW_TYPE_EVENT: + return stbSplSplitEvent(pCxt, pInfo); default: break; } diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index fd6cb730f8e7c9109e089b06aaf20cd8fd88ca28..4b8009347d5e44ce236f99652b4814e36b8af459 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -200,6 +200,15 @@ static int32_t adjustStateDataRequirement(SWindowLogicNode* pWindow, EDataOrderL return TSDB_CODE_SUCCESS; } +static int32_t adjustEventDataRequirement(SWindowLogicNode* pWindow, EDataOrderLevel requirement) { + if (requirement <= pWindow->node.resultDataOrder) { + return TSDB_CODE_SUCCESS; + } + pWindow->node.resultDataOrder = requirement; + pWindow->node.requireDataOrder = requirement; + return TSDB_CODE_SUCCESS; +} + static int32_t adjustWindowDataRequirement(SWindowLogicNode* pWindow, EDataOrderLevel requirement) { switch (pWindow->winType) { case WINDOW_TYPE_INTERVAL: @@ -208,6 +217,8 @@ static int32_t adjustWindowDataRequirement(SWindowLogicNode* pWindow, EDataOrder return adjustSessionDataRequirement(pWindow, requirement); case WINDOW_TYPE_STATE: return adjustStateDataRequirement(pWindow, requirement); + case WINDOW_TYPE_EVENT: + return adjustEventDataRequirement(pWindow, requirement); default: break; } diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp index 150df76416d71514fc226f35b48ec4f57701d31b..10ab71ab0e68e2a31c798c46e3c2f685a70d2201 100644 --- a/source/libs/planner/test/planBasicTest.cpp +++ b/source/libs/planner/test/planBasicTest.cpp @@ -104,6 +104,8 @@ TEST_F(PlanBasicTest, interpFunc) { run("SELECT _IROWTS, INTERP(c1) FROM t1 RANGE('2017-7-14 18:00:00', '2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR)"); + run("SELECT _IROWTS, INTERP(c1), _ISFILLED FROM t1 RANGE('2017-7-14 18:00:00', '2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR)"); + run("SELECT TBNAME, _IROWTS, INTERP(c1) FROM t1 PARTITION BY TBNAME " "RANGE('2017-7-14 18:00:00', '2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR)"); } diff --git a/source/libs/planner/test/planEventTest.cpp b/source/libs/planner/test/planEventTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c4db1459982de3c8daf6b8b37795a806ddd77d01 --- /dev/null +++ b/source/libs/planner/test/planEventTest.cpp @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanEventTest : public PlannerTestBase {}; + +TEST_F(PlanEventTest, basic) { + useDb("root", "test"); + + run("SELECT COUNT(*) FROM t1 EVENT_WINDOW START WITH c1 > 10 END WITH c2 = 'abc'"); +} + +TEST_F(PlanEventTest, stable) { + useDb("root", "test"); + + run("SELECT COUNT(*) FROM st1 EVENT_WINDOW START WITH c1 > 10 END WITH c2 = 'abc'"); +} diff --git a/source/libs/planner/test/planOtherTest.cpp b/source/libs/planner/test/planOtherTest.cpp index 4741d241b5ac8b5541b6f92d08dbe1dd26151099..3a12d62340f1744db5d351d7494e2e181b512d76 100644 --- a/source/libs/planner/test/planOtherTest.cpp +++ b/source/libs/planner/test/planOtherTest.cpp @@ -30,7 +30,7 @@ TEST_F(PlanOtherTest, createTopic) { TEST_F(PlanOtherTest, createStream) { useDb("root", "test"); - run("create stream if not exists s1 trigger window_close watermark 10s into st1 as select count(*) from t1 " + run("create stream if not exists s1 trigger window_close watermark 10s into st3 as select count(*) from t1 " "interval(10s)"); run("CREATE STREAM s1 INTO st3 TAGS(tname VARCHAR(10), id INT) SUBTABLE(CONCAT('new-', tname)) " @@ -43,9 +43,9 @@ TEST_F(PlanOtherTest, createStream) { TEST_F(PlanOtherTest, createStreamUseSTable) { useDb("root", "test"); - run("CREATE STREAM IF NOT EXISTS s1 into st1 as SELECT COUNT(*) FROM st1 INTERVAL(10s)"); + run("CREATE STREAM IF NOT EXISTS s1 into st3 as SELECT COUNT(*) FROM st1 INTERVAL(10s)"); - run("CREATE STREAM IF NOT EXISTS s1 into st1 as SELECT COUNT(*) FROM st1 PARTITION BY TBNAME INTERVAL(10s)"); + run("CREATE STREAM IF NOT EXISTS s1 into st3 as SELECT COUNT(*) FROM st1 PARTITION BY TBNAME INTERVAL(10s)"); } TEST_F(PlanOtherTest, createSmaIndex) { diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index c74882cf23dac901eab09b779afd56cbfb55dcaa..d8cd5f4e751ede58190a4458eb0e348e63210b00 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -116,8 +116,6 @@ int32_t cleanupTaskQueue() { } static void execHelper(struct SSchedMsg* pSchedMsg) { - assert(pSchedMsg != NULL && pSchedMsg->ahandle != NULL); - __async_exec_fn_t execFn = (__async_exec_fn_t)pSchedMsg->ahandle; int32_t code = execFn(pSchedMsg->thandle); if (code != 0 && pSchedMsg->msg != NULL) { @@ -126,8 +124,6 @@ static void execHelper(struct SSchedMsg* pSchedMsg) { } int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) { - assert(execFn != NULL); - SSchedMsg schedMsg = {0}; schedMsg.fp = execHelper; schedMsg.ahandle = execFn; @@ -138,7 +134,10 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) } void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { - assert(pMsgBody != NULL); + if (NULL == pMsgBody) { + return; + } + taosMemoryFreeClear(pMsgBody->target.dbFName); taosMemoryFreeClear(pMsgBody->msgInfo.pData); if (pMsgBody->paramFreeFp) { @@ -243,7 +242,8 @@ void destroyQueryExecRes(SExecResult* pRes) { break; } case TDMT_VND_SUBMIT: { - tFreeSSubmitRsp((SSubmitRsp*)pRes->res); + tDestroySSubmitRsp2((SSubmitRsp2*)pRes->res, TSDB_MSG_FLG_DECODE); + taosMemoryFreeClear(pRes->res); break; } case TDMT_SCH_QUERY: @@ -393,7 +393,7 @@ char* parseTagDatatoJson(void* p) { } else if (pTagVal->nData == 0) { value = cJSON_CreateString(""); } else { - ASSERT(0); + goto end; } cJSON_AddItemToObject(json, tagJsonKey, value); @@ -412,7 +412,7 @@ char* parseTagDatatoJson(void* p) { } cJSON_AddItemToObject(json, tagJsonKey, value); } else { - ASSERT(0); + goto end; } } string = cJSON_PrintUnformatted(json); @@ -447,7 +447,6 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { return TSDB_CODE_SUCCESS; } - void freeVgInfo(SDBVgInfo* vgInfo) { if (NULL == vgInfo) { return; @@ -459,7 +458,6 @@ void freeVgInfo(SDBVgInfo* vgInfo) { taosMemoryFreeClear(vgInfo); } - int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { if (NULL == pSrc) { *pDst = NULL; @@ -498,3 +496,53 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { return TSDB_CODE_SUCCESS; } + +int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst) { + if (NULL == pSrc) { + *pDst = NULL; + return TSDB_CODE_SUCCESS; + } + + *pDst = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == *pDst) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + (*pDst)->flags = pSrc->flags; + if (pSrc->name) { + (*pDst)->name = strdup(pSrc->name); + } + (*pDst)->uid = pSrc->uid; + (*pDst)->ctime = pSrc->ctime; + (*pDst)->ttl = pSrc->ttl; + (*pDst)->commentLen = pSrc->commentLen; + if (pSrc->comment) { + (*pDst)->comment = strdup(pSrc->comment); + } + (*pDst)->type = pSrc->type; + + if (pSrc->type == TSDB_CHILD_TABLE) { + if (pSrc->ctb.stbName) { + (*pDst)->ctb.stbName = strdup(pSrc->ctb.stbName); + } + (*pDst)->ctb.tagNum = pSrc->ctb.tagNum; + (*pDst)->ctb.suid = pSrc->ctb.suid; + if (pSrc->ctb.tagName) { + (*pDst)->ctb.tagName = taosArrayDup(pSrc->ctb.tagName, NULL); + } + STag* pTag = (STag*)pSrc->ctb.pTag; + if (pTag) { + (*pDst)->ctb.pTag = taosMemoryMalloc(pTag->len); + memcpy((*pDst)->ctb.pTag, pTag, pTag->len); + } + } else { + (*pDst)->ntb.schemaRow.nCols = pSrc->ntb.schemaRow.nCols; + (*pDst)->ntb.schemaRow.version = pSrc->ntb.schemaRow.nCols; + if (pSrc->ntb.schemaRow.nCols > 0 && pSrc->ntb.schemaRow.pSchema) { + (*pDst)->ntb.schemaRow.pSchema = taosMemoryMalloc(pSrc->ntb.schemaRow.nCols * sizeof(SSchema)); + memcpy((*pDst)->ntb.schemaRow.pSchema, pSrc->ntb.schemaRow.pSchema, pSrc->ntb.schemaRow.nCols * sizeof(SSchema)); + } + } + + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 35b2479a51b17877fd93103b03e6e6f726324ac4..66b8e48eb1a41d5669e008d3296359f208e0a19f 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -325,34 +325,34 @@ typedef struct SQWorkerMgmt { #define QW_LOCK(type, _lock) \ do { \ if (QW_READ == (type)) { \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before read lock"); \ QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRLockLatch(_lock); \ QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) > 0); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value after read lock"); \ } else { \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before write lock"); \ QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWLockLatch(_lock); \ QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value after write lock"); \ } \ } while (0) #define QW_UNLOCK(type, _lock) \ do { \ if (QW_READ == (type)) { \ - assert(atomic_load_32((_lock)) > 0); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value before read unlock"); \ QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRUnLockLatch(_lock); \ QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after read unlock"); \ } else { \ - assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value before write unlock"); \ QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWUnLockLatch(_lock); \ QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after write unlock"); \ } \ } while (0) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index dcb7c02580654ae629722f556e252d199eeec4cc..2f4e8001df1aeca484b6898f1ae6a94d82af44c1 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -147,7 +147,6 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { size_t numOfResBlock = taosArrayGetSize(pResList); for (int32_t j = 0; j < numOfResBlock; ++j) { SSDataBlock *pRes = taosArrayGetP(pResList, j); - ASSERT(pRes->info.rows > 0); SInputData inputData = {.pData = pRes}; code = dsPutDataBlock(sinkHandle, &inputData, &qcontinue); diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index e8216fcd7c7d6f50981218ae8c27531390903382..2a5eeecd36f03f26519768c822343b8cd7323b22 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -478,34 +478,34 @@ extern SSchedulerMgmt schMgmt; #define SCH_LOCK(type, _lock) \ do { \ if (SCH_READ == (type)) { \ - assert(atomic_load_32(_lock) >= 0); \ + ASSERTS(atomic_load_32(_lock) >= 0, "invalid lock value before read lock"); \ SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRLockLatch(_lock); \ SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32(_lock) > 0); \ + ASSERTS(atomic_load_32(_lock) > 0, "invalid lock value after read lock"); \ } else { \ - assert(atomic_load_32(_lock) >= 0); \ + ASSERTS(atomic_load_32(_lock) >= 0, "invalid lock value before write lock"); \ SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWLockLatch(_lock); \ SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32(_lock) == TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32(_lock) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value after write lock"); \ } \ } while (0) #define SCH_UNLOCK(type, _lock) \ do { \ if (SCH_READ == (type)) { \ - assert(atomic_load_32((_lock)) > 0); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value before read unlock"); \ SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRUnLockLatch(_lock); \ SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after read unlock"); \ } else { \ - assert(atomic_load_32((_lock)) & TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32((_lock)) & TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value before write unlock"); \ SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWUnLockLatch(_lock); \ SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after write unlock"); \ } \ } while (0) diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index b6de9383d7a6e59ac7453227ead12fb4ba095edc..5ceb8228b03bdd0b4811b7834b24fafe8b5e6a30 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -261,46 +261,51 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa if (msg) { SDecoder coder = {0}; - SSubmitRsp *rsp = taosMemoryMalloc(sizeof(*rsp)); + SSubmitRsp2 *rsp = taosMemoryMalloc(sizeof(*rsp)); tDecoderInit(&coder, msg, msgSize); - code = tDecodeSSubmitRsp(&coder, rsp); + code = tDecodeSSubmitRsp2(&coder, rsp); + tDecoderClear(&coder); if (code) { - SCH_TASK_ELOG("decode submitRsp failed, code:%d", code); - tFreeSSubmitRsp(rsp); + SCH_TASK_ELOG("tDecodeSSubmitRsp2 failed, code:%d", code); + tDestroySSubmitRsp2(rsp, TSDB_MSG_FLG_DECODE); + taosMemoryFree(rsp); SCH_ERR_JRET(code); } - if (rsp->nBlocks > 0) { - for (int32_t i = 0; i < rsp->nBlocks; ++i) { - SSubmitBlkRsp *blk = rsp->pBlocks + i; - if (TSDB_CODE_SUCCESS != blk->code) { - code = blk->code; - tFreeSSubmitRsp(rsp); - SCH_ERR_JRET(code); + atomic_add_fetch_64(&pJob->resNumOfRows, rsp->affectedRows); + + int32_t createTbRspNum = taosArrayGetSize(rsp->aCreateTbRsp); + SCH_TASK_DLOG("submit succeed, affectedRows:%d, createTbRspNum:%d", rsp->affectedRows, createTbRspNum); + + if (rsp->aCreateTbRsp && taosArrayGetSize(rsp->aCreateTbRsp) > 0) { + SCH_LOCK(SCH_WRITE, &pJob->resLock); + if (pJob->execRes.res) { + SSubmitRsp2 *sum = pJob->execRes.res; + sum->affectedRows += rsp->affectedRows; + if (sum->aCreateTbRsp) { + taosArrayAddAll(sum->aCreateTbRsp, rsp->aCreateTbRsp); + taosArrayDestroy(rsp->aCreateTbRsp); + } else { + TSWAP(sum->aCreateTbRsp, rsp->aCreateTbRsp); } + taosMemoryFree(rsp); + } else { + pJob->execRes.res = rsp; + pJob->execRes.msgType = TDMT_VND_SUBMIT; } - } - - atomic_add_fetch_64(&pJob->resNumOfRows, rsp->affectedRows); - SCH_TASK_DLOG("submit succeed, affectedRows:%d, blocks:%d", rsp->affectedRows, rsp->nBlocks); - - SCH_LOCK(SCH_WRITE, &pJob->resLock); - if (pJob->execRes.res) { - SSubmitRsp *sum = pJob->execRes.res; - sum->affectedRows += rsp->affectedRows; - sum->nBlocks += rsp->nBlocks; - if (rsp->nBlocks > 0 && rsp->pBlocks) { - sum->pBlocks = taosMemoryRealloc(sum->pBlocks, sum->nBlocks * sizeof(*sum->pBlocks)); - memcpy(sum->pBlocks + sum->nBlocks - rsp->nBlocks, rsp->pBlocks, rsp->nBlocks * sizeof(*sum->pBlocks)); + pJob->execRes.numOfBytes += pTask->msgLen; + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + } else { + SCH_LOCK(SCH_WRITE, &pJob->resLock); + pJob->execRes.numOfBytes += pTask->msgLen; + if (NULL == pJob->execRes.res) { + TSWAP(pJob->execRes.res, rsp); + pJob->execRes.msgType = TDMT_VND_SUBMIT; } - taosMemoryFree(rsp->pBlocks); + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + tDestroySSubmitRsp2(rsp, TSDB_MSG_FLG_DECODE); taosMemoryFree(rsp); - } else { - pJob->execRes.res = rsp; - pJob->execRes.msgType = TDMT_VND_SUBMIT; } - pJob->execRes.numOfBytes += pTask->msgLen; - SCH_UNLOCK(SCH_WRITE, &pJob->resLock); } taosMemoryFreeClear(msg); diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 6cc684dddf88c6d635f3299e4bd6f81ed811fb9e..3f2feb9d286a1cad87c45fe24bc0b5132b873a7f 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -66,12 +66,13 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock return 0; } -SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq) { - SStreamDataSubmit* pDataSubmit = (SStreamDataSubmit*)taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM, 0); +SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit) { + SStreamDataSubmit2* pDataSubmit = (SStreamDataSubmit2*)taosAllocateQitem(sizeof(SStreamDataSubmit2), DEF_QITEM, 0); + if (pDataSubmit == NULL) return NULL; pDataSubmit->dataRef = (int32_t*)taosMemoryMalloc(sizeof(int32_t)); if (pDataSubmit->dataRef == NULL) goto FAIL; - pDataSubmit->data = pReq; + pDataSubmit->submit = submit; *pDataSubmit->dataRef = 1; pDataSubmit->type = STREAM_INPUT__DATA_SUBMIT; return pDataSubmit; @@ -80,47 +81,49 @@ FAIL: return NULL; } -SStreamMergedSubmit* streamMergedSubmitNew() { - SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)taosAllocateQitem(sizeof(SStreamMergedSubmit), DEF_QITEM, 0); +SStreamMergedSubmit2* streamMergedSubmitNew() { + SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)taosAllocateQitem(sizeof(SStreamMergedSubmit2), DEF_QITEM, 0); + if (pMerged == NULL) return NULL; - pMerged->reqs = taosArrayInit(0, sizeof(void*)); + pMerged->submits = taosArrayInit(0, sizeof(SPackedData)); pMerged->dataRefs = taosArrayInit(0, sizeof(void*)); - if (pMerged->dataRefs == NULL || pMerged->reqs == NULL) goto FAIL; + if (pMerged->dataRefs == NULL || pMerged->submits == NULL) goto FAIL; pMerged->type = STREAM_INPUT__MERGED_SUBMIT; return pMerged; FAIL: - if (pMerged->reqs) taosArrayDestroy(pMerged->reqs); + if (pMerged->submits) taosArrayDestroy(pMerged->submits); if (pMerged->dataRefs) taosArrayDestroy(pMerged->dataRefs); taosFreeQitem(pMerged); return NULL; } -int32_t streamMergeSubmit(SStreamMergedSubmit* pMerged, SStreamDataSubmit* pSubmit) { +int32_t streamMergeSubmit(SStreamMergedSubmit2* pMerged, SStreamDataSubmit2* pSubmit) { taosArrayPush(pMerged->dataRefs, &pSubmit->dataRef); - taosArrayPush(pMerged->reqs, &pSubmit->data); + taosArrayPush(pMerged->submits, &pSubmit->submit); pMerged->ver = pSubmit->ver; return 0; } -static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit* pDataSubmit) { +static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit2* pDataSubmit) { atomic_add_fetch_32(pDataSubmit->dataRef, 1); } -SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit) { - SStreamDataSubmit* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM, 0); +SStreamDataSubmit2* streamSubmitRefClone(SStreamDataSubmit2* pSubmit) { + SStreamDataSubmit2* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit2), DEF_QITEM, 0); + if (pSubmitClone == NULL) { return NULL; } streamDataSubmitRefInc(pSubmit); - memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit)); + memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit2)); return pSubmitClone; } -void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit) { +void streamDataSubmitRefDec(SStreamDataSubmit2* pDataSubmit) { int32_t ref = atomic_sub_fetch_32(pDataSubmit->dataRef, 1); ASSERT(ref >= 0); if (ref == 0) { - taosMemoryFree(pDataSubmit->data); + taosMemoryFree(pDataSubmit->submit.msgStr); taosMemoryFree(pDataSubmit->dataRef); } } @@ -135,16 +138,16 @@ SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* taosFreeQitem(elem); return dst; } else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && elem->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)dst; - SStreamDataSubmit* pBlockSrc = (SStreamDataSubmit*)elem; + SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)dst; + SStreamDataSubmit2* pBlockSrc = (SStreamDataSubmit2*)elem; streamMergeSubmit(pMerged, pBlockSrc); taosFreeQitem(elem); return dst; } else if (dst->type == STREAM_INPUT__DATA_SUBMIT && elem->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamMergedSubmit* pMerged = streamMergedSubmitNew(); + SStreamMergedSubmit2* pMerged = streamMergedSubmitNew(); ASSERT(pMerged); - streamMergeSubmit(pMerged, (SStreamDataSubmit*)dst); - streamMergeSubmit(pMerged, (SStreamDataSubmit*)elem); + streamMergeSubmit(pMerged, (SStreamDataSubmit2*)dst); + streamMergeSubmit(pMerged, (SStreamDataSubmit2*)elem); taosFreeQitem(dst); taosFreeQitem(elem); return (SStreamQueueItem*)pMerged; @@ -162,22 +165,22 @@ void streamFreeQitem(SStreamQueueItem* data) { taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)blockDataFreeRes); taosFreeQitem(data); } else if (type == STREAM_INPUT__DATA_SUBMIT) { - streamDataSubmitRefDec((SStreamDataSubmit*)data); + streamDataSubmitRefDec((SStreamDataSubmit2*)data); taosFreeQitem(data); } else if (type == STREAM_INPUT__MERGED_SUBMIT) { - SStreamMergedSubmit* pMerge = (SStreamMergedSubmit*)data; - int32_t sz = taosArrayGetSize(pMerge->reqs); + SStreamMergedSubmit2* pMerge = (SStreamMergedSubmit2*)data; + int32_t sz = taosArrayGetSize(pMerge->submits); for (int32_t i = 0; i < sz; i++) { int32_t* pRef = taosArrayGetP(pMerge->dataRefs, i); int32_t ref = atomic_sub_fetch_32(pRef, 1); ASSERT(ref >= 0); if (ref == 0) { - void* dataStr = taosArrayGetP(pMerge->reqs, i); - taosMemoryFree(dataStr); + SPackedData* pSubmit = (SPackedData*)taosArrayGet(pMerge->submits, i); + taosMemoryFree(pSubmit->msgStr); taosMemoryFree(pRef); } } - taosArrayDestroy(pMerge->reqs); + taosArrayDestroy(pMerge->submits); taosArrayDestroy(pMerge->dataRefs); taosFreeQitem(pMerge); } else if (type == STREAM_INPUT__REF_DATA_BLOCK) { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 786ee9d079a229eb78e1c04ae3eba20bf5393351..8f63cbbd99dcc6f75dcdc4554ba3d7bf959f481c 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -26,17 +26,18 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); - const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)data; - qDebug("task %d %p set submit input %p %p %d 1", pTask->taskId, pTask, pSubmit, pSubmit->data, *pSubmit->dataRef); - qSetMultiStreamInput(exec, pSubmit->data, 1, STREAM_INPUT__DATA_SUBMIT); + const SStreamDataSubmit2* pSubmit = (const SStreamDataSubmit2*)data; + qDebug("task %d %p set submit input %p %p %d %" PRId64, pTask->taskId, pTask, pSubmit, pSubmit->submit.msgStr, + pSubmit->submit.msgLen, pSubmit->submit.ver); + qSetMultiStreamInput(exec, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { const SStreamDataBlock* pBlock = (const SStreamDataBlock*)data; SArray* blocks = pBlock->blocks; qDebug("task %d %p set ssdata input", pTask->taskId, pTask); qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { - const SStreamMergedSubmit* pMerged = (const SStreamMergedSubmit*)data; - SArray* blocks = pMerged->reqs; + const SStreamMergedSubmit2* pMerged = (const SStreamMergedSubmit2*)data; + SArray* blocks = pMerged->submits; qDebug("task %d %p set submit input (merged), batch num: %d", pTask->taskId, pTask, (int32_t)blocks->size); qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__MERGED_SUBMIT); } else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) { @@ -133,7 +134,11 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); - if (++batchCnt >= batchSz) break; + batchCnt++; + + qDebug("task %d scan exec block num %d, block limit %d", pTask->taskId, batchCnt, batchSz); + + if (batchCnt >= batchSz) break; } if (taosArrayGetSize(pRes) == 0) { taosArrayDestroy(pRes); @@ -151,6 +156,7 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { streamTaskOutput(pTask, qRes); if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + qDebug("task %d scan exec dispatch block num %d", pTask->taskId, batchCnt); streamDispatch(pTask); } if (finished) break; @@ -257,11 +263,11 @@ int32_t streamExecForAll(SStreamTask* pTask) { qRes->blocks = pRes; if (((SStreamQueueItem*)input)->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)input; + SStreamDataSubmit2* pSubmit = (SStreamDataSubmit2*)input; qRes->childId = pTask->selfChildId; qRes->sourceVer = pSubmit->ver; } else if (((SStreamQueueItem*)input)->type == STREAM_INPUT__MERGED_SUBMIT) { - SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)input; + SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)input; qRes->childId = pTask->selfChildId; qRes->sourceVer = pMerged->ver; } diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 6670bf463e5d69451bf5337a88c902edad6c014c..e26c8a5c5b128d6087c7c687a17cd3c5a2fa0b3e 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -159,6 +159,11 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int goto _err; } + if (tdbTbOpen("partag.state.db", sizeof(int64_t), -1, NULL, pState->pTdbState->db, &pState->pTdbState->pParTagDb, 0) < + 0) { + goto _err; + } + if (streamStateBegin(pState) < 0) { goto _err; } @@ -173,6 +178,7 @@ _err: tdbTbClose(pState->pTdbState->pFillStateDb); tdbTbClose(pState->pTdbState->pSessionStateDb); tdbTbClose(pState->pTdbState->pParNameDb); + tdbTbClose(pState->pTdbState->pParTagDb); tdbClose(pState->pTdbState->db); streamStateDestroy(pState); return NULL; @@ -186,6 +192,7 @@ void streamStateClose(SStreamState* pState) { tdbTbClose(pState->pTdbState->pFillStateDb); tdbTbClose(pState->pTdbState->pSessionStateDb); tdbTbClose(pState->pTdbState->pParNameDb); + tdbTbClose(pState->pTdbState->pParTagDb); tdbClose(pState->pTdbState->db); streamStateDestroy(pState); @@ -820,10 +827,17 @@ _end: return res; } +int32_t streamStatePutParTag(SStreamState* pState, int64_t groupId, const void* tag, int32_t tagLen) { + return tdbTbUpsert(pState->pTdbState->pParTagDb, &groupId, sizeof(int64_t), tag, tagLen, pState->pTdbState->txn); +} + +int32_t streamStateGetParTag(SStreamState* pState, int64_t groupId, void** tagVal, int32_t* tagLen) { + return tdbTbGet(pState->pTdbState->pParTagDb, &groupId, sizeof(int64_t), tagVal, tagLen); +} + int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]) { - tdbTbUpsert(pState->pTdbState->pParNameDb, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN, - pState->pTdbState->txn); - return 0; + return tdbTbUpsert(pState->pTdbState->pParNameDb, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN, + pState->pTdbState->txn); } int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal) { diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 1ce4a35dff6441a9ec289d1cb94eb24743122762..be12c72d004711e6c126792b5a7cfe4053ad1f18 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -85,9 +85,7 @@ static int64_t adjustWatermark(int64_t adjInterval, int64_t originInt, int64_t w watermark = TMAX(originInt / adjInterval, 1) * adjInterval; } else if (watermark > MAX_NUM_SCALABLE_BF * adjInterval) { watermark = MAX_NUM_SCALABLE_BF * adjInterval; - }/* else if (watermark < MIN_NUM_SCALABLE_BF * adjInterval) { - watermark = MIN_NUM_SCALABLE_BF * adjInterval; - }*/ // Todo(liuyao) save window info to tdb + } return watermark; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 65937fcfada1be57faac55f7cbe395021a8199b0..02f9795cad8a308a602fb314b8d87535e8f22cd3 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1703,8 +1703,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde _END: // log end config change - sNInfo(pSyncNode, "end do config change, from %d to %d", pSyncNode->vgId, oldConfig.replicaNum, - pNewConfig->replicaNum); + sNInfo(pSyncNode, "end do config change, from %d to %d", oldConfig.replicaNum, pNewConfig->replicaNum); } // raft state change -------------- diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 4f0682a617c8d91d1a48c6afa1a7630a7d332811..4cd6a39bfe00f56631abea8dfd3c935dff1e580b 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -74,7 +74,10 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg SBTree *pBt; int ret; - ASSERT(keyLen != 0); + if (keyLen == 0) { + tdbError("tdb/btree-open: key len cannot be zero."); + return -1; + } *ppBt = NULL; @@ -152,7 +155,11 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg tdbPostCommit(pPager->pEnv, txn); } - ASSERT(pgno != 0); + if (pgno == 0) { + tdbError("tdb/btree-open: pgno cannot be zero."); + tdbOsFree(pBt); + return -1; + } pBt->root = pgno; /* // TODO: pBt->root @@ -192,7 +199,7 @@ int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, in ret = tdbBtcMoveTo(&btc, pKey, kLen, &c); if (ret < 0) { tdbBtcClose(&btc); - ASSERT(0); + tdbError("tdb/btree-insert: btc move to failed with ret: %d.", ret); return -1; } @@ -202,17 +209,17 @@ int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, in if (c > 0) { btc.idx++; } else if (c == 0) { - // dup key not allowed - tdbError("unable to insert dup key. pKey: %p, kLen: %d, btc: %p, pTxn: %p", pKey, kLen, &btc, pTxn); - // ASSERT(0); + // dup key not allowed with insert + tdbBtcClose(&btc); + tdbError("tdb/btree-insert: dup key. pKey: %p, kLen: %d, btc: %p, pTxn: %p", pKey, kLen, &btc, pTxn); return -1; } } ret = tdbBtcUpsert(&btc, pKey, kLen, pVal, vLen, 1); if (ret < 0) { - ASSERT(0); tdbBtcClose(&btc); + tdbError("tdb/btree-insert: btc upsert failed with ret: %d.", ret); return -1; } @@ -233,7 +240,7 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) { ret = tdbBtcMoveTo(&btc, pKey, kLen, &c); if (ret < 0) { tdbBtcClose(&btc); - ASSERT(0); + tdbError("tdb/btree-delete: btc move to failed with ret: %d.", ret); return -1; } @@ -264,7 +271,7 @@ int tdbBtreeUpsert(SBTree *pBt, const void *pKey, int nKey, const void *pData, i // move the cursor ret = tdbBtcMoveTo(&btc, pKey, nKey, &c); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btree-upsert: btc move to failed with ret: %d.", ret); tdbBtcClose(&btc); return -1; } @@ -280,8 +287,8 @@ int tdbBtreeUpsert(SBTree *pBt, const void *pKey, int nKey, const void *pData, i ret = tdbBtcUpsert(&btc, pKey, nKey, pData, nData, c); if (ret < 0) { - ASSERT(0); tdbBtcClose(&btc); + tdbError("tdb/btree-upsert: btc upsert failed with ret: %d.", ret); return -1; } @@ -309,7 +316,8 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL ret = tdbBtcMoveTo(&btc, pKey, kLen, &cret); if (ret < 0) { tdbBtcClose(&btc); - ASSERT(0); + tdbError("tdb/btree-pget: btc move to failed with ret: %d.", ret); + return -1; } if (btc.idx < 0 || cret) { @@ -325,7 +333,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL pTKey = tdbRealloc(*ppKey, cd.kLen); if (pTKey == NULL) { tdbBtcClose(&btc); - ASSERT(0); + tdbError("tdb/btree-pget: realloc pTKey failed."); return -1; } *ppKey = pTKey; @@ -337,7 +345,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL pTVal = tdbRealloc(*ppVal, cd.vLen); if (pTVal == NULL) { tdbBtcClose(&btc); - ASSERT(0); + tdbError("tdb/btree-pget: realloc pTVal failed."); return -1; } *ppVal = pTVal; @@ -350,7 +358,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL } if (TDB_CELLDECODER_FREE_VAL(&cd)) { - tdbDebug("tdb btc/pget/2 decoder: %p pVal free: %p", &cd, cd.pVal); + tdbTrace("tdb btc/pget/2 decoder: %p pVal free: %p", &cd, cd.pVal); tdbFree(cd.pVal); } @@ -366,7 +374,9 @@ static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2 int mlen; int cret; - ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); + if (ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL)) { + // -1 is less than + } mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; cret = memcmp(pKey1, pKey2, mlen); @@ -381,36 +391,7 @@ static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2 } return cret; } -/* -static int tdbBtreeOpenImpl(SBTree *pBt) { - // Try to get the root page of the an existing btree - SPgno pgno; - SPage *pPage; - int ret; - - { - // 1. TODO: Search the main DB to check if the DB exists - ret = tdbPagerOpenDB(pBt->pPager, &pgno, true, pBt); - ASSERT(ret == 0); - } - if (pgno != 0) { - pBt->root = pgno; - return 0; - } - - // Try to create a new database - ret = tdbPagerAllocPage(pBt->pPager, &pgno); - if (ret < 0) { - ASSERT(0); - return -1; - } - - ASSERT(pgno != 0); - pBt->root = pgno; - return 0; -} -*/ int tdbBtreeInitPage(SPage *pPage, void *arg, int init) { SBTree *pBt; u8 flags; @@ -546,11 +527,15 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx nOlds = 3; } for (int i = 0; i < nOlds; i++) { - ASSERT(sIdx + i <= nCells); + if (ASSERT(sIdx + i <= nCells)) { + return -1; + } SPgno pgno; if (sIdx + i == nCells) { - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pParent)); + if (ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pParent))) { + return -1; + } pgno = ((SIntHdr *)(pParent->pData))->pgno; } else { pCell = tdbPageGetCell(pParent, sIdx + i); @@ -560,7 +545,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ret = tdbPagerFetchPage(pBt->pPager, &pgno, pOlds + i, tdbBtreeInitPage, &((SBtreeInitPageArg){.pBt = pBt, .flags = 0}), pTxn); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btree-balance: fetch page failed with ret: %d.", ret); return -1; } @@ -679,7 +664,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx szRCell = tdbBtreeCellSize(pPage, pCell, 0, NULL, NULL); } - ASSERT(infoNews[iNew - 1].cnt > 0); + if (ASSERT(infoNews[iNew - 1].cnt > 0)) { + return -1; + } if (infoNews[iNew].size + szRCell >= infoNews[iNew - 1].size - szRCell) { break; @@ -722,7 +709,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx iarg.flags = flags; ret = tdbPagerFetchPage(pBt->pPager, &pgno, pNews + iNew, tdbBtreeInitPage, &iarg, pTxn); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btree-balance: fetch page failed with ret: %d.", ret); + return -1; } ret = tdbPagerWrite(pBt->pPager, pNews[iNew]); @@ -766,8 +754,12 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx pCell = tdbPageGetCell(pPage, oIdx); szCell = tdbBtreeCellSize(pPage, pCell, 0, NULL, NULL); - ASSERT(nNewCells <= infoNews[iNew].cnt); - ASSERT(iNew < nNews); + if (ASSERT(nNewCells <= infoNews[iNew].cnt)) { + return -1; + } + if (ASSERT(iNew < nNews)) { + return -1; + } if (nNewCells < infoNews[iNew].cnt) { tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); @@ -806,14 +798,20 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } } } else { - ASSERT(childNotLeaf); - ASSERT(iNew < nNews - 1); + if (ASSERT(childNotLeaf)) { + return -1; + } + if (ASSERT(iNew < nNews - 1)) { + return -1; + } // set current new page right-most child ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; // insert to parent as divider cell - ASSERT(iNew < nNews - 1); + if (ASSERT(iNew < nNews - 1)) { + return -1; + } ((SPgno *)pCell)[0] = TDB_PAGE_PGNO(pNews[iNew]); tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); @@ -828,7 +826,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } if (childNotLeaf) { - ASSERT(TDB_PAGE_TOTAL_CELLS(pNews[nNews - 1]) == infoNews[nNews - 1].cnt); + if (ASSERT(TDB_PAGE_TOTAL_CELLS(pNews[nNews - 1]) == infoNews[nNews - 1].cnt)) { + return -1; + } ((SIntHdr *)(pNews[nNews - 1]->pData))->pgno = rPgno; SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; @@ -1018,7 +1018,9 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const nLeft -= kLen; // pack partial val to local if any space left if (nLocal > nHeader + kLen + sizeof(SPgno)) { - ASSERT(pVal != NULL && vLen != 0); + if (ASSERT(pVal != NULL && vLen != 0)) { + return -1; + } memcpy(pCell + nHeader + kLen, pVal, nLocal - nHeader - kLen - sizeof(SPgno)); nLeft -= nLocal - nHeader - kLen - sizeof(SPgno); } @@ -1183,9 +1185,15 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo int nPayload; int ret; - ASSERT(pPage->kLen == TDB_VARIANT_LEN || pPage->kLen == kLen); - ASSERT(pPage->vLen == TDB_VARIANT_LEN || pPage->vLen == vLen); - ASSERT(pKey != NULL && kLen > 0); + if (ASSERT(pPage->kLen == TDB_VARIANT_LEN || pPage->kLen == kLen)) { + return -1; + } + if (ASSERT(pPage->vLen == TDB_VARIANT_LEN || pPage->vLen == vLen)) { + return -1; + } + if (ASSERT(pKey != NULL && kLen > 0)) { + return -1; + } nPayload = 0; nHeader = 0; @@ -1194,7 +1202,10 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo // 1. Encode Header part /* Encode SPgno if interior page */ if (!leaf) { - ASSERT(pPage->vLen == sizeof(SPgno)); + if (pPage->vLen != sizeof(SPgno)) { + tdbError("tdb/btree-encode-cell: invalid cell."); + return -1; + } ((SPgno *)(pCell + nHeader))[0] = ((SPgno *)pVal)[0]; nHeader = nHeader + sizeof(SPgno); @@ -1219,8 +1230,8 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo ret = tdbBtreeEncodePayload(pPage, pCell, nHeader, pKey, kLen, pVal, vLen, &nPayload, pTxn, pBt); if (ret < 0) { // TODO - ASSERT(0); - return 0; + tdbError("tdb/btree-encode-cell: encode payload failed with ret: %d.", ret); + return -1; } *szCell = nHeader + nPayload; @@ -1237,7 +1248,10 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, int vLen = pDecoder->vLen; if (pDecoder->pVal) { - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pPage)); + if (TDB_BTREE_PAGE_IS_LEAF(pPage)) { + tdbError("tdb/btree-decode-payload: leaf page with non-null pVal."); + return -1; + } nPayload = pDecoder->kLen; } else { nPayload = pDecoder->kLen + pDecoder->vLen; @@ -1440,7 +1454,10 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD // 1. Decode header part if (!leaf) { - ASSERT(pPage->vLen == sizeof(SPgno)); + if (pPage->vLen != sizeof(SPgno)) { + tdbError("tdb/btree-decode-cell: invalid cell."); + return -1; + } pDecoder->pgno = ((SPgno *)(pCell + nHeader))[0]; pDecoder->pVal = (u8 *)(&(pDecoder->pgno)); @@ -1454,7 +1471,10 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD } if (pPage->vLen == TDB_VARIANT_LEN) { - ASSERT(leaf); + if (!leaf) { + tdbError("tdb/btree-decode-cell: not a leaf page."); + return -1; + } nHeader += tdbGetVarInt(pCell + nHeader, &(pDecoder->vLen)); } else { pDecoder->vLen = pPage->vLen; @@ -1486,7 +1506,10 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN * } if (pPage->vLen == TDB_VARIANT_LEN) { - ASSERT(leaf); + if (!leaf) { + tdbError("tdb/btree-cell-size: not a leaf page."); + return -1; + } nHeader += tdbGetVarInt(pCell + nHeader, &vLen); } else if (leaf) { vLen = pPage->vLen; @@ -1582,29 +1605,42 @@ int tdbBtcMoveToFirst(SBTC *pBtc) { ret = tdbPagerFetchPage(pPager, &pBt->root, &(pBtc->pPage), tdbBtreeInitPage, &((SBtreeInitPageArg){.pBt = pBt, .flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF}), pBtc->pTxn); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-tofirst: fetch page failed with ret: %d.", ret); return -1; } - ASSERT(TDB_BTREE_PAGE_IS_ROOT(pBtc->pPage)); + if (!TDB_BTREE_PAGE_IS_ROOT(pBtc->pPage)) { + tdbError("tdb/btc-move-tofirst: not a root page"); + return -1; + } pBtc->iPage = 0; if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) > 0) { pBtc->idx = 0; } else { // no any data, point to an invalid position - ASSERT(TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + if (!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + tdbError("tdb/btc-move-to-first: not a leaf page."); + return -1; + } + pBtc->idx = -1; return 0; } } else { - ASSERT(0); + // TODO + tdbError("tdb/btc-move-to-first: move from a dirty cursor."); + return -1; #if 0 // move from a position int iPage = 0; for (; iPage < pBtc->iPage; iPage++) { - ASSERT(pBtc->idxStack[iPage] >= 0); + if (pBtc->idxStack[iPage] < 0) { + tdbError("tdb/btc-move-to-first: invalid idx: %d.", pBtc->idxStack[iPage]); + return -1; + } + if (pBtc->idxStack[iPage]) break; } @@ -1626,7 +1662,7 @@ int tdbBtcMoveToFirst(SBTC *pBtc) { ret = tdbBtcMoveDownward(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-tofirst: btc move downward failed with ret: %d.", ret); return -1; } @@ -1651,7 +1687,7 @@ int tdbBtcMoveToLast(SBTC *pBtc) { ret = tdbPagerFetchPage(pPager, &pBt->root, &(pBtc->pPage), tdbBtreeInitPage, &((SBtreeInitPageArg){.pBt = pBt, .flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF}), pBtc->pTxn); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-tolast: fetch page failed with ret: %d.", ret); return -1; } @@ -1661,18 +1697,28 @@ int tdbBtcMoveToLast(SBTC *pBtc) { pBtc->idx = TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage) ? nCells - 1 : nCells; } else { // no data at all, point to an invalid position - ASSERT(TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + if (!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + tdbError("tdb/btc-move-to-last: not a leaf page."); + return -1; + } + pBtc->idx = -1; return 0; } } else { - ASSERT(0); + // TODO + tdbError("tdb/btc-move-to-last: move from a dirty cursor."); + return -1; #if 0 int iPage = 0; // downward search for (; iPage < pBtc->iPage; iPage++) { - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pBtc->pgStack[iPage])); + if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pgStack[iPage])) { + tdbError("tdb/btc-move-to-last: leaf page in cursor stack."); + return -1; + } + nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pgStack[iPage]); if (pBtc->idxStack[iPage] != nCells) break; } @@ -1699,7 +1745,7 @@ int tdbBtcMoveToLast(SBTC *pBtc) { ret = tdbBtcMoveDownward(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-tolast: btc move downward failed with ret: %d.", ret); return -1; } @@ -1761,7 +1807,7 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { ret = tdbBtcMoveToNext(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btree-next: btc move to next failed with ret: %d.", ret); return -1; } @@ -1807,7 +1853,7 @@ int tdbBtreePrev(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { ret = tdbBtcMoveToPrev(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btree-prev: btc move to prev failed with ret: %d.", ret); return -1; } @@ -1819,7 +1865,10 @@ int tdbBtcMoveToNext(SBTC *pBtc) { int ret; SCell *pCell; - ASSERT(TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + if (!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + tdbError("tdb/btc-move-to-next: not a leaf page."); + return -1; + } if (pBtc->idx < 0) return -1; @@ -1838,7 +1887,10 @@ int tdbBtcMoveToNext(SBTC *pBtc) { tdbBtcMoveUpward(pBtc); pBtc->idx++; - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + tdbError("tdb/btree-decode-cell: should not be a leaf page here."); + return -1; + } if (pBtc->idx <= TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { break; } @@ -1850,7 +1902,7 @@ int tdbBtcMoveToNext(SBTC *pBtc) { ret = tdbBtcMoveDownward(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-tonext: btc move downward failed with ret: %d.", ret); return -1; } @@ -1902,8 +1954,15 @@ static int tdbBtcMoveDownward(SBTC *pBtc) { SPgno pgno; SCell *pCell; - ASSERT(pBtc->idx >= 0); - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + if (pBtc->idx < 0) { + tdbError("tdb/btc-move-downward: invalid idx: %d.", pBtc->idx); + return -1; + } + + if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + tdbError("tdb/btc-move-downward: should not be a leaf page here."); + return -1; + } if (pBtc->idx < TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); @@ -1912,7 +1971,10 @@ static int tdbBtcMoveDownward(SBTC *pBtc) { pgno = ((SIntHdr *)pBtc->pPage->pData)->pgno; } - ASSERT(pgno); + if (!pgno) { + tdbError("tdb/btc-move-downward: invalid pgno."); + return -1; + } pBtc->pgStack[pBtc->iPage] = pBtc->pPage; pBtc->idxStack[pBtc->iPage] = pBtc->idx; @@ -1923,7 +1985,7 @@ static int tdbBtcMoveDownward(SBTC *pBtc) { ret = tdbPagerFetchPage(pBtc->pBt->pPager, &pgno, &pBtc->pPage, tdbBtreeInitPage, &((SBtreeInitPageArg){.pBt = pBtc->pBt, .flags = 0}), pBtc->pTxn); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-downward: fetch page failed with ret: %d.", ret); return -1; } @@ -1978,7 +2040,10 @@ int tdbBtcDelete(SBTC *pBtc) { int nKey; int ret; - ASSERT(idx >= 0 && idx < nCells); + if (idx < 0 || idx >= nCells) { + tdbError("tdb/btc-delete: idx: %d out of range[%d, %d).", idx, 0, nCells); + return -1; + } // drop the cell on the leaf ret = tdbPagerWrite(pPager, pBtc->pPage); @@ -2016,7 +2081,7 @@ int tdbBtcDelete(SBTC *pBtc) { ret = tdbPageUpdateCell(pPage, idx, pCell, szCell, pBtc->pTxn, pBtc->pBt); if (ret < 0) { tdbOsFree(pCell); - ASSERT(0); + tdbError("tdb/btc-delete: page update cell failed with ret: %d.", ret); return -1; } tdbOsFree(pCell); @@ -2027,11 +2092,14 @@ int tdbBtcDelete(SBTC *pBtc) { } } else { // delete the leaf page and do balance - ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0); + if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) != 0) { + tdbError("tdb/btc-delete: page to be deleted should be empty."); + return -1; + } ret = tdbBtreeBalance(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-delete: btree balance failed with ret: %d.", ret); return -1; } } @@ -2048,13 +2116,16 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int void *pBuf; int ret; - ASSERT(pBtc->idx >= 0); + if (pBtc->idx < 0) { + tdbError("tdb/btc-upsert: invalid idx: %d.", pBtc->idx); + return -1; + } // alloc space szBuf = kLen + nData + 14; pBuf = tdbRealloc(pBtc->pBt->pBuf, pBtc->pBt->pageSize > szBuf ? szBuf : pBtc->pBt->pageSize); if (pBuf == NULL) { - ASSERT(0); + tdbError("tdb/btc-upsert: realloc pBuf failed."); return -1; } pBtc->pBt->pBuf = pBuf; @@ -2063,7 +2134,7 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int // encode cell ret = tdbBtreeEncodeCell(pBtc->pPage, pKey, kLen, pData, nData, pCell, &szCell, pBtc->pTxn, pBtc->pBt); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-upsert: btree encode cell failed with ret: %d.", ret); return -1; } @@ -2076,16 +2147,22 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int // insert or update if (insert) { - ASSERT(pBtc->idx <= nCells); + if (pBtc->idx > nCells) { + tdbError("tdb/btc-upsert: invalid idx: %d, nCells: %d.", pBtc->idx, nCells); + return -1; + } ret = tdbPageInsertCell(pBtc->pPage, pBtc->idx, pCell, szCell, 0); } else { - ASSERT(pBtc->idx < nCells); + if (pBtc->idx >= nCells) { + tdbError("tdb/btc-upsert: invalid idx: %d, nCells: %d.", pBtc->idx, nCells); + return -1; + } ret = tdbPageUpdateCell(pBtc->pPage, pBtc->idx, pCell, szCell, pBtc->pTxn, pBtc->pBt); } if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-upsert: page insert/update cell failed with ret: %d.", ret); return -1; } @@ -2093,7 +2170,7 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int if (pBtc->pPage->nOverflow > 0) { ret = tdbBtreeBalance(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-upsert: btree balance failed with ret: %d.", ret); return -1; } } @@ -2118,8 +2195,8 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { &((SBtreeInitPageArg){.pBt = pBt, .flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF}), pBtc->pTxn); if (ret < 0) { // TODO - ASSERT(0); - return 0; + tdbError("tdb/btc-move-to: fetch page failed with ret: %d.", ret); + return -1; } pBtc->iPage = 0; @@ -2127,7 +2204,9 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { // for empty tree, just return with an invalid position if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0) return 0; } else { - ASSERT(0); + // TODO + tdbError("tdb/btc-move-to: move from a dirty cursor."); + return -1; #if 0 SPage *pPage; int idx; @@ -2139,7 +2218,10 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { idx = pBtc->idxStack[iPage]; nCells = TDB_PAGE_TOTAL_CELLS(pPage); - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pPage)); + if (TDB_BTREE_PAGE_IS_LEAF(pPage)) { + tdbError("tdb/btc-move-to: leaf page in cursor stack."); + return -1; + } // check if key <= current position if (idx < nCells) { @@ -2242,7 +2324,10 @@ int tdbBtcClose(SBTC *pBtc) { if (pBtc->iPage < 0) return 0; for (;;) { - ASSERT(pBtc->pPage); + if (NULL == pBtc->pPage) { + tdbError("tdb/btc-close: null ptr pPage."); + return -1; + } tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage, pBtc->pTxn); diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index c79279c658a746e83a40eaed6fd6cb8cbe6e31d7..952c49db73a2c8986d67c8e327175ad00e778ae6 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -247,7 +247,10 @@ void tdbEnvRemovePager(TDB *pDb, SPager *pPager) { // remove from the list for (ppPager = &pDb->pgrList; *ppPager && (*ppPager != pPager); ppPager = &((*ppPager)->pNext)) { } - ASSERT(*ppPager == pPager); + if (*ppPager != pPager) { + tdbError("tdb/db: invalid pPager: %p, *ppPager: %p", pPager, *ppPager); + return; + } *ppPager = pPager->pNext; // remove from hash @@ -255,7 +258,10 @@ void tdbEnvRemovePager(TDB *pDb, SPager *pPager) { ppPager = &pDb->pgrHash[hash % pDb->nPgrHash]; for (; *ppPager && *ppPager != pPager; ppPager = &((*ppPager)->pHashNext)) { } - ASSERT(*ppPager == pPager); + if (*ppPager != pPager) { + tdbError("tdb/db: invalid pPager: %p, *ppPager: %p", pPager, *ppPager); + return; + } *ppPager = pPager->pNext; // decrease the counter diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 4896568c7f5ce45ce816f16d3eec0436cf51dfc5..262f3d27e648cfebb4f081c725f443756cb6a06d 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -236,10 +236,10 @@ void tdbPCacheInvalidatePage(SPCache *pCache, SPager *pPager, SPgno pgno) { void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) { i32 nRef; - ASSERT(pTxn); - - // nRef = tdbUnrefPage(pPage); - // ASSERT(nRef >= 0); + if (!pTxn) { + tdbError("tdb/pcache: null ptr pTxn, release failed."); + return; + } tdbPCacheLock(pCache); nRef = tdbUnrefPage(pPage); @@ -275,7 +275,10 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) SPage *pPage = NULL; SPage *pPageH = NULL; - ASSERT(pTxn); + if (!pTxn) { + tdbError("tdb/pcache: null ptr pTxn, fetch impl failed."); + return NULL; + } // 1. Search the hash table pPage = pCache->pgHash[tdbPCachePageHash(pPgid) % pCache->nHash]; @@ -315,8 +318,8 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) if (!pPage && pTxn->xMalloc != NULL) { ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg); if (ret < 0 || pPage == NULL) { - // TODO - ASSERT(0); + tdbError("tdb/pcache: ret: %" PRId32 " pPage: %p, page create failed.", ret, pPage); + // TODO: recycle other backup pages return NULL; } @@ -370,7 +373,11 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) static void tdbPCachePinPage(SPCache *pCache, SPage *pPage) { if (pPage->pLruNext != NULL) { - ASSERT(tdbGetPageRef(pPage) == 0); + int32_t nRef = tdbGetPageRef(pPage); + if (nRef != 0) { + tdbError("tdb/pcache: pin page's ref not zero: %" PRId32, nRef); + return; + } pPage->pLruPrev->pLruNext = pPage->pLruNext; pPage->pLruNext->pLruPrev = pPage->pLruPrev; @@ -383,13 +390,23 @@ static void tdbPCachePinPage(SPCache *pCache, SPage *pPage) { } static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) { - i32 nRef; - - ASSERT(pPage->isLocal); - ASSERT(!pPage->isDirty); - ASSERT(tdbGetPageRef(pPage) == 0); - - ASSERT(pPage->pLruNext == NULL); + i32 nRef = tdbGetPageRef(pPage); + if (nRef != 0) { + tdbError("tdb/pcache: unpin page's ref not zero: %" PRId32, nRef); + return; + } + if (!pPage->isLocal) { + tdbError("tdb/pcache: unpin page's not local: %" PRIu8, pPage->isLocal); + return; + } + if (pPage->isDirty) { + tdbError("tdb/pcache: unpin page's dirty: %" PRIu8, pPage->isDirty); + return; + } + if (NULL != pPage->pLruNext) { + tdbError("tdb/pcache: unpin page's pLruNext not null."); + return; + } tdbTrace("pCache:%p unpin page %p/%d, nPages:%d, pgno:%d, ", pCache, pPage, pPage->id, pCache->nPages, TDB_PAGE_PGNO(pPage)); diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index 50dc8e0a6506abf40cd85b33acdd0e50f737168b..f19c3f28c05e4ec586328320228c59966ad1e3ba 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -43,9 +43,15 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) u8 *ptr; int size; - ASSERT(xMalloc); + if (!xMalloc) { + tdbError("tdb/page-create: null xMalloc."); + return -1; + } - ASSERT(TDB_IS_PGSIZE_VLD(pageSize)); + if (!TDB_IS_PGSIZE_VLD(pageSize)) { + tdbError("tdb/page-create: invalid pageSize: %d.", pageSize); + return -1; + } *ppPage = NULL; size = pageSize + sizeof(*pPage); @@ -69,16 +75,24 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) *ppPage = pPage; - tdbTrace("page/create: %p/%d %p", pPage, pPage->id, xMalloc); + tdbTrace("tdb/page-create: %p/%d %p", pPage, pPage->id, xMalloc); return 0; } int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) { u8 *ptr; - tdbTrace("page/destroy: %p/%d %p", pPage, pPage->id, xFree); - ASSERT(!pPage->isDirty); - ASSERT(xFree); + tdbTrace("tdb/page-destroy: %p/%d %p", pPage, pPage->id, xFree); + + if (pPage->isDirty) { + tdbError("tdb/page-destroy: dirty page: %" PRIu8 ".", pPage->isDirty); + return -1; + } + + if (!xFree) { + tdbError("tdb/page-destroy: null xFree."); + return -1; + } for (int iOvfl = 0; iOvfl < pPage->nOverflow; iOvfl++) { tdbTrace("tdbPage/destroy/free ovfl cell: %p/%p", pPage->apOvfl[iOvfl], pPage); @@ -105,7 +119,10 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell pPage->nOverflow = 0; pPage->xCellSize = xCellSize; - ASSERT((u8 *)pPage->pPageFtr == pPage->pFreeEnd); + if ((u8 *)pPage->pPageFtr != pPage->pFreeEnd) { + tdbError("tdb/page-zero: invalid page, pFreeEnd: %p, pPageFtr: %p", pPage->pFreeEnd, pPage->pPageFtr); + return; + } } void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)) { @@ -121,8 +138,15 @@ void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell pPage->nOverflow = 0; pPage->xCellSize = xCellSize; - ASSERT(pPage->pFreeEnd >= pPage->pFreeStart); - ASSERT(pPage->pFreeEnd - pPage->pFreeStart <= TDB_PAGE_NFREE(pPage)); + if (pPage->pFreeEnd < pPage->pFreeStart) { + tdbError("tdb/page-init: invalid page, pFreeEnd: %p, pFreeStart: %p", pPage->pFreeEnd, pPage->pFreeStart); + return; + } + if (pPage->pFreeEnd - pPage->pFreeStart > TDB_PAGE_NFREE(pPage)) { + tdbError("tdb/page-init: invalid page, pFreeEnd: %p, pFreeStart: %p, NFREE: %d", pPage->pFreeEnd, pPage->pFreeStart, + TDB_PAGE_NFREE(pPage)); + return; + } } int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl) { @@ -132,7 +156,11 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl int lidx; // local idx SCell *pNewCell; - ASSERT(szCell <= TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)); + if (szCell > TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)) { + tdbError("tdb/page-insert-cell: invalid page, szCell: %d, max free: %lu", szCell, + TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)); + return -1; + } nFree = TDB_PAGE_NFREE(pPage); nCells = TDB_PAGE_NCELLS(pPage); @@ -176,7 +204,11 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl TDB_PAGE_CELL_OFFSET_AT_SET(pPage, lidx, pNewCell - pPage->pData); TDB_PAGE_NCELLS_SET(pPage, nCells + 1); - ASSERT(pPage->pFreeStart == pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)); + if (pPage->pFreeStart != pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)) { + tdbError("tdb/page-insert-cell: invalid page, pFreeStart: %p, pCellIdx: %p, nCells: %d", pPage->pFreeStart, + pPage->pCellIdx, nCells); + return -1; + } } for (; iOvfl < pPage->nOverflow; iOvfl++) { @@ -200,7 +232,10 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) { nCells = TDB_PAGE_NCELLS(pPage); - ASSERT(idx >= 0 && idx < nCells + pPage->nOverflow); + if (idx < 0 || idx >= nCells + pPage->nOverflow) { + tdbError("tdb/page-drop-cell: idx: %d out of range, nCells: %d, nOvfl: %d.", idx, nCells, pPage->nOverflow); + return -1; + } iOvfl = 0; for (; iOvfl < pPage->nOverflow; iOvfl++) { @@ -228,7 +263,10 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) { for (; iOvfl < pPage->nOverflow; iOvfl++) { pPage->aiOvfl[iOvfl]--; - ASSERT(pPage->aiOvfl[iOvfl] > 0); + if (pPage->aiOvfl[iOvfl] <= 0) { + tdbError("tdb/page-drop-cell: invalid ai idx: %d", pPage->aiOvfl[iOvfl]); + return -1; + } } return 0; @@ -240,12 +278,19 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { pToPage->pFreeStart = pToPage->pPageHdr + (pFromPage->pFreeStart - pFromPage->pPageHdr); pToPage->pFreeEnd = (u8 *)(pToPage->pPageFtr) - ((u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); - ASSERT(pToPage->pFreeEnd >= pToPage->pFreeStart); + if (pToPage->pFreeEnd < pToPage->pFreeStart) { + tdbError("tdb/page-copy: invalid to page, pFreeStart: %p, pFreeEnd: %p", pToPage->pFreeStart, pToPage->pFreeEnd); + return; + } memcpy(pToPage->pPageHdr, pFromPage->pPageHdr, pFromPage->pFreeStart - pFromPage->pPageHdr); memcpy(pToPage->pFreeEnd, pFromPage->pFreeEnd, (u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); - ASSERT(TDB_PAGE_CCELLS(pToPage) == pToPage->pFreeEnd - pToPage->pData); + if (TDB_PAGE_CCELLS(pToPage) != pToPage->pFreeEnd - pToPage->pData) { + tdbError("tdb/page-copy: invalid to page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pToPage), + pToPage->pFreeEnd - pToPage->pData); + return; + } delta = (pToPage->pPageHdr - pToPage->pData) - (pFromPage->pPageHdr - pFromPage->pData); if (delta != 0) { @@ -295,8 +340,16 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { *ppCell = NULL; nFree = TDB_PAGE_NFREE(pPage); - ASSERT(nFree >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)); - ASSERT(TDB_PAGE_CCELLS(pPage) == pPage->pFreeEnd - pPage->pData); + if (nFree < szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { + tdbError("tdb/page-allocate: invalid cell size, nFree: %d, szCell: %d, szOffset: %d", nFree, szCell, + TDB_PAGE_OFFSET_SIZE(pPage)); + return -1; + } + if (TDB_PAGE_CCELLS(pPage) != pPage->pFreeEnd - pPage->pData) { + tdbError("tdb/page-allocate: invalid page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pPage), + pPage->pFreeEnd - pPage->pData); + return -1; + } // 1. Try to allocate from the free space block area if (pPage->pFreeEnd - pPage->pFreeStart >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { @@ -308,7 +361,10 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { // 2. Try to allocate from the page free list cellFree = TDB_PAGE_FCELL(pPage); - ASSERT(cellFree == 0 || cellFree >= pPage->pFreeEnd - pPage->pData); + if (cellFree != 0 && cellFree < pPage->pFreeEnd - pPage->pData) { + tdbError("tdb/page-allocate: cellFree: %d, pFreeEnd: %p, pData: %p.", cellFree, pPage->pFreeEnd, pPage->pData); + return -1; + } if (cellFree && pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) { SCell *pPrevFreeCell = NULL; int szPrevFreeCell; @@ -353,16 +409,30 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { // 3. Try to dfragment and allocate again tdbPageDefragment(pPage); - ASSERT(pPage->pFreeEnd - pPage->pFreeStart == nFree); - ASSERT(nFree == TDB_PAGE_NFREE(pPage)); - ASSERT(pPage->pFreeEnd - pPage->pData == TDB_PAGE_CCELLS(pPage)); + if (pPage->pFreeEnd - pPage->pFreeStart != nFree) { + tdbError("tdb/page-allocate: nFree: %d, pFreeStart: %p, pFreeEnd: %p.", nFree, pPage->pFreeStart, pPage->pFreeEnd); + return -1; + } + if (TDB_PAGE_NFREE(pPage) != nFree) { + tdbError("tdb/page-allocate: nFree: %d, page free: %d.", nFree, TDB_PAGE_NFREE(pPage)); + return -1; + } + if (pPage->pFreeEnd - pPage->pData != TDB_PAGE_CCELLS(pPage)) { + tdbError("tdb/page-allocate: ccells: %d, pFreeStart: %p, pData: %p.", TDB_PAGE_CCELLS(pPage), pPage->pFreeStart, + pPage->pData); + return -1; + } pPage->pFreeEnd -= szCell; pCell = pPage->pFreeEnd; TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); _alloc_finish: - ASSERT(pCell); + if (NULL == pCell) { + tdbError("tdb/page-allocate: null ptr pCell."); + return -1; + } + pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); TDB_PAGE_NFREE_SET(pPage, nFree - szCell - TDB_PAGE_OFFSET_SIZE(pPage)); *ppCell = pCell; @@ -375,9 +445,18 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) { u8 *dest; u8 *src; - ASSERT(pCell >= pPage->pFreeEnd); - ASSERT(pCell + szCell <= (u8 *)(pPage->pPageFtr)); - ASSERT(pCell == TDB_PAGE_CELL_AT(pPage, idx)); + if (pCell < pPage->pFreeEnd) { + tdbError("tdb/page-free: invalid cell, cell: %p, free end: %p", pCell, pPage->pFreeEnd); + return -1; + } + if (pCell + szCell > (u8 *)(pPage->pPageFtr)) { + tdbError("tdb/page-free: cell crosses page footer, cell: %p, size: %d footer: %p", pCell, szCell, pPage->pFreeEnd); + return -1; + } + if (pCell != TDB_PAGE_CELL_AT(pPage, idx)) { + tdbError("tdb/page-free: cell pos incorrect, cell: %p, pos: %p", pCell, TDB_PAGE_CELL_AT(pPage, idx)); + return -1; + } nFree = TDB_PAGE_NFREE(pPage); @@ -390,7 +469,8 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) { pPage->pPageMethods->setFreeCellInfo(pCell, szCell, cellFree); TDB_PAGE_FCELL_SET(pPage, pCell - pPage->pData); } else { - ASSERT(0); + tdbError("tdb/page-free: invalid cell size: %d", szCell); + return -1; } } @@ -417,7 +497,10 @@ static int tdbPageDefragment(SPage *pPage) { nFree = TDB_PAGE_NFREE(pPage); nCells = TDB_PAGE_NCELLS(pPage); - ASSERT(pPage->pFreeEnd - pPage->pFreeStart < nFree); + if (pPage->pFreeEnd - pPage->pFreeStart >= nFree) { + tdbError("tdb/page-defragment: invalid free range, nFree: %d.", nFree); + return -1; + } // Loop to compact the page content // Here we use an O(n^2) algorithm to do the job since @@ -443,11 +526,19 @@ static int tdbPageDefragment(SPage *pPage) { } } - ASSERT(pCell != NULL); + if (NULL == pCell) { + tdbError("tdb/page-defragment: null ptr pCell."); + return -1; + } szCell = (*pPage->xCellSize)(pPage, pCell, 0, NULL, NULL); - ASSERT(pCell + szCell <= pNextCell); + if (pCell + szCell > pNextCell) { + tdbError("tdb/page-defragment: invalid cell range, pCell: %p, szCell: %d, pNextCell: %p.", pCell, szCell, + pNextCell); + return -1; + } + if (pCell + szCell < pNextCell) { memmove(pNextCell - szCell, pCell, szCell); } @@ -457,7 +548,11 @@ static int tdbPageDefragment(SPage *pPage) { TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pNextCell - pPage->pData); } - ASSERT(pPage->pFreeEnd - pPage->pFreeStart == nFree); + if (pPage->pFreeEnd - pPage->pFreeStart != nFree) { + tdbError("tdb/page-defragment: invalid free range, nFree: %d.", nFree); + return -1; + } + TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); TDB_PAGE_FCELL_SET(pPage, 0); @@ -483,39 +578,59 @@ typedef struct { // cellNum static inline int getPageCellNum(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellNum; } static inline void setPageCellNum(SPage *pPage, int cellNum) { - ASSERT(cellNum < 65536); + if (cellNum >= 65536) { + tdbError("tdb/page-set-cell-num: invalid cellNum: %d.", cellNum); + return; + } ((SPageHdr *)(pPage->pPageHdr))[0].cellNum = (u16)cellNum; } // cellBody static inline int getPageCellBody(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellBody; } static inline void setPageCellBody(SPage *pPage, int cellBody) { - ASSERT(cellBody < 65536); + if (cellBody >= 65536) { + tdbError("tdb/page-set-cell-body: invalid cellBody: %d.", cellBody); + return; + } ((SPageHdr *)(pPage->pPageHdr))[0].cellBody = (u16)cellBody; } // cellFree static inline int getPageCellFree(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellFree; } static inline void setPageCellFree(SPage *pPage, int cellFree) { - ASSERT(cellFree < 65536); + if (cellFree >= 65536) { + tdbError("tdb/page-set-cell-free: invalid cellFree: %d.", cellFree); + return; + } ((SPageHdr *)(pPage->pPageHdr))[0].cellFree = (u16)cellFree; } // nFree static inline int getPageNFree(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].nFree; } static inline void setPageNFree(SPage *pPage, int nFree) { - ASSERT(nFree < 65536); + if (nFree >= 65536) { + tdbError("tdb/page-set-nfree: invalid nFree: %d.", nFree); + return; + } ((SPageHdr *)(pPage->pPageHdr))[0].nFree = (u16)nFree; } // cell offset static inline int getPageCellOffset(SPage *pPage, int idx) { - ASSERT(idx >= 0 && idx < getPageCellNum(pPage)); + int cellNum = getPageCellNum(pPage); + if (idx < 0 || idx >= cellNum) { + tdbError("tdb/page-cell-offset: idx: %d out of range[%d, %d).", idx, 0, cellNum); + return -1; + } + return ((u16 *)pPage->pCellIdx)[idx]; } static inline void setPageCellOffset(SPage *pPage, int idx, int offset) { - ASSERT(offset < 65536); + if (offset >= 65536) { + tdbError("tdb/page-set-cell-offset: invalid offset: %d.", offset); + return; + } ((u16 *)pPage->pCellIdx)[idx] = (u16)offset; } @@ -590,7 +705,12 @@ static inline void setLPageNFree(SPage *pPage, int nFree) { // cell offset static inline int getLPageCellOffset(SPage *pPage, int idx) { - ASSERT(idx >= 0 && idx < getLPageCellNum(pPage)); + int cellNum = getLPageCellNum(pPage); + if (idx < 0 || idx >= cellNum) { + tdbError("tdb/lpage-cell-offset: idx: %d out of range[%d, %d).", idx, 0, cellNum); + return -1; + } + return TDB_GET_U24(pPage->pCellIdx + 3 * idx); } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 62d82edeb1a3fc59c7dbf34a9ec40ba2e9eaa9d1..ced867e938fc66574e70774e0e99000be2bf2fa5 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -14,7 +14,7 @@ */ #include "tdbInt.h" - +/* #pragma pack(push, 1) typedef struct { u8 hdrString[16]; @@ -26,7 +26,7 @@ typedef struct { #pragma pack(pop) TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct"); - +*/ struct hashset_st { size_t nbits; size_t mask; @@ -234,7 +234,6 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { int ret; SPage **ppPage; - // ASSERT(pPager->inTran); if (pPage->isDirty) return 0; // ref page one more time so the page will not be release @@ -243,23 +242,8 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { // Set page as dirty pPage->isDirty = 1; - /* - // Add page to dirty list(TODO: NOT use O(n^2) algorithm) - for (ppPage = &pPager->pDirty; (*ppPage) && TDB_PAGE_PGNO(*ppPage) < TDB_PAGE_PGNO(pPage); - ppPage = &((*ppPage)->pDirtyNext)) { - } - - if (*ppPage && TDB_PAGE_PGNO(*ppPage) == TDB_PAGE_PGNO(pPage)) { - tdbUnrefPage(pPage); - return 0; - } - - ASSERT(*ppPage == NULL || TDB_PAGE_PGNO(*ppPage) > TDB_PAGE_PGNO(pPage)); - pPage->pDirtyNext = *ppPage; - *ppPage = pPage; - */ - tdbTrace("put page: %p %d to dirty tree: %p", pPage, TDB_PAGE_PGNO(pPage), &pPager->rbt); + tdbTrace("tdb/pager-write: put page: %p %d to dirty tree: %p", pPage, TDB_PAGE_PGNO(pPage), &pPager->rbt); tRBTreePut(&pPager->rbt, (SRBTreeNode *)pPage); // Write page to journal if neccessary @@ -327,7 +311,11 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { while ((pNode = tRBTreeIterNext(&iter)) != NULL) { pPage = (SPage *)pNode; - ASSERT(pPage->nOverflow == 0); + if (pPage->nOverflow != 0) { + tdbError("tdb/pager-commit: %p, pPage: %p, ovfl: %d, commit page failed.", pPager, pPage, pPage->nOverflow); + return -1; + } + ret = tdbPagerPWritePageToDB(pPager, pPage); if (ret < 0) { tdbError("failed to write page to db since %s", tstrerror(terrno)); @@ -652,12 +640,15 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa loadPage = 0; ret = tdbPagerAllocPage(pPager, &pgno); if (ret < 0) { - ASSERT(0); + tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno); return -1; } } - ASSERT(pgno > 0); + if (pgno == 0) { + tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno); + return -1; + } // fetch a page container memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN); @@ -671,7 +662,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa if (!TDB_PAGE_INITIALIZED(pPage)) { ret = tdbPagerInitPage(pPager, pPage, initPage, arg, loadPage); if (ret < 0) { - ASSERT(0); + tdbError("tdb/pager: %p, pPage: %p, init page failed.", pPager, pPage); return -1; } } @@ -679,8 +670,14 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa // printf("thread %" PRId64 " pager fetch page %d pgno %d ppage %p\n", taosGetSelfPthreadId(), pPage->id, // TDB_PAGE_PGNO(pPage), pPage); - ASSERT(TDB_PAGE_INITIALIZED(pPage)); - ASSERT(pPage->pPager == pPager); + if (!TDB_PAGE_INITIALIZED(pPage)) { + tdbError("tdb/pager: %p, pPage: %p, fetch page uninited.", pPager, pPage); + return -1; + } + if (pPage->pPager != pPager) { + tdbError("tdb/pager: %p/%p, fetch page failed.", pPager, pPage->pPager); + return -1; + } *ppgno = pgno; *ppPage = pPage; @@ -722,8 +719,10 @@ int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno) { return -1; } - ASSERT(*ppgno != 0); - + if (*ppgno == 0) { + tdbError("tdb/pager:%p, alloc new page failed.", pPager); + return -1; + } return 0; } @@ -732,8 +731,8 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage int ret; int lcode; int nLoops; - i64 nRead; - SPgno pgno; + i64 nRead = 0; + SPgno pgno = 0; int init = 0; lcode = TDB_TRY_LOCK_PAGE(pPage); @@ -752,7 +751,6 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage nRead = tdbOsPRead(pPager->fd, pPage->pData, pPage->pageSize, ((i64)pPage->pageSize) * (pgno - 1)); tdbTrace("tdb/pager:%p, pgno:%d, nRead:%" PRId64, pPager, pgno, nRead); if (nRead < pPage->pageSize) { - ASSERT(0); tdbError("tdb/pager:%p, pgno:%d, nRead:%" PRId64 "pgSize:%" PRId32, pPager, pgno, nRead, pPage->pageSize); TDB_UNLOCK_PAGE(pPage); return -1; @@ -763,7 +761,8 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage ret = (*initPage)(pPage, arg, init); if (ret < 0) { - ASSERT(0); + tdbError("tdb/pager:%p, pgno:%d, nRead:%" PRId64 "pgSize:%" PRId32 " init page failed.", pPager, pgno, nRead, + pPage->pageSize); TDB_UNLOCK_PAGE(pPage); return -1; } @@ -782,7 +781,8 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage } } } else { - ASSERT(0); + tdbError("tdb/pager:%p, pgno:%d, nRead:%" PRId64 "pgSize:%" PRId32 " lock page failed.", pPager, pgno, nRead, + pPage->pageSize); return -1; } diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index 972e2f29e545ae24c9fd4dd19ec29f553480c602..18d14fa474f51104442cf689bd52649598b9c396 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -105,8 +105,6 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF #endif - ASSERT(pPager != NULL); - if (rollback) { ret = tdbPagerRestoreJournals(pPager); if (ret < 0) { diff --git a/source/libs/tdb/src/db/tdbTxn.c b/source/libs/tdb/src/db/tdbTxn.c index 24f955fe2fde29c089b4768baae54eaea892d09e..0aeed3c140f8317c6abcec16a9fd049f6be89e1a 100644 --- a/source/libs/tdb/src/db/tdbTxn.c +++ b/source/libs/tdb/src/db/tdbTxn.c @@ -18,7 +18,10 @@ int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg, int flags) { // not support read-committed version at the moment - ASSERT(flags == 0 || flags == (TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)); + if (flags != 0 && flags != (TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)) { + tdbError("tdb/txn: invalid txn flags: %" PRId32, flags); + return -1; + } pTxn->flags = flags; pTxn->txnId = txnid; diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 943611ee27c6f1d6bfa51dfff211bf723764f88f..9a71fc7f30ffebb5cf864d30faea3b8690438112 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -284,7 +284,9 @@ int32_t tfsMkdir(STfs *pTfs, const char *rname) { } int32_t tfsRmdir(STfs *pTfs, const char *rname) { - ASSERT(rname[0] != 0); + if (rname[0] == 0) { + return 0; + } char aname[TMPNAME_LEN] = "\0"; diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 44e88a4dcc425a728789a5e95b9fbf19bcaae415..4a0570bee34cafb95f2ce389fa48dbde831383c0 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -25,7 +25,7 @@ bool FORCE_INLINE walLogExist(SWal* pWal, int64_t ver) { } bool FORCE_INLINE walIsEmpty(SWal* pWal) { - return (pWal->vers.firstVer == -1 || pWal->vers.lastVer < pWal->vers.firstVer); // [firstVer, lastVer + 1) + return (pWal->vers.firstVer == -1 || pWal->vers.lastVer < pWal->vers.firstVer); // [firstVer, lastVer + 1) } int64_t FORCE_INLINE walGetFirstVer(SWal* pWal) { return pWal->vers.firstVer; } @@ -49,7 +49,6 @@ static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) { static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { int32_t sz = taosArrayGetSize(pWal->fileInfoSet); terrno = TSDB_CODE_SUCCESS; - ASSERT(fileIdx >= 0 && fileIdx < sz); SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); char fnameStr[WAL_FILE_LEN]; @@ -101,7 +100,6 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { offsetBackward = offset; } - ASSERT(offset <= end); readSize = end - offset; capacity = readSize + sizeof(magic); @@ -257,7 +255,6 @@ static void walRebuildFileInfoSet(SArray* metaLogList, SArray* actualLogList) { SWalFileInfo* pLogInfo = taosArrayGet(actualLogList, i); while (j < metaFileNum) { SWalFileInfo* pMetaInfo = taosArrayGet(metaLogList, j); - ASSERT(pMetaInfo != NULL); if (pMetaInfo->firstVer < pLogInfo->firstVer) { j++; } else if (pMetaInfo->firstVer == pLogInfo->firstVer) { @@ -376,7 +373,6 @@ int walCheckAndRepairMeta(SWal* pWal) { taosArrayDestroy(actualLog); int32_t sz = taosArrayGetSize(pWal->fileInfoSet); - ASSERT(sz == actualFileNum); // scan and determine the lastVer int32_t fileIdx = sz; @@ -394,8 +390,6 @@ int walCheckAndRepairMeta(SWal* pWal) { return -1; } - ASSERT(pFileInfo->firstVer >= 0); - if (pFileInfo->lastVer >= pFileInfo->firstVer && fileSize == pFileInfo->fileSize) { totSize += pFileInfo->fileSize; continue; @@ -408,7 +402,6 @@ int walCheckAndRepairMeta(SWal* pWal) { wError("failed to scan wal last ver since %s", terrstr()); return -1; } - ASSERT(pFileInfo->fileSize == 0); // remove the empty wal log, and its idx wInfo("vgId:%d, wal remove empty file %s", pWal->cfg.vgId, fnameStr); taosRemoveFile(fnameStr); @@ -468,8 +461,7 @@ int walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pCkHead) { } int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { - int32_t sz = taosArrayGetSize(pWal->fileInfoSet); - ASSERT(fileIdx >= 0 && fileIdx < sz); + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); @@ -483,7 +475,6 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { return -1; } - ASSERT(pFileInfo->fileSize > 0 && pFileInfo->firstVer >= 0 && pFileInfo->lastVer >= pFileInfo->firstVer); if (fileSize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) { return 0; } @@ -547,7 +538,7 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { } offset += sizeof(SWalIdxEntry); - ASSERT(offset == (idxEntry.ver - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)); + /*A(offset == (idxEntry.ver - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry));*/ // ftruncate idx file if (offset < fileSize) { @@ -568,7 +559,7 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { } while (idxEntry.ver < pFileInfo->lastVer) { - ASSERT(idxEntry.ver == ckHead.head.version); + /*A(idxEntry.ver == ckHead.head.version);*/ idxEntry.ver += 1; idxEntry.offset += sizeof(SWalCkHead) + ckHead.head.bodyLen; @@ -640,8 +631,7 @@ int walRollFileInfo(SWal* pWal) { } char* walMetaSerialize(SWal* pWal) { - char buf[30]; - ASSERT(pWal->fileInfoSet); + char buf[30]; int sz = taosArrayGetSize(pWal->fileInfoSet); cJSON* pRoot = cJSON_CreateObject(); cJSON* pMeta = cJSON_CreateObject(); @@ -698,7 +688,7 @@ char* walMetaSerialize(SWal* pWal) { } int walMetaDeserialize(SWal* pWal, const char* bytes) { - ASSERT(taosArrayGetSize(pWal->fileInfoSet) == 0); + /*A(taosArrayGetSize(pWal->fileInfoSet) == 0);*/ cJSON *pRoot, *pMeta, *pFiles, *pInfoJson, *pField; pRoot = cJSON_Parse(bytes); if (!pRoot) goto _err; @@ -814,7 +804,9 @@ int walSaveMeta(SWal* pWal) { // flush to a tmpfile n = walBuildTmpMetaName(pWal, tmpFnameStr); - ASSERT(n < sizeof(tmpFnameStr) && "Buffer overflow of file name"); + if (n >= sizeof(tmpFnameStr)) { + return -1; + } TdFilePtr pMetaFile = taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pMetaFile == NULL) { @@ -845,7 +837,9 @@ int walSaveMeta(SWal* pWal) { // rename it n = walBuildMetaName(pWal, metaVer + 1, fnameStr); - ASSERT(n < sizeof(fnameStr) && "Buffer overflow of file name"); + if (n >= sizeof(fnameStr)) { + goto _err; + } if (taosRenameFile(tmpFnameStr, fnameStr) < 0) { wError("failed to rename file due to %s. dest:%s", strerror(errno), fnameStr); @@ -868,7 +862,6 @@ _err: } int walLoadMeta(SWal* pWal) { - ASSERT(pWal->fileInfoSet->size == 0); // find existing meta file int metaVer = walFindCurMetaVer(pWal); if (metaVer == -1) { diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 07228bddd82ee6711627e38a2fd497495752251d..d068ede52d217a389cce32bc8ff01b5a813bd47c 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -97,7 +97,6 @@ int32_t walNextValidMsg(SWalReader *pReader) { return -1; } fetchVer++; - ASSERT(fetchVer == pReader->curVersion); } } pReader->curStopped = 1; @@ -132,7 +131,6 @@ static int64_t walReadSeekFilePos(SWalReader *pReader, int64_t fileFirstVer, int return -1; } - ASSERT(entry.ver == ver); ret = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -241,7 +239,6 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { if (pRead->curInvalid || pRead->curVersion != fetchVer) { if (walReadSeekVer(pRead, fetchVer) < 0) { - ASSERT(0); pRead->curVersion = fetchVer; pRead->curInvalid = 1; return -1; @@ -262,7 +259,6 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { } else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } - ASSERT(0); pRead->curInvalid = 1; return -1; } @@ -271,71 +267,65 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { return 0; } -static int32_t walFetchBodyNew(SWalReader *pRead) { - SWalCont *pReadHead = &pRead->pHead->head; +static int32_t walFetchBodyNew(SWalReader *pReader) { + SWalCont *pReadHead = &pReader->pHead->head; int64_t ver = pReadHead->version; - wDebug("vgId:%d, wal starts to fetch body, index:%" PRId64, pRead->pWal->cfg.vgId, ver); + wDebug("vgId:%d, wal starts to fetch body, ver:%" PRId64 " ,len:%d", pReader->pWal->cfg.vgId, ver, + pReadHead->bodyLen); - if (pRead->capacity < pReadHead->bodyLen) { - SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen); + if (pReader->capacity < pReadHead->bodyLen) { + SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen); if (ptr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pRead->pHead = ptr; - pReadHead = &pRead->pHead->head; - pRead->capacity = pReadHead->bodyLen; + pReader->pHead = ptr; + pReadHead = &pReader->pHead->head; + pReader->capacity = pReadHead->bodyLen; } - if (pReadHead->bodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, pReadHead->bodyLen)) { + if (pReadHead->bodyLen != taosReadFile(pReader->pLogFile, pReadHead->body, pReadHead->bodyLen)) { if (pReadHead->bodyLen < 0) { terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s", - pRead->pWal->cfg.vgId, pRead->pHead->head.version, ver, tstrerror(terrno)); + pReader->pWal->cfg.vgId, pReader->pHead->head.version, ver, tstrerror(terrno)); } else { wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since file corrupted", - pRead->pWal->cfg.vgId, pRead->pHead->head.version, ver); + pReader->pWal->cfg.vgId, pReader->pHead->head.version, ver); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } - pRead->curInvalid = 1; - ASSERT(0); + pReader->curInvalid = 1; return -1; } if (pReadHead->version != ver) { - wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64, pRead->pWal->cfg.vgId, - pRead->pHead->head.version, ver); - pRead->curInvalid = 1; + wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64, pReader->pWal->cfg.vgId, + pReader->pHead->head.version, ver); + pReader->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - ASSERT(0); return -1; } - if (walValidBodyCksum(pRead->pHead) != 0) { - wError("vgId:%d, wal fetch body error:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, ver); - pRead->curInvalid = 1; + if (walValidBodyCksum(pReader->pHead) != 0) { + wError("vgId:%d, wal fetch body error:%" PRId64 ", since body checksum not passed", pReader->pWal->cfg.vgId, ver); + pReader->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - ASSERT(0); return -1; } - wDebug("vgId:%d, index:%" PRId64 " is fetched, cursor advance", pRead->pWal->cfg.vgId, ver); - pRead->curVersion = ver + 1; + wDebug("vgId:%d, index:%" PRId64 " is fetched, cursor advance", pReader->pWal->cfg.vgId, ver); + pReader->curVersion = ver + 1; return 0; } static int32_t walSkipFetchBodyNew(SWalReader *pRead) { int64_t code; - ASSERT(pRead->curVersion == pRead->pHead->head.version); - ASSERT(pRead->curInvalid == 0); - code = taosLSeekFile(pRead->pLogFile, pRead->pHead->head.bodyLen, SEEK_CUR); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); pRead->curInvalid = 1; - ASSERT(0); return -1; } @@ -384,7 +374,6 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) { } else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } - ASSERT(0); pRead->curInvalid = 1; return -1; } @@ -410,9 +399,6 @@ int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead) { pRead->pWal->cfg.vgId, pHead->head.version, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer, pRead->pWal->vers.lastVer, pRead->pWal->vers.appliedVer); - ASSERT(pRead->curVersion == pHead->head.version); - ASSERT(pRead->curInvalid == 0); - code = taosLSeekFile(pRead->pLogFile, pHead->head.bodyLen, SEEK_CUR); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -447,7 +433,6 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) { if (pReadHead->bodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, pReadHead->bodyLen)) { if (pReadHead->bodyLen < 0) { - ASSERT(0); terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s", pRead->pWal->cfg.vgId, pReadHead->version, ver, tstrerror(terrno)); @@ -457,12 +442,10 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } pRead->curInvalid = 1; - ASSERT(0); return -1; } if (pReadHead->version != ver) { - ASSERT(0); wError("vgId:%d, wal fetch body error, index:%" PRId64 ", read request index:%" PRId64, pRead->pWal->cfg.vgId, pReadHead->version, ver); pRead->curInvalid = 1; @@ -471,7 +454,6 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) { } if (walValidBodyCksum(*ppHead) != 0) { - ASSERT(0); wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, ver); pRead->curInvalid = 1; diff --git a/source/libs/wal/src/walRef.c b/source/libs/wal/src/walRef.c index e86111109ce4fc9768be8b662670ae711f935e39..fa04ba3e58a05c3ee902bde8ace004e821683f37 100644 --- a/source/libs/wal/src/walRef.c +++ b/source/libs/wal/src/walRef.c @@ -61,7 +61,7 @@ int32_t walRefVer(SWalRef *pRef, int64_t ver) { SWalFileInfo tmpInfo; tmpInfo.firstVer = ver; SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); - ASSERT(pRet != NULL); + /*A(pRet != NULL);*/ pRef->refFile = pRet->firstVer; taosThreadMutexUnlock(&pWal->mutex); @@ -80,6 +80,7 @@ void walUnrefVer(SWalRef *pRef) { SWalRef *walRefCommittedVer(SWal *pWal) { SWalRef *pRef = walOpenRef(pWal); if (pRef == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } taosThreadMutexLock(&pWal->mutex); @@ -91,7 +92,7 @@ SWalRef *walRefCommittedVer(SWal *pWal) { SWalFileInfo tmpInfo; tmpInfo.firstVer = ver; SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); - ASSERT(pRet != NULL); + /*A(pRet != NULL);*/ pRef->refFile = pRet->firstVer; taosThreadMutexUnlock(&pWal->mutex); diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 2cb6614b01bab90cc41e0d11408f3380de4a95fc..cbfd0ef7417389b662ece46d4032f2f7a1a31cbd 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -40,7 +40,6 @@ static int64_t walSeekWritePos(SWal* pWal, int64_t ver) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - ASSERT(entry.ver == ver); code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -53,8 +52,7 @@ static int64_t walSeekWritePos(SWal* pWal, int64_t ver) { int walInitWriteFile(SWal* pWal) { TdFilePtr pIdxTFile, pLogTFile; SWalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); - ASSERT(pRet != NULL); - int64_t fileFirstVer = pRet->firstVer; + int64_t fileFirstVer = pRet->firstVer; char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, fileFirstVer, fnameStr); @@ -109,9 +107,8 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) { tmpInfo.firstVer = ver; // bsearch in fileSet int32_t idx = taosArraySearchIdx(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); - ASSERT(idx != -1); + /*A(idx != -1);*/ SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, idx); - /*ASSERT(pFileInfo != NULL);*/ int64_t fileFirstVer = pFileInfo->firstVer; walBuildIdxName(pWal, fileFirstVer, fnameStr); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index d4ea526b7839193a8bebac5e5f4205ba688900d9..0562dc10cee9b4aef88c76a2257cb728207b6d14 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -87,12 +87,10 @@ int32_t walApplyVer(SWal *pWal, int64_t ver) { } int32_t walCommit(SWal *pWal, int64_t ver) { - ASSERT(pWal->vers.commitVer >= pWal->vers.snapshotVer); - ASSERT(pWal->vers.commitVer <= pWal->vers.lastVer); if (ver < pWal->vers.commitVer) { return 0; } - if (ver > pWal->vers.lastVer) { + if (ver > pWal->vers.lastVer || pWal->vers.commitVer < pWal->vers.snapshotVer) { terrno = TSDB_CODE_WAL_INVALID_VER; return -1; } @@ -138,25 +136,21 @@ int32_t walRollback(SWal *pWal, int64_t ver) { TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND); if (pIdxFile == NULL) { - ASSERT(0); taosThreadMutexUnlock(&pWal->mutex); return -1; } int64_t idxOff = walGetVerIdxOffset(pWal, ver); code = taosLSeekFile(pIdxFile, idxOff, SEEK_SET); if (code < 0) { - ASSERT(0); taosThreadMutexUnlock(&pWal->mutex); return -1; } // read idx file and get log file pos SWalIdxEntry entry; if (taosReadFile(pIdxFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { - ASSERT(0); taosThreadMutexUnlock(&pWal->mutex); return -1; } - ASSERT(entry.ver == ver); walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr); TdFilePtr pLogFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND); @@ -176,24 +170,19 @@ int32_t walRollback(SWal *pWal, int64_t ver) { } // validate offset SWalCkHead head; - ASSERT(taosValidFile(pLogFile)); - int64_t size = taosReadFile(pLogFile, &head, sizeof(SWalCkHead)); + int64_t size = taosReadFile(pLogFile, &head, sizeof(SWalCkHead)); if (size != sizeof(SWalCkHead)) { - ASSERT(0); taosThreadMutexUnlock(&pWal->mutex); return -1; } code = walValidHeadCksum(&head); - ASSERT(code == 0); if (code != 0) { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - ASSERT(0); taosThreadMutexUnlock(&pWal->mutex); return -1; } if (head.head.version != ver) { - ASSERT(0); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; taosThreadMutexUnlock(&pWal->mutex); return -1; @@ -202,22 +191,22 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // truncate old files code = taosFtruncateFile(pLogFile, entry.offset); if (code < 0) { - ASSERT(0); terrno = TAOS_SYSTEM_ERROR(errno); taosThreadMutexUnlock(&pWal->mutex); return -1; } code = taosFtruncateFile(pIdxFile, idxOff); if (code < 0) { - ASSERT(0); terrno = TAOS_SYSTEM_ERROR(errno); taosThreadMutexUnlock(&pWal->mutex); return -1; } pWal->vers.lastVer = ver - 1; +#if 0 if (pWal->vers.lastVer < pWal->vers.firstVer) { - ASSERT(pWal->vers.lastVer == pWal->vers.firstVer - 1); + A(pWal->vers.lastVer == pWal->vers.firstVer - 1); } +#endif ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; taosCloseFile(&pIdxFile); @@ -386,7 +375,8 @@ int32_t walEndSnapshot(SWal *pWal) { walBuildIdxName(pWal, pInfo->firstVer, fnameStr); wDebug("vgId:%d, wal remove file %s", pWal->cfg.vgId, fnameStr); if (taosRemoveFile(fnameStr) < 0 && errno != ENOENT) { - ASSERT(0); + wError("vgId:%d, failed to remove idx file %s due to %s", pWal->cfg.vgId, fnameStr, strerror(errno)); + goto END; } } taosArrayClear(pWal->toDeleteFiles); @@ -441,7 +431,6 @@ int32_t walRollImpl(SWal *pWal) { pWal->pIdxFile = pIdxFile; pWal->pLogFile = pLogFile; pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; - ASSERT(pWal->writeCur >= 0); pWal->lastRollSeq = walGetSeq(); @@ -458,8 +447,7 @@ END: static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { SWalIdxEntry entry = {.ver = ver, .offset = offset}; SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal); - ASSERT(pFileInfo != NULL); - ASSERT(pFileInfo->firstVer >= 0); + int64_t idxOffset = (entry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry); wDebug("vgId:%d, write index, index:%" PRId64 ", offset:%" PRId64 ", at %" PRId64, pWal->cfg.vgId, ver, offset, idxOffset); @@ -476,7 +464,6 @@ static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { if (endOffset < 0) { wFatal("vgId:%d, failed to seek end of idxfile due to %s. ver:%" PRId64 "", pWal->cfg.vgId, strerror(errno), ver); } - ASSERT(endOffset == idxOffset + sizeof(SWalIdxEntry) && "Offset of idx entries misaligned"); return 0; } @@ -486,9 +473,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy int64_t offset = walGetCurFileOffset(pWal); SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal); - ASSERT(pFileInfo != NULL); - ASSERT(pFileInfo->firstVer != -1); pWal->writeHead.head.version = index; pWal->writeHead.head.bodyLen = bodyLen; pWal->writeHead.head.msgType = msgType; @@ -525,7 +510,6 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy // set status if (pWal->vers.firstVer == -1) { - ASSERT(index == 0); pWal->vers.firstVer = 0; } pWal->vers.lastVer = index; @@ -541,7 +525,6 @@ END: wFatal("vgId:%d, failed to ftruncate logfile to offset:%" PRId64 " during recovery due to %s", pWal->cfg.vgId, offset, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); - ASSERT(0 && "failed to recover from error"); } int64_t idxOffset = (index - pFileInfo->firstVer) * sizeof(SWalIdxEntry); @@ -549,7 +532,6 @@ END: wFatal("vgId:%d, failed to ftruncate idxfile to offset:%" PRId64 "during recovery due to %s", pWal->cfg.vgId, idxOffset, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); - ASSERT(0 && "failed to recover from error"); } return -1; } @@ -576,8 +558,6 @@ int64_t walAppendLog(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syn } } - ASSERT(pWal->pLogFile != NULL && pWal->pIdxFile != NULL && pWal->writeCur >= 0); - if (walWriteImpl(pWal, index, msgType, syncMeta, body, bodyLen) < 0) { taosThreadMutexUnlock(&pWal->mutex); return -1; @@ -614,8 +594,6 @@ int32_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SWalSync } } - ASSERT(pWal->pIdxFile != NULL && pWal->pLogFile != NULL && pWal->writeCur >= 0); - if (walWriteImpl(pWal, index, msgType, syncMeta, body, bodyLen) < 0) { taosThreadMutexUnlock(&pWal->mutex); return -1; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 3f527e913025af968753a47a77e713ac165874fc..71b3125de8ff42f4a3ea1ef19690c6053a00e707 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -191,7 +191,7 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { int32_t code = _stati64(path, &fileStat); #else struct stat fileStat; - int32_t code = stat(path, &fileStat); + int32_t code = stat(path, &fileStat); #endif if (code < 0) { return code; @@ -208,10 +208,9 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { return 0; } int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { - if (pFile == NULL) { - return 0; + if (pFile == NULL || pFile->fd < 0) { + return -1; } - assert(pFile->fd >= 0); // Please check if you have closed the file. #ifdef WINDOWS @@ -233,7 +232,7 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { #else struct stat fileStat; - int32_t code = fstat(pFile->fd, &fileStat); + int32_t code = fstat(pFile->fd, &fileStat); if (code < 0) { printf("taosFStatFile run fstat fail."); return code; @@ -265,7 +264,10 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { } else { mode = (tdFileOptions & TD_FILE_TEXT) ? "rt+" : "rb+"; } - assert(!(tdFileOptions & TD_FILE_EXCL)); + ASSERT(!(tdFileOptions & TD_FILE_EXCL)); + if (tdFileOptions & TD_FILE_EXCL) { + return NULL; + } fp = fopen(path, mode); if (fp == NULL) { return NULL; @@ -364,7 +366,13 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + ASSERT(pFile->fd >= 0); // Please check if you have closed the file. + if (pFile->fd < 0) { +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif + return -1; + } int64_t leftbytes = count; int64_t readbytes; char *tbuf = (char *)buf; @@ -408,7 +416,13 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + ASSERT(pFile->fd >= 0); // Please check if you have closed the file. + if (pFile->fd < 0) { +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif + return -1; + } #ifdef WINDOWS size_t pos = _lseeki64(pFile->fd, 0, SEEK_CUR); _lseeki64(pFile->fd, offset, SEEK_SET); @@ -431,6 +445,9 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { taosThreadRwlockWrlock(&(pFile->rwlock)); #endif if (pFile->fd < 0) { +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif return 0; } @@ -466,7 +483,13 @@ int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t #if FILE_WITH_LOCK taosThreadRwlockWrlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + ASSERT(pFile->fd >= 0); // Please check if you have closed the file. + if (pFile->fd < 0) { +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif + return 0; + } #ifdef WINDOWS size_t pos = _lseeki64(pFile->fd, 0, SEEK_CUR); _lseeki64(pFile->fd, offset, SEEK_SET); @@ -485,7 +508,10 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + ASSERT(pFile->fd >= 0); // Please check if you have closed the file. + if (pFile->fd < 0) { + return -1; + } #ifdef WINDOWS int64_t ret = _lseeki64(pFile->fd, offset, whence); #else @@ -501,7 +527,10 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); // Please check if you have closed the file. + ASSERT(pFile->fd >= 0); // Please check if you have closed the file. + if (pFile->fd < 0) { + return -1; + } struct stat fileStat; #ifdef WINDOWS @@ -525,6 +554,10 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { } int32_t taosLockFile(TdFilePtr pFile) { + ASSERT(pFile->fd >= 0); // Please check if you have closed the file. + if (pFile->fd < 0) { + return -1; + } #ifdef WINDOWS BOOL fSuccess = FALSE; LARGE_INTEGER fileSize; @@ -543,17 +576,19 @@ int32_t taosLockFile(TdFilePtr pFile) { } return 0; #else - assert(pFile->fd >= 0); // Please check if you have closed the file. - return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB); #endif } int32_t taosUnLockFile(TdFilePtr pFile) { + ASSERT(pFile->fd >= 0); + if (pFile->fd < 0) { + return 0; + } #ifdef WINDOWS - BOOL fSuccess = FALSE; - OVERLAPPED overlapped = {0}; - HANDLE hFile = (HANDLE)_get_osfhandle(pFile->fd); + BOOL fSuccess = FALSE; + OVERLAPPED overlapped = {0}; + HANDLE hFile = (HANDLE)_get_osfhandle(pFile->fd); fSuccess = UnlockFileEx(hFile, 0, ~0, ~0, &overlapped); if (!fSuccess) { @@ -561,19 +596,19 @@ int32_t taosUnLockFile(TdFilePtr pFile) { } return 0; #else - assert(pFile->fd >= 0); // Please check if you have closed the file. - return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB); #endif } int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { -#ifdef WINDOWS + if (pFile == NULL) { + return 0; + } if (pFile->fd < 0) { - errno = EBADF; printf("Ftruncate file error, fd arg was negative\n"); return -1; } +#ifdef WINDOWS HANDLE h = (HANDLE)_get_osfhandle(pFile->fd); @@ -618,11 +653,6 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { return 0; #else - if (pFile == NULL) { - return 0; - } - assert(pFile->fd >= 0); // Please check if you have closed the file. - return ftruncate(pFile->fd, l_size); #endif } @@ -650,7 +680,10 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in if (pFileOut == NULL || pFileIn == NULL) { return 0; } - assert(pFileIn->fd >= 0 && pFileOut->fd >= 0); + ASSERT(pFileIn->fd >= 0 && pFileOut->fd >= 0); + if (pFileIn->fd < 0 || pFileOut->fd < 0) { + return 0; + } #ifdef WINDOWS @@ -743,11 +776,9 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in } void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { - if (pFile == NULL) { + if (pFile == NULL || pFile->fp == NULL) { return; } - assert(pFile->fp != NULL); - va_list ap; va_start(ap, format); vfprintf(pFile->fp, format, ap); @@ -772,7 +803,10 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { if (*ptrBuf != NULL) { taosMemoryFreeClear(*ptrBuf); } - assert(pFile->fp != NULL); + ASSERT(pFile->fp != NULL); + if (pFile->fp == NULL) { + return -1; + } #ifdef WINDOWS *ptrBuf = taosMemoryMalloc(1024); if (*ptrBuf == NULL) return -1; @@ -792,7 +826,10 @@ int64_t taosGetsFile(TdFilePtr pFile, int32_t maxSize, char *__restrict buf) { if (pFile == NULL || buf == NULL) { return -1; } - assert(pFile->fp != NULL); + ASSERT(pFile->fp != NULL); + if (pFile->fp == NULL) { + return -1; + } if (fgets(buf, maxSize, pFile->fp) == NULL) { return -1; } @@ -801,9 +838,12 @@ int64_t taosGetsFile(TdFilePtr pFile, int32_t maxSize, char *__restrict buf) { int32_t taosEOFFile(TdFilePtr pFile) { if (pFile == NULL) { - return 0; + return -1; + } + ASSERT(pFile->fp != NULL); + if (pFile->fp == NULL) { + return -1; } - assert(pFile->fp != NULL); return feof(pFile->fp); } diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c index 7319181a777cb8140396f592009507a151d2620b..b4a2845e961aa8a4c8d993304f869f8c66f3e9fb 100644 --- a/source/os/src/osLocale.c +++ b/source/os/src/osLocale.c @@ -71,7 +71,7 @@ char *taosCharsetReplace(char *charsetstr) { * seems does not response as expected. * * In some Linux systems, setLocale(LC_CTYPE, "") may return NULL, in which case the launch of - * both the TDengine Server and the Client may be interrupted. + * both the Server and the Client may be interrupted. * * In case that the setLocale failed to be executed, the right charset needs to be set. */ diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 2e68bd5ccd3ad937dbcfce6988fcb7ef6060f31a..3825b8e074b73cff3b2caec533671405916009e1 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -266,7 +266,10 @@ void *taosMemoryRealloc(void *ptr, int64_t size) { if (ptr == NULL) return taosMemoryMalloc(size); TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char *)ptr - sizeof(TdMemoryInfo)); - assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); + ASSERT(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); + if (tpTdMemoryInfo->symbol != TD_MEMORY_SYMBOL) { ++ return NULL; ++ } TdMemoryInfo tdMemoryInfo; memcpy(&tdMemoryInfo, pTdMemoryInfo, sizeof(TdMemoryInfo)); @@ -288,8 +291,10 @@ void *taosMemoryStrDup(const char *ptr) { if (ptr == NULL) return NULL; TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char *)ptr - sizeof(TdMemoryInfo)); - assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); - + ASSERT(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); + if (pTdMemoryInfo->symbol != TD_MEMORY_SYMBOL) { ++ return NULL; ++ } void *tmp = tstrdup(pTdMemoryInfo); if (tmp == NULL) return NULL; @@ -323,7 +328,10 @@ int64_t taosMemorySize(void *ptr) { #ifdef USE_TD_MEMORY TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char *)ptr - sizeof(TdMemoryInfo)); - assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); + ASSERT(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); + if (pTdMemoryInfo->symbol != TD_MEMORY_SYMBOL) { ++ return NULL; ++ } return pTdMemoryInfo->memorySize; #else @@ -348,7 +356,7 @@ void taosMemoryTrim(int32_t size) { void* taosMemoryMallocAlign(uint32_t alignment, int64_t size) { #ifdef USE_TD_MEMORY - assert(0); + ASSERT(0); #else #if defined(LINUX) void* p = memalign(alignment, size); diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 2f947d325263d45025a7cff835a715ac108674e7..cc018bf842f7e8a6835999211f08bbca2882db34 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -86,8 +86,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t ms) { while ((rc = sem_timedwait(sem, &ts)) == -1 && errno == EINTR) continue; return rc; /* This should have timed out */ - // assert(errno == ETIMEDOUT); - // assert(rc != 0); + // ASSERT(errno == ETIMEDOUT); + // ASSERT(rc != 0); // GetSystemTimeAsFileTime(&ft_after); // // We specified a non-zero wait. Time must advance. // if (ft_before.dwLowDateTime == ft_after.dwLowDateTime && ft_before.dwHighDateTime == ft_after.dwHighDateTime) diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 6611a937f2486693e5cf82eff7d9d1c7ec1394a1..715c2632e0ba0cf299d1180038e319571604d81f 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -298,7 +298,7 @@ int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void return -1; } #ifdef WINDOWS - assert(0); + ASSERT(0); return 0; #else return getsockopt(pSocket->fd, level, optname, optval, (int *)optlen); @@ -662,7 +662,7 @@ int32_t taosKeepTcpAlive(TdSocketPtr pSocket) { int taosGetLocalIp(const char *eth, char *ip) { #if defined(WINDOWS) // DO NOTHAING - assert(0); + ASSERT(0); return 0; #else int fd; @@ -689,7 +689,7 @@ int taosGetLocalIp(const char *eth, char *ip) { int taosValidIp(uint32_t ip) { #if defined(WINDOWS) // DO NOTHAING - assert(0); + ASSERT(0); return 0; #else int ret = -1; @@ -924,7 +924,7 @@ uint32_t ip2uint(const char *const ip_addr) { void taosBlockSIGPIPE() { #ifdef WINDOWS - // assert(0); + // ASSERT(0); #else sigset_t signal_mask; sigemptyset(&signal_mask); @@ -994,7 +994,7 @@ int32_t taosGetFqdn(char *fqdn) { #else printf("failed to get hostname, reason:%s\n", strerror(errno)); #endif - assert(0); + ASSERT(0); return -1; } @@ -1031,7 +1031,7 @@ void taosIgnSIGPIPE() { signal(SIGPIPE, SIG_IGN); } void taosSetMaskSIGPIPE() { #ifdef WINDOWS - // assert(0); + // ASSERT(0); #else sigset_t signal_mask; sigemptyset(&signal_mask); diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 5419da1c0ce49fe8e61cbf030ad540918835f465..b9f537ad34a15c89a003390c2173f70b227beb2b 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -112,7 +112,7 @@ int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) { } TdUcs4 *tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) { - assert(taosMemorySize(target_ucs4) >= len_ucs4 * sizeof(TdUcs4)); + ASSERT(taosMemorySize(target_ucs4) >= len_ucs4 * sizeof(TdUcs4)); return memcpy(target_ucs4, source_ucs4, len_ucs4 * sizeof(TdUcs4)); } @@ -217,7 +217,7 @@ void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type) { bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len) { #ifdef DISALLOW_NCHAR_WITHOUT_ICONV - printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n"); + printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n"); return -1; #else memset(ucs4, 0, ucs4_max_len); @@ -245,7 +245,7 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4 int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) { #ifdef DISALLOW_NCHAR_WITHOUT_ICONV - printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n"); + printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n"); return -1; #else @@ -263,7 +263,7 @@ int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) { } bool taosValidateEncodec(const char *encodec) { #ifdef DISALLOW_NCHAR_WITHOUT_ICONV - printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n"); + printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n"); return true; #else iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC); @@ -355,8 +355,8 @@ int64_t taosStr2Int64(const char *str, char **pEnd, int32_t radix) { if (errno == EINVAL) errno = 0; #endif #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); #endif return tmp; } @@ -367,8 +367,8 @@ uint64_t taosStr2UInt64(const char *str, char **pEnd, int32_t radix) { if (errno == EINVAL) errno = 0; #endif #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); #endif return tmp; } @@ -379,8 +379,8 @@ int32_t taosStr2Int32(const char *str, char **pEnd, int32_t radix) { if (errno == EINVAL) errno = 0; #endif #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); #endif return tmp; } @@ -391,8 +391,8 @@ uint32_t taosStr2UInt32(const char *str, char **pEnd, int32_t radix) { if (errno == EINVAL) errno = 0; #endif #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); #endif return tmp; } @@ -403,10 +403,10 @@ int16_t taosStr2Int16(const char *str, char **pEnd, int32_t radix) { if (errno == EINVAL) errno = 0; #endif #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); - assert(tmp >= SHRT_MIN); - assert(tmp <= SHRT_MAX); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); + ASSERT(tmp >= SHRT_MIN); + ASSERT(tmp <= SHRT_MAX); #endif return (int16_t)tmp; } @@ -417,9 +417,9 @@ uint16_t taosStr2UInt16(const char *str, char **pEnd, int32_t radix) { if (errno == EINVAL) errno = 0; #endif #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); - assert(tmp <= USHRT_MAX); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); + ASSERT(tmp <= USHRT_MAX); #endif return (uint16_t)tmp; } @@ -427,10 +427,10 @@ uint16_t taosStr2UInt16(const char *str, char **pEnd, int32_t radix) { int8_t taosStr2Int8(const char *str, char **pEnd, int32_t radix) { int32_t tmp = strtol(str, pEnd, radix); #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); - assert(tmp >= SCHAR_MIN); - assert(tmp <= SCHAR_MAX); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); + ASSERT(tmp >= SCHAR_MIN); + ASSERT(tmp <= SCHAR_MAX); #endif return tmp; } @@ -441,9 +441,9 @@ uint8_t taosStr2UInt8(const char *str, char **pEnd, int32_t radix) { if (errno == EINVAL) errno = 0; #endif #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); - assert(tmp <= UCHAR_MAX); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); + ASSERT(tmp <= UCHAR_MAX); #endif return tmp; } @@ -451,9 +451,9 @@ uint8_t taosStr2UInt8(const char *str, char **pEnd, int32_t radix) { double taosStr2Double(const char *str, char **pEnd) { double tmp = strtod(str, pEnd); #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); - assert(tmp != HUGE_VAL); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); + ASSERT(tmp != HUGE_VAL); #endif return tmp; } @@ -461,10 +461,10 @@ double taosStr2Double(const char *str, char **pEnd) { float taosStr2Float(const char *str, char **pEnd) { float tmp = strtof(str, pEnd); #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); - assert(tmp != HUGE_VALF); - assert(tmp != NAN); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); + ASSERT(tmp != HUGE_VALF); + ASSERT(tmp != NAN); #endif return tmp; } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 7521ae4e0fe6593eb26894dbf829da948a2382b6..35e76f0e67779189ecec915dee0231b5afe63bf4 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -249,7 +249,7 @@ void taosGetSystemInfo() { int32_t taosGetEmail(char *email, int32_t maxLen) { #ifdef WINDOWS - // assert(0); + // ASSERT(0); #elif defined(_TD_DARWIN_64) const char *filepath = "/usr/local/taos/email"; @@ -357,6 +357,10 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { code = 0; done |= 1; } + int endPos = strlen(cpuModel)-1; + if (cpuModel[endPos] == '\n') { + cpuModel[endPos] = '\0'; + } taosCloseCmd(&pCmd); pCmd = taosOpenCmd("sysctl -n machdep.cpu.core_count"); @@ -859,7 +863,7 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { char *taosGetCmdlineByPID(int pid) { #ifdef WINDOWS - assert(0); + ASSERT(0); return ""; #elif defined(_TD_DARWIN_64) static char cmdline[1024]; diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index c972aebbcaf8e89b7104e9c9241b5f7d11b6449d..7b6c77f7bc7ec969438689068a521ed20c123940 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -90,7 +90,7 @@ typedef struct FILE TdCmd; void* taosLoadDll(const char* filename) { #if defined(WINDOWS) - assert(0); + ASSERT(0); return NULL; #elif defined(_TD_DARWIN_64) return NULL; @@ -109,7 +109,7 @@ void* taosLoadDll(const char* filename) { void* taosLoadSym(void* handle, char* name) { #if defined(WINDOWS) - assert(0); + ASSERT(0); return NULL; #elif defined(_TD_DARWIN_64) return NULL; @@ -130,7 +130,7 @@ void* taosLoadSym(void* handle, char* name) { void taosCloseDll(void* handle) { #if defined(WINDOWS) - assert(0); + ASSERT(0); return; #elif defined(_TD_DARWIN_64) return; diff --git a/source/os/src/osThread.c b/source/os/src/osThread.c index 32c58695cfdac108e38112ccbfc73f35faf5ed42..39ba92fdc5e93d45facd85a2015461ad1fd68d8c 100644 --- a/source/os/src/osThread.c +++ b/source/os/src/osThread.c @@ -233,7 +233,8 @@ int32_t taosThreadSpinDestroy(TdThreadSpinlock *lock) { int32_t taosThreadSpinInit(TdThreadSpinlock *lock, int32_t pshared) { #ifdef TD_USE_SPINLOCK_AS_MUTEX - assert(pshared == 0); + ASSERT(pshared == 0); + if (pshared != 0) return -1; return pthread_mutex_init((pthread_mutex_t *)lock, NULL); #else return pthread_spin_init((pthread_spinlock_t *)lock, pshared); diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 4bd829442372305501d1d571bdec910b2182fcfe..6920925e5f82b16ba6d213ffbe03d6fa40d21d06 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -191,6 +191,8 @@ void* taosArrayReserve(SArray* pArray, int32_t num) { void* dst = TARRAY_GET_ELEM(pArray, pArray->size); pArray->size += num; + memset(dst, 0, num * pArray->elemSize); + return dst; } @@ -226,7 +228,7 @@ size_t taosArrayGetSize(const SArray* pArray) { if (pArray == NULL) { return 0; } - return pArray->size; + return TARRAY_SIZE(pArray); } void taosArraySetSize(SArray* pArray, size_t size) { @@ -295,6 +297,20 @@ void taosArrayRemove(SArray* pArray, size_t index) { pArray->size -= 1; } +void taosArrayRemoveBatch(SArray* pArray, size_t index, size_t num, FDelete fp) { + ASSERT(index + num <= pArray->size); + + if (fp) { + for (int32_t i = 0; i < num; i++) { + fp(taosArrayGet(pArray, index + i)); + } + } + + memmove((char*)pArray->pData + index * pArray->elemSize, (char*)pArray->pData + (index + num) * pArray->elemSize, + (pArray->size - index - num) * pArray->elemSize); + pArray->size -= num; +} + SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize) { assert(src != NULL && elemSize > 0); SArray* pDst = taosArrayInit(size, elemSize); diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 7a52991e817e1031de80c7ec5fd5b51c84c3015e..75d6a43b249fd56f8b6274b0f68db27627182d08 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -17,11 +17,11 @@ #define _XOPEN_SOURCE #define _DEFAULT_SOURCE #include "tcompare.h" -#include "tutil.h" #include "regex.h" #include "tdef.h" #include "thash.h" #include "tlog.h" +#include "tutil.h" #include "types.h" #include "osString.h" @@ -1010,7 +1010,8 @@ int32_t compareJsonValDesc(const void *pLeft, const void *pRight) { return compa * '_': Matches one character * */ -int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t ssize, const SPatternCompareInfo *pInfo) { +int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t ssize, + const SPatternCompareInfo *pInfo) { char c, c1; int32_t i = 0; @@ -1020,9 +1021,9 @@ int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t while ((i < psize) && ((c = pattern[i++]) != 0)) { if (c == pInfo->matchAll) { /* Match "*" */ - while ((i < psize) && ((c = pattern[i++]) == pInfo->matchAll || c == pInfo->matchOne)) { + while ((i < psize) && ((c = pattern[i++]) == pInfo->matchAll || c == pInfo->matchOne)) { if (c == pInfo->matchOne) { - if (j >= ssize || str[j++] == 0) { // empty string, return not match + if (j >= ssize || str[j++] == 0) { // empty string, return not match return TSDB_PATTERN_NOWILDCARDMATCH; } else { ++nMatchChar; @@ -1077,7 +1078,8 @@ int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t return (j >= ssize || str[j] == 0) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; } -int32_t wcsPatternMatch(const TdUcs4 *pattern, size_t psize, const TdUcs4 *str, size_t ssize, const SPatternCompareInfo *pInfo) { +int32_t wcsPatternMatch(const TdUcs4 *pattern, size_t psize, const TdUcs4 *str, size_t ssize, + const SPatternCompareInfo *pInfo) { TdUcs4 c, c1; int32_t i = 0; @@ -1189,10 +1191,11 @@ int32_t comparestrRegexMatch(const void *pLeft, const void *pRight) { taosMemoryFree(str); taosMemoryFree(pattern); - return (ret == 0) ? 0 : 1;; + return (ret == 0) ? 0 : 1; + ; } -int32_t comparewcsRegexMatch(const void* pString, const void* pPattern) { +int32_t comparewcsRegexMatch(const void *pString, const void *pPattern) { size_t len = varDataLen(pPattern); char *pattern = taosMemoryMalloc(len + 1); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 14c3df38a7583f9e38acf936cebce78d7d6f77aa..bab3edc870bb888bf2ad6030ff102c88daa05be6 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -137,6 +137,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CLAUSE_ERROR, "not supported stmt cl TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_KILLED, "Query killed") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_EXEC_NODE, "No available execution node in current query policy configuration") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NOT_STABLE_ERROR, "Table is not a super table") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CACHE_ERROR, "Stmt cache error") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INTERNAL_ERROR, "Internal error") // mnode-common TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS, "Insufficient privilege for operation") @@ -318,6 +320,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_SUBSCRIBED, "Table column is subsc TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_AVAIL_BUFPOOL, "No availabe buffer pool") TAOS_DEFINE_ERROR(TSDB_CODE_VND_STOPPED, "Vnode stopped") TAOS_DEFINE_ERROR(TSDB_CODE_VND_DUP_REQUEST, "Duplicate write request") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_QUERY_BUSY, "Query busy") // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") @@ -577,6 +580,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE, "Invalid timestamp p TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DATA, "Invalid data format") TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DB_CONF, "Invalid schemaless db config") TAOS_DEFINE_ERROR(TSDB_CODE_SML_NOT_SAME_TYPE, "Not the same type like before") +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INTERNAL_ERROR, "Internal error") //tsma TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_INIT_FAILED, "Tsma init failed") @@ -592,6 +596,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_NO_INDEX_IN_CACHE, "No tsma index in ca TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_ENV, "Invalid rsma env") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_STAT, "Invalid rsma state") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_QTASKINFO_CREATE, "Rsma qtaskinfo creation error") +TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FS_COMMIT, "Rsma fs commit error") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_REMOVE_EXISTS, "Rsma remove exists") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP, "Rsma fetch msg is messed up") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_EMPTY_INFO, "Rsma info is empty") @@ -599,6 +604,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_SCHEMA, "Rsma invalid schema TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_REGEX_MATCH, "Rsma regex match") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_STREAM_STATE_OPEN, "Rsma stream state open") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_STREAM_STATE_COMMIT, "Rsma stream state commit") +TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FS_REF, "Rsma fs ref error") +TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FS_SYNC, "Rsma fs sync error") +TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FS_UPDATE, "Rsma fs update error") //index TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING, "Index is rebuilding") diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 42b63588930712d2e132fd7e8bf860aef3622ff9..3769da6ccd60a2fe4a6a949ae9b66e8a5a3e1486 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -339,7 +339,6 @@ void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue) { STaosQueue *prev = qset->head; tqueue = qset->head->next; while (tqueue) { - assert(tqueue->qset); if (tqueue == queue) { prev->next = tqueue->next; break; diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index a9a84c186033f8b9574f3ef3f22fed10e6794207..5581931178486ba05f36640edcf7500477cbc96c 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -227,6 +227,7 @@ STaosQueue *tAutoQWorkerAllocQueue(SAutoQWorkerPool *pool, void *ahandle, FItem uError("worker:%s:%d failed to create", pool->name, curWorkerNum); taosMemoryFree(worker); taosCloseQueue(queue); + taosThreadMutexUnlock(&pool->mutex); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } diff --git a/tests/develop-test/2-query/table_count_scan.py b/tests/develop-test/2-query/table_count_scan.py new file mode 100644 index 0000000000000000000000000000000000000000..1ef65bfc67083719afbdbe499ab9a2d6ec3d3ead --- /dev/null +++ b/tests/develop-test/2-query/table_count_scan.py @@ -0,0 +1,238 @@ +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from math import inf + +class TDTestCase: + def caseDescription(self): + ''' + case1: [TD-21890] table count scan test case + ''' + return + + def init(self, conn, logSql, replicaVer=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), False) + self._conn = conn + + def restartTaosd(self, index=1, dbname="db"): + tdDnodes.stop(index) + tdDnodes.startWithoutSleep(index) + tdSql.execute(f"use tbl_count") + + def run(self): + print("running {}".format(__file__)) + tdSql.execute("drop database if exists tbl_count") + tdSql.execute("create database if not exists tbl_count") + tdSql.execute('use tbl_count') + tdSql.execute('create table stb1 (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(10), c9 nchar(10), c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned) TAGS(t1 int, t2 binary(10), t3 double);') + + tdSql.execute("create table tb1 using stb1 tags(1,'1',1.0);") + + tdSql.execute("create table tb2 using stb1 tags(2,'2',2.0);") + + tdSql.execute("create table tb3 using stb1 tags(3,'3',3.0);") + + tdSql.execute('insert into tb1 values (\'2021-11-11 09:00:00\',true,1,1,1,1,1,1,"123","1234",1,1,1,1);') + + tdSql.execute("insert into tb1 values ('2021-11-11 09:00:01',true,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);") + + tdSql.execute('insert into tb1 values (\'2021-11-11 09:00:02\',true,2,NULL,2,NULL,2,NULL,"234",NULL,2,NULL,2,NULL);') + + tdSql.execute('insert into tb1 values (\'2021-11-11 09:00:03\',false,NULL,3,NULL,3,NULL,3,NULL,"3456",NULL,3,NULL,3);') + + tdSql.execute('insert into tb1 values (\'2021-11-11 09:00:04\',true,4,4,4,4,4,4,"456","4567",4,4,4,4);') + + tdSql.execute('insert into tb1 values (\'2021-11-11 09:00:05\',true,127,32767,2147483647,9223372036854775807,3.402823466e+38,1.79769e+308,"567","5678",254,65534,4294967294,9223372036854775807);') + + tdSql.execute('insert into tb1 values (\'2021-11-11 09:00:06\',true,-127,-32767,-2147483647,-9223372036854775807,-3.402823466e+38,-1.79769e+308,"678","6789",0,0,0,0);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:00\',true,1,1,1,1,1,1,"111","1111",1,1,1,1);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:01\',true,2,2,2,2,2,2,"222","2222",2,2,2,2);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:02\',true,3,3,2,3,3,3,"333","3333",3,3,3,3);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:03\',false,4,4,4,4,4,4,"444","4444",4,4,4,4);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:04\',true,5,5,5,5,5,5,"555","5555",5,5,5,5);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:05\',true,6,6,6,6,6,6,"666","6666",6,6,6,6);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:06\',true,7,7,7,7,7,7,"777","7777",7,7,7,7);') + + tdSql.query('select count(*),db_name, stable_name from information_schema.ins_tables group by db_name, stable_name;') + tdSql.checkRows(3) + tdSql.checkData(0, 0, 23) + tdSql.checkData(0, 1, 'information_schema') + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 3) + tdSql.checkData(1, 1, 'tbl_count') + tdSql.checkData(1, 2, 'stb1') + tdSql.checkData(2, 0, 5) + tdSql.checkData(2, 1, 'performance_schema') + tdSql.checkData(2, 2, None) + + tdSql.query('select count(1),db_name, stable_name from information_schema.ins_tables group by db_name, stable_name;') + tdSql.checkRows(3) + tdSql.checkData(0, 0, 23) + tdSql.checkData(0, 1, 'information_schema') + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 5) + tdSql.checkData(1, 1, 'performance_schema') + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 0, 3) + tdSql.checkData(2, 1, 'tbl_count') + tdSql.checkData(2, 2, 'stb1') + + tdSql.query('select count(1),db_name from information_schema.ins_tables group by db_name') + tdSql.checkRows(3) + tdSql.checkData(0, 0, 5) + tdSql.checkData(0, 1, 'performance_schema') + tdSql.checkData(1, 0, 3) + tdSql.checkData(1, 1, 'tbl_count') + tdSql.checkData(2, 0, 23) + tdSql.checkData(2, 1, 'information_schema') + + tdSql.query("select count(*) from information_schema.ins_tables where db_name='tbl_count'") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 3) + + tdSql.query('select count(*) from information_schema.ins_tables where db_name=\'tbl_count\' and stable_name="stb1";') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 3) + + tdSql.query('select count(*) from information_schema.ins_tables') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 31) + + + tdSql.execute('create table stba (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(10), c9 nchar(10), c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned) TAGS(t1 int, t2 binary(10), t3 double);') + + tdSql.execute("create table tba1 using stba tags(1,'1',1.0);") + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:00\',true, 1,1,1,1,1,1,"111","1111",1,1,1,1);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:01\',true, 2,2,2,2,2,2,"222","2222",2,2,2,2);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:02\',true, 3,3,2,3,3,3,"333","3333",3,3,3,3);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:03\',false,4,4,4,4,4,4,"444","4444",4,4,4,4);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:04\',true, 5,5,5,5,5,5,"555","5555",5,5,5,5);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:05\',true, 6,6,6,6,6,6,"666","6666",6,6,6,6);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:06\',true, 7,7,7,7,7,7,"777","7777",7,7,7,7);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:07\',true, 8,8,8,8,8,8,"888","8888",8,8,8,8);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:08\',true, 9,9,9,9,9,9,"999","9999",9,9,9,9);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:09\',true, 0,0,0,0,0,0,"000","0000",0,0,0,0);') + + self.restartTaosd(1, dbname='tbl_count') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:10\',true, 1,1,1,1,1,1,"111","1111",1,1,1,1);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:11\',true, 2,2,2,2,2,2,"222","2222",2,2,2,2);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:12\',true, 3,3,2,3,3,3,"333","3333",3,3,3,3);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:13\',false,4,4,4,4,4,4,"444","4444",4,4,4,4);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:14\',true, 5,5,5,5,5,5,"555","5555",5,5,5,5);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:15\',true, 6,6,6,6,6,6,"666","6666",6,6,6,6);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:16\',true, 7,7,7,7,7,7,"777","7777",7,7,7,7);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:17\',true, 8,8,8,8,8,8,"888","8888",8,8,8,8);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:18\',true, 9,9,9,9,9,9,"999","9999",9,9,9,9);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:19\',true, 0,0,0,0,0,0,"000","0000",0,0,0,0);') + + self.restartTaosd(1, dbname='tbl_count') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:20\',true, 1,1,1,1,1,1,"111","1111",1,1,1,1);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:21\',true, 2,2,2,2,2,2,"222","2222",2,2,2,2);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:22\',true, 3,3,2,3,3,3,"333","3333",3,3,3,3);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:23\',false,4,4,4,4,4,4,"444","4444",4,4,4,4);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:24\',true, 5,5,5,5,5,5,"555","5555",5,5,5,5);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:25\',true, 6,6,6,6,6,6,"666","6666",6,6,6,6);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:26\',true, 7,7,7,7,7,7,"777","7777",7,7,7,7);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:27\',true, 8,8,8,8,8,8,"888","8888",8,8,8,8);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:28\',true, 9,9,9,9,9,9,"999","9999",9,9,9,9);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:29\',true, 0,0,0,0,0,0,"000","0000",0,0,0,0);') + + tdSql.query('select count(*),db_name, stable_name from information_schema.ins_tables group by db_name, stable_name;') + tdSql.checkRows(4) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 'tbl_count') + tdSql.checkData(0, 2, 'stba') + tdSql.checkData(1, 0, 23) + tdSql.checkData(1, 1, 'information_schema') + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 0, 3) + tdSql.checkData(2, 1, 'tbl_count') + tdSql.checkData(2, 2, 'stb1') + tdSql.checkData(3, 0, 5) + tdSql.checkData(3, 1, 'performance_schema') + tdSql.checkData(3, 2, None) + + tdSql.query('select count(1),db_name, stable_name from information_schema.ins_tables group by db_name, stable_name;') + tdSql.checkRows(4) + tdSql.checkData(0, 0, 23) + tdSql.checkData(0, 1, 'information_schema') + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 5) + tdSql.checkData(1, 1, 'performance_schema') + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 0, 1) + tdSql.checkData(2, 1, 'tbl_count') + tdSql.checkData(2, 2, 'stba') + tdSql.checkData(3, 0, 3) + tdSql.checkData(3, 1, 'tbl_count') + tdSql.checkData(3, 2, 'stb1') + + tdSql.query('select count(1),db_name from information_schema.ins_tables group by db_name') + tdSql.checkRows(3) + tdSql.checkData(0, 0, 5) + tdSql.checkData(0, 1, 'performance_schema') + tdSql.checkData(1, 0, 4) + tdSql.checkData(1, 1, 'tbl_count') + tdSql.checkData(2, 0, 23) + tdSql.checkData(2, 1, 'information_schema') + + tdSql.query("select count(*) from information_schema.ins_tables where db_name='tbl_count'") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 4) + + tdSql.query('select count(*) from information_schema.ins_tables where db_name=\'tbl_count\' and stable_name="stb1";') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 3) + + tdSql.query('select count(*) from information_schema.ins_tables') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 32) + + + tdSql.execute('drop database tbl_count') + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py index 080d6aea8731562c1aaa4a6aacb6aca726059b1a..54657995e2b0abcfa80b1b08be0b40b9bf415126 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py @@ -320,7 +320,7 @@ class TDTestCase: tdSql.checkData(0, 0, 160) tdSql.query("select count(*) from db.stb where t13 like 'b1%' or t13 like 'b2%'") tdSql.checkData(0, 0, 160) - + tAdapter.stop() def stop(self): diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/query_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/query_json.py index c4b1750340eb73016e0fa623bdc484a0d33788a4..c906f6167526c707f1a065b32dfd65fe5bdc7eac 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/query_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/query_json.py @@ -120,7 +120,7 @@ class TDTestCase: def stop(self): - tdSql.close() + tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index c8a42e11b8d07789f1019b15d1057bddef907678..4055bf0168376669392da0868b7de9fb90ed90e2 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -82,6 +82,7 @@ ,,y,script,./test.sh -f tsim/insert/tcp.sim ,,y,script,./test.sh -f tsim/insert/update0.sim ,,y,script,./test.sh -f tsim/insert/update1_sort_merge.sim +,,y,script,./test.sh -f tsim/insert/update2.sim ,,y,script,./test.sh -f tsim/parser/alter__for_community_version.sim ,,y,script,./test.sh -f tsim/parser/alter_column.sim ,,y,script,./test.sh -f tsim/parser/alter_stable.sim @@ -246,6 +247,8 @@ ,,y,script,./test.sh -f tsim/stream/fillIntervalPartitionBy.sim ,,y,script,./test.sh -f tsim/stream/fillIntervalPrevNext.sim ,,y,script,./test.sh -f tsim/stream/fillIntervalValue.sim +,,y,script,./test.sh -f tsim/stream/udTableAndTag0.sim +,,y,script,./test.sh -f tsim/stream/udTableAndTag1.sim ,,y,script,./test.sh -f tsim/trans/lossdata1.sim ,,y,script,./test.sh -f tsim/trans/create_db.sim ,,y,script,./test.sh -f tsim/tmq/basic1.sim @@ -657,6 +660,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 -n 3 + +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py -N 6 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeModifyMeta.py -N 6 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3 @@ -667,9 +673,10 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3 + ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 +#,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -N 5 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 @@ -734,7 +741,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-19201.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py @@ -1050,6 +1057,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py #develop test +,,n,develop-test,python3 ./test.py -f 2-query/table_count_scan.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py diff --git a/tests/pytest/util/cluster.py b/tests/pytest/util/cluster.py index 72b7e1fddff7fcfce3c9fb83eeb74f2cefc01f18..2607cf63c29488b05a998fafd1565f29917d8e5e 100644 --- a/tests/pytest/util/cluster.py +++ b/tests/pytest/util/cluster.py @@ -53,7 +53,7 @@ class ConfigureyCluster: # configure dnoe of independent mnodes if num <= self.mnodeNums and self.mnodeNums != 0 and independentMnode == True : - dnode.addExtraCfg("supportVnodes", 0) + dnode.addExtraCfg("supportVnodes", 1024) # print(dnode) self.dnodes.append(dnode) return self.dnodes diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 9cfd1d368edfa319aa4e80b307cbcc09020245ff..bdf3f20e1546e029dfbf7b41b280e857fb732020 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -261,6 +261,70 @@ class TDSql: tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + # return true or false replace exit, no print out + def checkRowColNoExit(self, row, col): + caller = inspect.getframeinfo(inspect.stack()[2][0]) + if row < 0: + args = (caller.filename, caller.lineno, self.sql, row) + return False + if col < 0: + args = (caller.filename, caller.lineno, self.sql, row) + return False + if row > self.queryRows: + args = (caller.filename, caller.lineno, self.sql, row, self.queryRows) + return False + if col > self.queryCols: + args = (caller.filename, caller.lineno, self.sql, col, self.queryCols) + return False + + return True + + + # return true or false replace exit, no print out + def checkDataNoExit(self, row, col, data): + if self.checkRowColNoExit(row, col) == False: + return False + if self.queryResult[row][col] != data: + if self.cursor.istype(col, "TIMESTAMP"): + # suppose user want to check nanosecond timestamp if a longer data passed + if (len(data) >= 28): + if pd.to_datetime(self.queryResult[row][col]) == pd.to_datetime(data): + return True + else: + if self.queryResult[row][col] == _parse_datetime(data): + return True + return False + + if str(self.queryResult[row][col]) == str(data): + return True + elif isinstance(data, float): + if abs(data) >= 1 and abs((self.queryResult[row][col] - data) / data) <= 0.000001: + return True + elif abs(data) < 1 and abs(self.queryResult[row][col] - data) <= 0.000001: + return True + else: + return False + else: + return False + + return True + + + # loop execute sql then sleep(waitTime) , if checkData ok break loop + def checkDataLoop(self, row, col, data, sql, loopCount, waitTime): + # loop check util checkData return true + for i in range(loopCount): + self.query(sql) + if self.checkDataNoExit(row, col, data) : + self.checkData(row, col, data) + return + time.sleep(waitTime) + + # last check + self.query(sql) + self.checkData(row, col, data) + + def getData(self, row, col): self.checkRowCol(row, col) return self.queryResult[row][col] diff --git a/tests/pytest/util/taosadapter.py b/tests/pytest/util/taosadapter.py index 7dd62266319bd1462ce7bad6ff6ec358cfe31678..79ea86c5bd820c1327cf844b35eb8800fefdfe60 100644 --- a/tests/pytest/util/taosadapter.py +++ b/tests/pytest/util/taosadapter.py @@ -238,14 +238,13 @@ class TAdapter: if self.running != 0: psCmd = f"ps -ef|grep -w {toBeKilled}| grep -v grep | awk '{{print $2}}'" + # psCmd = f"pgrep {toBeKilled}" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() - # psCmd = f"pgrep {toBeKilled}" - - while(processID): - killCmd = "kill %s %s > /dev/null 2>&1" % (signal, processID) + while(processID): + killCmd = "kill %s %s > /dev/null 2>&1" % (signal, processID) os.system(killCmd) time.sleep(1) - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() if not platform.system().lower() == 'windows': port = 6041 fuserCmd = f"fuser -k -n tcp {port} > /dev/null" diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 60df188d7b84c2f202b6ec11665013da143393e9..88dada44accf588357314843d2a7d09964c896c0 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -232,8 +232,8 @@ CaseCtrl gCaseCtrl = { .printCreateTblSql = true, .printQuerySql = true, .printStmtSql = true, - .printVerbose = false, - .printRes = false, + .printVerbose = true, + .printRes = true, .autoCreateTbl = false, .numericParam = false, .rowNum = 0, diff --git a/tests/script/coverage_test.sh b/tests/script/coverage_test.sh index e0e7490d3ab7616cfc44ceab7c4c951b21663004..3f99dde5442c5f0c9e88b5ea1df37f7a64e54003 100755 --- a/tests/script/coverage_test.sh +++ b/tests/script/coverage_test.sh @@ -130,6 +130,9 @@ function runSimCases() { function runPythonCases() { echo "=== Run python cases ===" + + cd $TDENGINE_DIR/tests/parallel_test + sed -i '/compatibility.py/d' cases.task cd $TDENGINE_DIR/tests/system-test runCasesOneByOne ../parallel_test/cases.task system-test diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 7df17b22da1c8a8c3b7cf95315a3aea417d56eae..72257227916785de3d174349d4c83d3302064295 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -39,7 +39,7 @@ python_error=`cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l` # /root/TDengine/source/libs/scalar/src/sclvector.c:1075:66: runtime error: signed integer overflow: 9223372034707292160 + 1668838476672 cannot be represented in type 'long int' # /root/TDengine/source/common/src/tdataformat.c:1876:7: runtime error: signed integer overflow: 8252423483843671206 + 2406154664059062870 cannot be represented in type 'long int' -runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type"| grep -v "signed integer overflow" | wc -l` +runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type"| grep -v "signed integer overflow" |grep -v "strerror.c"| grep -v "asan_malloc_linux.cc" |wc -l` echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m" echo -e "\033[44;32;1m"asan memory_leak: $memory_leak"\033[0m" diff --git a/tests/script/tsim/insert/update2.sim b/tests/script/tsim/insert/update2.sim new file mode 100644 index 0000000000000000000000000000000000000000..d2b5b052678a93499157dea219571df6ed67c6b7 --- /dev/null +++ b/tests/script/tsim/insert/update2.sim @@ -0,0 +1,221 @@ +################################################################################################ +# migrate from 2.0 insert_update2.sim +################################################################################################ + +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/exec.sh -n dnode1 -s start + +sleep 2000 +sql connect + +### 4096*0.8 - 1 = 3275 +$rowSuperBlk = 3275 +### 4096 - 3275 -1 - 1 = 819 +$rowSubBlk = 819 +$ts0 = 1672372800000 +$ts1 = 1672372900000 +$ts2 = 1672686800000 + +$i = 0 +$db = db0 +$stb1 = stb1 +$tb1 = tb1 +$stb2 = stb2 +$tb2 = tb2 + +print ====== create database +sql drop database if exists $db +sql create database $db keep 1000 duration 10 +print ====== create tables +sql use $db +sql create table $stb1 (ts timestamp, c1 bigint, c2 bigint, c3 bigint) tags(t1 int) +sql create table $stb2 (ts timestamp, c1 bigint, c2 bigint, c3 bigint, c4 bigint, c5 bigint, c6 bigint, c7 bigint, c8 bigint, c9 bigint, c10 bigint, c11 bigint, c12 bigint, c13 bigint, c14 bigint, c15 bigint, c16 bigint, c17 bigint, c18 bigint, c19 bigint, c20 bigint, c21 bigint, c22 bigint, c23 bigint, c24 bigint, c25 bigint, c26 bigint, c27 bigint, c28 bigint, c29 bigint, c30 bigint) tags(t1 int) +sql create table $tb1 using $stb1 tags(1) +sql create table $tb2 using $stb2 tags(2) +print ====== tables created + + +print ========== step 1: merge dataRow in mem + +$i = 0 +while $i < $rowSuperBlk + $xs = $i * 10 + $ts = $ts2 + $xs + sql insert into $tb1 (ts,c1) values ( $ts , $i ) + $i = $i + 1 +endw + +sql insert into $tb1 values ( $ts0 , 1,NULL,0) +sql insert into $tb1 (ts,c2,c3) values ( $ts0 , 1,1) + +sql select * from $tb1 where ts = $ts0 +if $rows != 1 then + return -1 +endi +if $data01 != 1 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data03 != 1 then + return -1 +endi + +print ========== step 2: merge kvRow in mem +$i = 0 +while $i < $rowSuperBlk + $xs = $i * 10 + $ts = $ts2 + $xs + sql insert into $tb2 (ts,c1) values ( $ts , $i ) + $i = $i + 1 +endw + +sql insert into $tb2 (ts,c3,c8,c10) values ( $ts0 , 1,NULL,0) +sql insert into $tb2 (ts,c8,c10) values ( $ts0 , 1,1) + +sql select ts,c1,c3,c8,c10 from $tb2 where ts = $ts0 +if $rows != 1 then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data03 != 1 then + return -1 +endi +if $data04 != 1 then + return -1 +endi + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 2000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sleep 2000 + +print ========== step 3: merge dataRow in file +sql insert into $tb1 (ts,c1) values ( $ts0 , 2) +print ========== step 4: merge kvRow in file +sql insert into $tb2 (ts,c3) values ( $ts0 , 2) + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 2000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sleep 2000 + +sql select * from $tb1 where ts = $ts0 +if $rows != 1 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data03 != 1 then + return -1 +endi + +sql select ts,c1,c3,c8,c10 from $tb2 where ts = $ts0 +if $rows != 1 then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data02 != 2 then + return -1 +endi +if $data03 != 1 then + return -1 +endi +if $data04 != 1 then + return -1 +endi + + +print ========== step 5: merge dataRow in file/mem +$i = 0 +while $i < $rowSubBlk + $xs = $i * 1 + $ts = $ts1 + $xs + sql insert into $tb1 (ts,c1) values ( $ts , $i ) + $i = $i + 1 +endw +print ========== step 6: merge kvRow in file/mem +$i = 0 +while $i < $rowSubBlk + $xs = $i * 1 + $ts = $ts1 + $xs + sql insert into $tb2 (ts,c1) values ( $ts , $i ) + $i = $i + 1 +endw + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 2000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sleep 2000 + +sql insert into $tb1 (ts,c3) values ( $ts0 , 3) +sql insert into $tb2 (ts,c3) values ( $ts0 , 3) +$tsN = $ts0 + 1 +sql insert into $tb1 (ts,c1,c3) values ( $tsN , 1,0) +sql insert into $tb2 (ts,c3,c8) values ( $tsN , 100,200) +$tsN = $ts0 + 2 +sql insert into $tb1 (ts,c1,c3) values ( $tsN , 1,0) +sql insert into $tb2 (ts,c3,c8) values ( $tsN , 100,200) + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 2000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sleep 2000 + + +sql select * from $tb1 where ts = $ts0 +if $rows != 1 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data03 != 3 then + return -1 +endi + +sql select ts,c1,c3,c8,c10 from $tb2 where ts = $ts0 +if $rows != 1 then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data02 != 3 then + return -1 +endi +if $data03 != 1 then + return -1 +endi + +if $data04 != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/stream/checkStreamSTable.sim b/tests/script/tsim/stream/checkStreamSTable.sim new file mode 100644 index 0000000000000000000000000000000000000000..2ed6958196e7defc114378f06f889d5b00b032a1 --- /dev/null +++ b/tests/script/tsim/stream/checkStreamSTable.sim @@ -0,0 +1,310 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 + +print ===== step1 + +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print ===== step2 + +sql create database result vgroups 1; + +sql create database test vgroups 4; +sql use test; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stable result.streamt0(ts timestamp,a int,b int) tags(ta int,tb int,tc int); + +sql create stream streams0 trigger at_once into result.streamt0 as select _wstart, count(*) c1, max(a) c2 from st partition by tbname interval(10s); +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,2,2,3); + +$loop_count = 0 + +sql select _wstart, count(*) c1, max(a) c2 from st partition by tbname interval(10s); +print $data00, $data01, $data02 +print $data10, $data11, $data12 +print $data20, $data21, $data22 + +loop0: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result.streamt0 order by ta; + +if $rows != 2 then + print =====rows=$rows + print $data00, $data01, $data02 + print $data10, $data11, $data12 + print $data20, $data21, $data22 + goto loop0 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop0 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop0 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop0 +endi + +if $data12 != 2 then + print =====data12=$data12 + goto loop0 +endi + +print ===== step3 + +sql create database result1 vgroups 1; + +sql create database test1 vgroups 4; +sql use test1; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stable result1.streamt1(ts timestamp,a int,b int,c int) tags(ta bigint unsigned,tb int,tc int); + +sql create stream streams1 trigger at_once into result1.streamt1(ts,c,a,b) as select _wstart, count(*) c1, max(a),min(b) c2 from st partition by tbname interval(10s); +sql insert into t1 values(1648791213000,10,20,30); +sql insert into t2 values(1648791213000,40,50,60); + +$loop_count = 0 + +sql select _wstart, count(*) c1, max(a),min(b) c2 from st partition by tbname interval(10s); +print $data00, $data01, $data02, $data03 +print $data10, $data11, $data12, $data13 +print $data20, $data21, $data22, $data23 + +loop1: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result1.streamt1 order by ta; + +if $rows != 2 then + print =====rows=$rows + print $data00, $data01, $data02, $data03 + print $data10, $data11, $data12, $data13 + print $data20, $data21, $data22, $data23 + goto loop1 +endi + +if $data01 != 10 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != 20 then + print =====data02=$data02 + goto loop1 +endi + +if $data03 != 1 then + print =====data03=$data03 + goto loop1 +endi + +if $data11 != 40 then + print =====data11=$data11 + goto loop1 +endi + +if $data12 != 50 then + print =====data12=$data12 + goto loop1 +endi + +if $data13 != 1 then + print =====data13=$data13 + goto loop1 +endi + + +print ===== step4 + +sql create database result2 vgroups 1; + +sql create database test2 vgroups 4; +sql use test2; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stable result2.streamt2(ts timestamp, a int , b int) tags(ta varchar(20)); + +# tag dest 1, source 2 +##sql_error create stream streams2 trigger at_once into result2.streamt2 TAGS(aa varchar(100), ta int) as select _wstart, count(*) c1, max(a) from st partition by tbname as aa, ta interval(10s); + +# column dest 3, source 4 +sql_error create stream streams2 trigger at_once into result2.streamt2 as select _wstart, count(*) c1, max(a), max(b) from st partition by tbname interval(10s); + +# column dest 3, source 4 +sql_error create stream streams2 trigger at_once into result2.streamt2(ts, a, b) as select _wstart, count(*) c1, max(a), max(b) from st partition by tbname interval(10s); + +# column dest 3, source 2 +sql_error create stream streams2 trigger at_once into result2.streamt2 as select _wstart, count(*) c1 from st partition by tbname interval(10s); + +# column dest 3, source 2 +sql create stream streams2 trigger at_once into result2.streamt2(ts, a) as select _wstart, count(*) c1 from st partition by tbname interval(10s); + + +print ===== step5 + +sql create database result3 vgroups 1; + +sql create database test3 vgroups 4; +sql use test3; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,2,3); +sql create table t2 using st tags(4,5,6); + +sql create stable result3.streamt3(ts timestamp,a int,b int,c int, d int) tags(ta int,tb int,tc int); + +sql create stream streams3 trigger at_once into result3.streamt3(ts,c,a,b) as select _wstart, count(*) c1, max(a),min(b) c2 from st interval(10s); + +sql insert into t1 values(1648791213000,10,20,30); +sql insert into t2 values(1648791213000,40,50,60); + +$loop_count = 0 + +sql select _wstart, count(*) c1, max(a),min(b) c2 from st interval(10s); +print $data00, $data01, $data02, $data03, $data04 +print $data10, $data11, $data12, $data13, $data14 +print $data20, $data21, $data22, $data23, $data24 + +loop2: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result3.streamt3; + +if $rows != 1 then + print =====rows=$rows + print $data00, $data01, $data02, $data03 + print $data10, $data11, $data12, $data13 + print $data20, $data21, $data22, $data23 + goto loop2 +endi + +if $data01 != 40 then + print =====data01=$data01 + goto loop2 +endi + +if $data02 != 20 then + print =====data02=$data02 + goto loop2 +endi + +if $data03 != 2 then + print =====data03=$data03 + goto loop2 +endi + +if $data04 != NULL then + print =====data04=$data04 + goto loop2 +endi + +print ===== step6 + +sql create database result4 vgroups 1; + +sql create database test4 vgroups 4; +sql use test4; + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,2,3); +sql create table t2 using st tags(4,5,6); + +sql create stable result4.streamt4(ts timestamp,a int,b int,c int, d int) tags(ta int,tb int,tc int); + +sql create stream streams4 trigger at_once into result4.streamt4(ts,c,a,b) tags(tg2 int, tg3 varchar(100), tg1 bigint) subtable(concat("tbl-", tg1)) as select _wstart, count(*) c1, max(a),min(b) c2 from st partition by ta+1 as tg1, cast(tb as bigint) as tg2, tc as tg3 interval(10s); + +sql insert into t1 values(1648791213000,10,20,30); +sql insert into t2 values(1648791213000,40,50,60); + +$loop_count = 0 + +sql select _wstart, count(*) c1, max(a),min(b) c2 from st interval(10s); +print $data00, $data01, $data02, $data03 +print $data10, $data11, $data12, $data13 +print $data20, $data21, $data22, $data23 + +loop2: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result4.streamt4; + +if $rows != 2 then + print =====rows=$rows + print $data00, $data01, $data02, $data03 + print $data10, $data11, $data12, $data13 + print $data20, $data21, $data22, $data23 + goto loop2 +endi + +if $data01 != 40 then + print =====data01=$data01 + goto loop2 +endi + +if $data02 != 20 then + print =====data02=$data02 + goto loop2 +endi + +if $data03 != 2 then + print =====data03=$data03 + goto loop2 +endi + +if $data04 != NULL then + print =====data04=$data04 + goto loop2 +endi + +print ======over + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/state0.sim b/tests/script/tsim/stream/state0.sim index 7c922658c91cd3c6529b981208b31754226db71d..ea7528b1cb3d2ca33e805bb0e516168a26d8ef59 100644 --- a/tests/script/tsim/stream/state0.sim +++ b/tests/script/tsim/stream/state0.sim @@ -552,7 +552,7 @@ sql use test4; sql create table st (ts timestamp, c1 tinyint, c2 smallint) tags (t1 tinyint) ; sql create table t1 using st tags (-81) ; sql create table t2 using st tags (-81) ; -sql create stream if not exists streams4 trigger window_close into streamt4 as select _wstart AS start, min(c1),count(c1) from t1 state_window(c1); +sql create stream if not exists streams4 trigger window_close into streamt4 as select _wstart AS startts, min(c1),count(c1) from t1 state_window(c1); sql insert into t1 (ts, c1) values (1668073288209, 11); sql insert into t1 (ts, c1) values (1668073288210, 11); @@ -567,7 +567,7 @@ loop7: sleep 200 -sql select * from streamt4 order by start; +sql select * from streamt4 order by startts; $loop_count = $loop_count + 1 if $loop_count == 20 then @@ -606,7 +606,7 @@ loop8: sleep 200 -sql select * from streamt4 order by start; +sql select * from streamt4 order by startts; $loop_count = $loop_count + 1 if $loop_count == 20 then @@ -640,7 +640,7 @@ loop8: sleep 200 -sql select * from streamt4 order by start; +sql select * from streamt4 order by startts; $loop_count = $loop_count + 1 if $loop_count == 20 then @@ -679,7 +679,7 @@ loop9: sleep 200 -sql select * from streamt4 order by start; +sql select * from streamt4 order by startts; $loop_count = $loop_count + 1 if $loop_count == 20 then diff --git a/tests/script/tsim/stream/udTableAndTag0.sim b/tests/script/tsim/stream/udTableAndTag0.sim new file mode 100644 index 0000000000000000000000000000000000000000..86feca19188b6fa1034830755de809cef9944490 --- /dev/null +++ b/tests/script/tsim/stream/udTableAndTag0.sim @@ -0,0 +1,372 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 + +print ===== step1 + +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print ===== step2 +print ===== table name + +sql create database result vgroups 1; + +sql create database test vgroups 4; +sql use test; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql_error create stream streams1 trigger at_once into result.streamt SUBTABLE("aaa") as select _wstart, count(*) c1 from st interval(10s); +sql create stream streams1 trigger at_once into result.streamt SUBTABLE(concat("aaa-", tbname)) as select _wstart, count(*) c1 from st partition by tbname interval(10s); +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,1,2,3); + +$loop_count = 0 +loop0: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select table_name from information_schema.ins_tables where db_name="result" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop0 +endi + +if $data00 != aaa-t1 then + print =====data00=$data00 + goto loop0 +endi + +if $data10 != aaa-t2 then + print =====data10=$data10 + goto loop0 +endi + +$loop_count = 0 +loop1: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result.streamt; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop1 +endi + + +print ===== step3 +print ===== tag name + +sql create database result2 vgroups 1; + +sql create database test2 vgroups 4; +sql use test2; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams2 trigger at_once into result2.streamt2 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as cc interval(10s); +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,1,2,3); + + +$loop_count = 0 +loop2: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select tag_name from information_schema.ins_tags where db_name="result2" and stable_name = "streamt2" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop2 +endi + +if $data00 != cc then + print data00 != cc + print =====data00=$data00 + goto loop2 +endi + +if $data10 != cc then + print =====data10=$data10 + goto loop2 +endi + +sql select cc from result2.streamt2 order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop2 +endi + +if $data00 != tag-t1 then + print data00 != tag-t1 + print =====data00=$data00 + goto loop2 +endi + +if $data10 != tag-t2 then + print =====data10=$data10 + goto loop2 +endi + +$loop_count = 0 +loop3: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result2.streamt2; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop3 +endi + + +print ===== step4 +print ===== tag name + table name + +sql create database result3 vgroups 1; + +sql create database test3 vgroups 4; +sql use test3; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams3 trigger at_once into result3.streamt3 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", tbname)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as dd, tbname interval(10s); +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,1,2,3); + + +$loop_count = 0 +loop4: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select tag_name from information_schema.ins_tags where db_name="result3" and stable_name = "streamt3" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop4 +endi + +if $data00 != dd then + print =====data00=$data00 + goto loop4 +endi + +if $data10 != dd then + print =====data10=$data10 + goto loop4 +endi + +sql select dd from result3.streamt3 order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop4 +endi + +if $data00 != tag-t1 then + print =====data00=$data00 + goto loop4 +endi + +if $data10 != tag-t2 then + print =====data10=$data10 + goto loop4 +endi + +$loop_count = 0 +loop5: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result3.streamt3; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop5 +endi + +$loop_count = 0 +loop6: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select table_name from information_schema.ins_tables where db_name="result3" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop6 +endi + +if $data00 != tbn-t1 then + print =====data00=$data00 + goto loop6 +endi + +if $data10 != tbn-t2 then + print =====data10=$data10 + goto loop6 +endi + + +print ===== step5 +print ===== tag name + table name + +sql create database result4 vgroups 1; + +sql create database test4 vgroups 4; +sql use test4; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create table t3 using st tags(3,3,3); + +sql create stream streams4 trigger at_once into result4.streamt4 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", tbname)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as dd, tbname interval(10s); +sql insert into t1 values(1648791213000,1,1,1) t2 values(1648791213000,2,2,2) t3 values(1648791213000,3,3,3); + + +$loop_count = 0 +loop7: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select table_name from information_schema.ins_tables where db_name="result4" order by 1; + +if $rows != 3 then + print =====rows=$rows + print $data00 $data10 + goto loop7 +endi + +if $data00 != tbn-t1 then + print =====data00=$data00 + goto loop7 +endi + +if $data10 != tbn-t2 then + print =====data10=$data10 + goto loop7 +endi + +if $data20 != tbn-t3 then + print =====data20=$data20 + goto loop7 +endi + +$loop_count = 0 +loop8: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result4.streamt4 order by 3; + +if $rows != 3 then + print =====rows=$rows + print $data00 $data10 + goto loop8 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop8 +endi + +if $data02 != tag-t1 then + print =====data02=$data02 + goto loop8 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop8 +endi + +if $data12 != tag-t2 then + print =====data12=$data12 + goto loop8 +endi + +if $data21 != 1 then + print =====data21=$data21 + goto loop8 +endi + +if $data22 != tag-t3 then + print =====data22=$data22 + goto loop8 +endi + +print ======over + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/udTableAndTag1.sim b/tests/script/tsim/stream/udTableAndTag1.sim new file mode 100644 index 0000000000000000000000000000000000000000..a0393a03cdd6930ad767a797f4275890ed160fb4 --- /dev/null +++ b/tests/script/tsim/stream/udTableAndTag1.sim @@ -0,0 +1,373 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 + +print ===== step1 + +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print ===== step2 +print ===== table name + +sql create database result vgroups 1; + +sql create database test vgroups 4; +sql use test; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql_error create stream streams1 trigger at_once into result.streamt SUBTABLE("aaa") as select _wstart, count(*) c1 from st interval(10s); +sql create stream streams1 trigger at_once into result.streamt SUBTABLE( concat("aaa-", cast(a as varchar(10) ) ) ) as select _wstart, count(*) c1 from st partition by a interval(10s); +print ===== insert into 1 +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,2,2,3); + +$loop_count = 0 +loop0: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select table_name from information_schema.ins_tables where db_name="result" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + print $data20 $data30 + goto loop0 +endi + +if $data00 != aaa-1 then + print =====data00=$data00 + goto loop0 +endi + +if $data10 != aaa-2 then + print =====data10=$data10 + goto loop0 +endi + +$loop_count = 0 +loop1: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result.streamt; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop1 +endi + + +print ===== step3 +print ===== column name + +sql create database result2 vgroups 1; + +sql create database test2 vgroups 4; +sql use test2; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams2 trigger at_once into result2.streamt2 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st partition by concat("col-", cast(a as varchar(10) ) ) as cc interval(10s); +print ===== insert into 2 +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,2,2,3); + + +$loop_count = 0 +loop2: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select tag_name from information_schema.ins_tags where db_name="result2" and stable_name = "streamt2" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop2 +endi + +if $data00 != cc then + print =====data00=$data00 + goto loop2 +endi + +if $data10 != cc then + print =====data10=$data10 + goto loop2 +endi + +sql select cc from result2.streamt2 order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop2 +endi + +if $data00 != col-1 then + print =====data00=$data00 + goto loop2 +endi + +if $data10 != col-2 then + print =====data10=$data10 + goto loop2 +endi + +$loop_count = 0 +loop3: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result2.streamt2; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop3 +endi + + +print ===== step4 +print ===== column name + table name + +sql create database result3 vgroups 1; + +sql create database test3 vgroups 4; +sql use test3; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams3 trigger at_once into result3.streamt3 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", cast(a as varchar(10) ) ) ) as select _wstart, count(*) c1 from st partition by concat("col-", cast(a as varchar(10) ) ) as dd, a interval(10s); +print ===== insert into 3 +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,2,2,3); + + +$loop_count = 0 +loop4: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select tag_name from information_schema.ins_tags where db_name="result3" and stable_name = "streamt3" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop4 +endi + +if $data00 != dd then + print =====data00=$data00 + goto loop4 +endi + +if $data10 != dd then + print =====data10=$data10 + goto loop4 +endi + +sql select dd from result3.streamt3 order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop4 +endi + +if $data00 != col-1 then + print =====data00=$data00 + goto loop4 +endi + +if $data10 != col-2 then + print =====data10=$data10 + goto loop4 +endi + +$loop_count = 0 +loop5: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result3.streamt3; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop5 +endi + +$loop_count = 0 +loop6: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select table_name from information_schema.ins_tables where db_name="result3" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop6 +endi + +if $data00 != tbn-1 then + print =====data00=$data00 + goto loop6 +endi + +if $data10 != tbn-2 then + print =====data10=$data10 + goto loop6 +endi + +print ===== step5 +print ===== tag name + table name + +sql create database result4 vgroups 1; + +sql create database test4 vgroups 4; +sql use test4; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create table t3 using st tags(3,3,3); + +sql create stream streams4 trigger at_once into result4.streamt4 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", dd)) as select _wstart, count(*) c1 from st partition by concat("t", cast(a as varchar(10) ) ) as dd interval(10s); +sql insert into t1 values(1648791213000,1,1,1) t2 values(1648791213000,2,2,2) t3 values(1648791213000,3,3,3); + + +$loop_count = 0 +loop7: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select table_name from information_schema.ins_tables where db_name="result4" order by 1; + +if $rows != 3 then + print =====rows=$rows + print $data00 $data10 + goto loop7 +endi + +if $data00 != tbn-t1 then + print =====data00=$data00 + goto loop7 +endi + +if $data10 != tbn-t2 then + print =====data10=$data10 + goto loop7 +endi + +if $data20 != tbn-t3 then + print =====data20=$data20 + goto loop7 +endi + +$loop_count = 0 +loop8: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result4.streamt4 order by 3; + +if $rows != 3 then + print =====rows=$rows + print $data00 $data10 + goto loop8 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop8 +endi + +if $data02 != t1 then + print =====data02=$data02 + goto loop8 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop8 +endi + +if $data12 != t2 then + print =====data12=$data12 + goto loop8 +endi + +if $data21 != 1 then + print =====data21=$data21 + goto loop8 +endi + +if $data22 != t3 then + print =====data22=$data22 + goto loop8 +endi + +print ======over + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/windowClose.sim b/tests/script/tsim/stream/windowClose.sim index 9fcdcfb9599db5bbd2d4e93a069647a2518abaac..d3bc25d731f7a3800f0ae84d4ce2177cd573242a 100644 --- a/tests/script/tsim/stream/windowClose.sim +++ b/tests/script/tsim/stream/windowClose.sim @@ -68,7 +68,7 @@ sql select * from streamt2; if $rows != 1 then print ======streamt2=$rows - return -1 + goto loop1 endi sql select * from streamt3; @@ -80,7 +80,7 @@ endi sql select * from streamt4; if $rows != 1 then print ======streamt4=$rows - return -1 + goto loop1 endi sql select * from streamt5; @@ -92,7 +92,7 @@ endi sql select * from streamt6; if $rows != 1 then print ======streamt6=$rows - return -1 + goto loop1 endi sql select * from streamt7; @@ -104,7 +104,7 @@ endi sql select * from streamt8; if $rows != 1 then print ======streamt8=$rows - return -1 + goto loop1 endi sql select * from streamt9; @@ -116,7 +116,7 @@ endi sql select * from streamt10; if $rows != 1 then print ======streamt10=$rows - return -1 + goto loop1 endi sql select * from streamt11; @@ -125,4 +125,6 @@ if $rows != 2 then goto loop1 endi +print ======over + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index cd4acae43826ec942a6a46b97a501b888ec951ff..1834432bc9a9364e039df2c95ce47fb01e945eab 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -12,6 +12,7 @@ from util.dnodes import * from util.dnodes import TDDnodes from util.dnodes import TDDnode from util.cluster import * +import subprocess BASEVERSION = "3.0.1.8" class TDTestCase: @@ -26,8 +27,21 @@ class TDTestCase: self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) + + def checkProcessPid(self,processName): + i=0 + while i<60: + print(f"wait stop {processName}") + processPid = subprocess.getstatusoutput(f'ps aux|grep {processName} |grep -v "grep"|awk \'{{print $2}}\'')[1] + print(f"times:{i},{processName}-pid:{processPid}") + if(processPid == ""): + break + i += 1 + sleep(1) + else: + print(f'this processName is not stoped in 60s') - + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -68,6 +82,7 @@ class TDTestCase: my_file = Path(f"{packagePath}/{packageName}") if not my_file.exists(): print(f"{packageName} is not exists") + tdLog.info(f"cd {packagePath} && wget https://www.tdengine.com/assets-download/3.0/{packageName}") os.system(f"cd {packagePath} && wget https://www.tdengine.com/assets-download/3.0/{packageName}") else: print(f"{packageName} has been exists") @@ -114,15 +129,15 @@ class TDTestCase: # tdsqlF.query(f"select count(*) from {stb}") # tdsqlF.checkData(0,0,tableNumbers*recordNumbers1) os.system("pkill taosd") - sleep(2) + self.checkProcessPid("taosd") print(f"start taosd: nohup taosd -c {cPath} & ") os.system(f" nohup taosd -c {cPath} & " ) sleep(10) tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y") - os.system("pkill -9 taosd") - + os.system("pkill taosd") # make sure all the data are saved in disk. + self.checkProcessPid("taosd") tdLog.printNoPrefix("==========step2:update new version ") diff --git a/tests/system-test/0-others/taosdShell.py b/tests/system-test/0-others/taosdShell.py index 7ad7e4d0ef90b7f30f692cb40596c14b653e903d..2125b5eebe448b4ac044158f7157cd689eaec29e 100644 --- a/tests/system-test/0-others/taosdShell.py +++ b/tests/system-test/0-others/taosdShell.py @@ -136,7 +136,7 @@ class TDTestCase: tdSql.query("use source_db") tdSql.query("create table if not exists source_db.stb (ts timestamp, k int) tags (a int);") tdSql.query("create table source_db.ct1 using source_db.stb tags(1000);create table source_db.ct2 using source_db.stb tags(2000);create table source_db.ct3 using source_db.stb tags(3000);") - tdSql.query("create stream s1 into source_db.output_stb as select _wstart AS start, min(k), max(k), sum(k) from source_db.stb interval(10m);") + tdSql.query("create stream s1 into source_db.output_stb as select _wstart AS startts, min(k), max(k), sum(k) from source_db.stb interval(10m);") #TD-19944 -Q=3 diff --git a/tests/system-test/1-insert/database_pre_suf.py b/tests/system-test/1-insert/database_pre_suf.py index 862edbdde9b795f151b9d17f3c74abc8ebc4de63..488dfebff50c77d1dd08c6d4e46fdbf3bbe3ea7f 100755 --- a/tests/system-test/1-insert/database_pre_suf.py +++ b/tests/system-test/1-insert/database_pre_suf.py @@ -108,7 +108,7 @@ class TDTestCase: # create stream - tdSql.execute('''create stream current_stream into stream_max_stable_1 as select _wstart as start, _wend as wend, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 where ts is not null interval (5s);''') + tdSql.execute('''create stream current_stream into stream_max_stable_1 as select _wstart as startts, _wend as wend, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 where ts is not null interval (5s);''') # insert data for i in range(num_random*n): @@ -187,20 +187,20 @@ class TDTestCase: sleep(5) # stream data check - tdSql.query("select start,wend,max_int from stream_max_stable_1 ;") + tdSql.query("select startts,wend,max_int from stream_max_stable_1 ;") tdSql.checkRows(20) tdSql.query("select sum(max_int) from stream_max_stable_1 ;") stream_data_1 = tdSql.queryResult[0][0] tdSql.query("select sum(min_int) from stream_max_stable_1 ;") stream_data_2 = tdSql.queryResult[0][0] - tdSql.query("select sum(max_int),sum(min_int) from (select _wstart as start, _wend as wend, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 where ts is not null interval (5s));") + tdSql.query("select sum(max_int),sum(min_int) from (select _wstart as startts, _wend as wend, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 where ts is not null interval (5s));") sql_data_1 = tdSql.queryResult[0][0] sql_data_2 = tdSql.queryResult[0][1] self.stream_value_check(stream_data_1,sql_data_1) self.stream_value_check(stream_data_2,sql_data_2) - tdSql.query("select sum(max_int),sum(min_int) from (select _wstart as start, _wend as wend, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 interval (5s));") + tdSql.query("select sum(max_int),sum(min_int) from (select _wstart as startts, _wend as wend, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 interval (5s));") sql_data_1 = tdSql.queryResult[0][0] sql_data_2 = tdSql.queryResult[0][1] diff --git a/tests/system-test/1-insert/opentsdb_json_taosc_insert.py b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py index 44243fe029233dc3458a7b6822600bc198e11824..5e493eab0af2bd0cf5ebe463cbde889fa957041b 100644 --- a/tests/system-test/1-insert/opentsdb_json_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py @@ -1432,9 +1432,9 @@ class TDTestCase: self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) query_sql = 'select * from `rFa$sta`' query_res = tdSql.query(query_sql, True) - tdSql.checkEqual(query_res, [(datetime.datetime(2021, 7, 11, 20, 33, 54), True, 'rFas$ta_1', 'ncharTagValue', 2147483647, 9223372036854775807, 22.123456789, 'binaryTagValue', 32767, 11.12345027923584, False, 127)]) + tdSql.checkEqual(query_res, [(datetime.datetime(2021, 7, 11, 20, 33, 54), True, False, 127, 32767, 2147483647, 9223372036854775807, 11.12345027923584, 22.123456789, 'binaryTagValue', 'ncharTagValue', 'rFas$ta_1')]) col_tag_res = tdSql.getColNameList(query_sql) - tdSql.checkEqual(col_tag_res, ['_ts', '_value', 'id', 't!@#$%^&*()_+[];:<>?,9', 't$3', 't%4', 't&6', 't*7', 't@2', 't^5', 'Tt!0', 'tT@1']) + tdSql.checkEqual(col_tag_res, ['_ts', '_value', 'Tt!0', 'tT@1', 't@2', 't$3', 't%4', 't^5', 't&6', 't*7', 't!@#$%^&*()_+[];:<>?,9', 'id']) tdSql.execute('drop table `rFa$sta`') def pointTransCheckCase(self, value_type="obj"): diff --git a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py index f58882720610b32794fc48e4cd652ae70e63b5d7..351cf49e3a217a44e93bbaf9c8c69ce2fa76c190 100644 --- a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py @@ -243,7 +243,7 @@ class TDTestCase: if t_add_tag is not None: sql_seq = f'{stb_name} {ts} {value} t0={t0} t1={t1} t2={t2} t3={t3} t4={t4} t5={t5} t6={t6} t7={t7} t8={t8} t9={t8}' if id_change_tag is not None: - sql_seq = f'{stb_name} {ts} {value} t0={t0} {id}={tb_name} t1={t1} t2={t2} t3={t3} t4={t4} t5={t5} t6={t6} t7={t7} t8={t8}' + sql_seq = f'{stb_name} {ts} {value} {id}={tb_name} t0={t0} t1={t1} t2={t2} t3={t3} t4={t4} t5={t5} t6={t6} t7={t7} t8={t8}' if id_double_tag is not None: sql_seq = f'{stb_name} {ts} {value} {id}=\"{tb_name}_1\" t0={t0} t1={t1} {id}=\"{tb_name}_2\" t2={t2} t3={t3} t4={t4} t5={t5} t6={t6} t7={t7} t8={t8}' if t_add_tag is not None: @@ -1126,9 +1126,9 @@ class TDTestCase: self._conn.schemaless_insert([input_sql], TDSmlProtocolType.TELNET.value, None) query_sql = 'select * from `rFa$sta`' query_res = tdSql.query(query_sql, True) - tdSql.checkEqual(query_res, [(datetime.datetime(2021, 7, 11, 20, 33, 54), 9.223372036854776e+18, '2147483647i32', 'L"ncharTagValue"', '32767i16', '9223372036854775807i64', '22.123456789f64', '"ddzhiksj"', '11.12345f32', 'true', '127Ii8')]) + tdSql.checkEqual(query_res, [(datetime.datetime(2021, 7, 11, 20, 33, 54), 9.223372036854776e+18, 'true', '127Ii8', '32767i16', '2147483647i32', '9223372036854775807i64', '11.12345f32', '22.123456789f64', '"ddzhiksj"', 'L"ncharTagValue"')]) col_tag_res = tdSql.getColNameList(query_sql) - tdSql.checkEqual(col_tag_res, ['_ts', '_value', '"t$3"', 't!@#$%^&*()_+[];:<>?,9', 't#2', 't%4', 't&6', 't*7', 't^5', 'Tt!0', 'tT@1']) + tdSql.checkEqual(col_tag_res, ['_ts', '_value', 'Tt!0', 'tT@1', 't#2', '"t$3"', 't%4', 't^5', 't&6', 't*7', 't!@#$%^&*()_+[];:<>?,9']) tdSql.execute('drop table `rFa$sta`') def tcpKeywordsCheckCase(self, protocol="telnet-tcp"): @@ -1207,7 +1207,6 @@ class TDTestCase: tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') tdCom.cleanTb(dbname="test") input_sql = self.genSqlList()[0] - print(input_sql) self.multiThreadRun(self.genMultiThreadSeq(input_sql)) tdSql.query(f"show tables;") tdSql.checkRows(5) @@ -1416,7 +1415,7 @@ class TDTestCase: self.symbolsCheckCase() self.tsCheckCase() self.openTstbTelnetTsCheckCase() - self.idSeqCheckCase() + # self.idSeqCheckCase() self.idLetterCheckCase() self.noIdCheckCase() self.maxColTagCheckCase() diff --git a/tests/system-test/2-query/insert_null_none.py b/tests/system-test/2-query/insert_null_none.py index cf5636fb1fa5599cde9f096915667f0f4af7a3e8..4304dee89ef71b0d44c26567f393404fea8bedc2 100755 --- a/tests/system-test/2-query/insert_null_none.py +++ b/tests/system-test/2-query/insert_null_none.py @@ -24,7 +24,7 @@ from util.dnodes import tdDnodes from util.dnodes import * class TDTestCase: - updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"querySmaOptimize":1} + updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 131 ,"querySmaOptimize":1} def init(self, conn, logSql, replicaVar): tdLog.debug("start to execute %s" % __file__) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index ce57357abda6564238362bcb861abab5e92f3947..5077b70d725d26c1c4adb897c55a1b57bcfb219b 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -362,12 +362,12 @@ class TDTestCase: tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(linear)") tdSql.checkRows(0) - tdLog.printNoPrefix("==========step8:test _irowts with interp") + tdLog.printNoPrefix("==========step8:test _irowts,_isfilled with interp") # fill null - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(null)") + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(null)") tdSql.checkRows(9) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:08.000') tdSql.checkData(1, 0, '2020-02-01 00:00:08.500') @@ -379,9 +379,19 @@ class TDTestCase: tdSql.checkData(7, 0, '2020-02-01 00:00:11.500') tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(null)") + tdSql.checkData(0, 1, True) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, False) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, True) + tdSql.checkData(8, 1, True) + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(null)") tdSql.checkRows(13) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') tdSql.checkData(1, 0, '2020-02-01 00:00:05.000') @@ -397,9 +407,23 @@ class TDTestCase: tdSql.checkData(11, 0, '2020-02-01 00:00:15.000') tdSql.checkData(12, 0, '2020-02-01 00:00:16.000') - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(null)") + tdSql.checkData(0, 1, True) + tdSql.checkData(1, 1, False) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, False) + tdSql.checkData(7, 1, True) + tdSql.checkData(8, 1, True) + tdSql.checkData(9, 1, True) + tdSql.checkData(10, 1, True) + tdSql.checkData(11, 1, False) + tdSql.checkData(12, 1, True) + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(null)") tdSql.checkRows(6) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') tdSql.checkData(1, 0, '2020-02-01 00:00:07.000') @@ -408,10 +432,16 @@ class TDTestCase: tdSql.checkData(4, 0, '2020-02-01 00:00:13.000') tdSql.checkData(5, 0, '2020-02-01 00:00:15.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) # fill value - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(value, 1)") + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(value, 1)") tdSql.checkRows(9) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:08.000') tdSql.checkData(1, 0, '2020-02-01 00:00:08.500') @@ -423,9 +453,19 @@ class TDTestCase: tdSql.checkData(7, 0, '2020-02-01 00:00:11.500') tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)") + tdSql.checkData(0, 1, True) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, False) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, True) + tdSql.checkData(8, 1, True) + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)") tdSql.checkRows(13) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') tdSql.checkData(1, 0, '2020-02-01 00:00:05.000') @@ -441,9 +481,23 @@ class TDTestCase: tdSql.checkData(11, 0, '2020-02-01 00:00:15.000') tdSql.checkData(12, 0, '2020-02-01 00:00:16.000') - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(value, 1)") + tdSql.checkData(0, 1, True) + tdSql.checkData(1, 1, False) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, False) + tdSql.checkData(7, 1, True) + tdSql.checkData(8, 1, True) + tdSql.checkData(9, 1, True) + tdSql.checkData(10, 1, True) + tdSql.checkData(11, 1, False) + tdSql.checkData(12, 1, True) + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(value, 1)") tdSql.checkRows(6) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') tdSql.checkData(1, 0, '2020-02-01 00:00:07.000') @@ -452,10 +506,17 @@ class TDTestCase: tdSql.checkData(4, 0, '2020-02-01 00:00:13.000') tdSql.checkData(5, 0, '2020-02-01 00:00:15.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) + # fill prev - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(prev)") + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(prev)") tdSql.checkRows(9) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:08.000') tdSql.checkData(1, 0, '2020-02-01 00:00:08.500') @@ -467,9 +528,19 @@ class TDTestCase: tdSql.checkData(7, 0, '2020-02-01 00:00:11.500') tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(prev)") + tdSql.checkData(0, 1, True) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, False) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, True) + tdSql.checkData(8, 1, True) + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(prev)") tdSql.checkRows(12) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') tdSql.checkData(1, 0, '2020-02-01 00:00:06.000') @@ -484,9 +555,22 @@ class TDTestCase: tdSql.checkData(10, 0, '2020-02-01 00:00:15.000') tdSql.checkData(11, 0, '2020-02-01 00:00:16.000') - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(prev)") + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, True) + tdSql.checkData(8, 1, True) + tdSql.checkData(9, 1, True) + tdSql.checkData(10, 1, False) + tdSql.checkData(11, 1, True) + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(prev)") tdSql.checkRows(6) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') tdSql.checkData(1, 0, '2020-02-01 00:00:07.000') @@ -495,10 +579,16 @@ class TDTestCase: tdSql.checkData(4, 0, '2020-02-01 00:00:13.000') tdSql.checkData(5, 0, '2020-02-01 00:00:15.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) # fill next - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(next)") + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(next)") tdSql.checkRows(9) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:08.000') tdSql.checkData(1, 0, '2020-02-01 00:00:08.500') @@ -510,9 +600,19 @@ class TDTestCase: tdSql.checkData(7, 0, '2020-02-01 00:00:11.500') tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(next)") + tdSql.checkData(0, 1, True) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, False) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, True) + tdSql.checkData(8, 1, True) + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(next)") tdSql.checkRows(12) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') tdSql.checkData(1, 0, '2020-02-01 00:00:05.000') @@ -527,9 +627,22 @@ class TDTestCase: tdSql.checkData(10, 0, '2020-02-01 00:00:14.000') tdSql.checkData(11, 0, '2020-02-01 00:00:15.000') - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(next)") + tdSql.checkData(0, 1, True) + tdSql.checkData(1, 1, False) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, False) + tdSql.checkData(7, 1, True) + tdSql.checkData(8, 1, True) + tdSql.checkData(9, 1, True) + tdSql.checkData(10, 1, True) + tdSql.checkData(11, 1, False) + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(next)") tdSql.checkRows(6) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') tdSql.checkData(1, 0, '2020-02-01 00:00:07.000') @@ -538,10 +651,17 @@ class TDTestCase: tdSql.checkData(4, 0, '2020-02-01 00:00:13.000') tdSql.checkData(5, 0, '2020-02-01 00:00:15.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) + # fill linear - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(linear)") + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(linear)") tdSql.checkRows(9) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:08.000') tdSql.checkData(1, 0, '2020-02-01 00:00:08.500') @@ -553,9 +673,19 @@ class TDTestCase: tdSql.checkData(7, 0, '2020-02-01 00:00:11.500') tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") + tdSql.checkData(0, 1, True) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, False) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, True) + tdSql.checkData(8, 1, True) + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") tdSql.checkRows(11) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') tdSql.checkData(1, 0, '2020-02-01 00:00:06.000') @@ -569,9 +699,21 @@ class TDTestCase: tdSql.checkData(9, 0, '2020-02-01 00:00:14.000') tdSql.checkData(10, 0, '2020-02-01 00:00:15.000') - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(linear)") + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, True) + tdSql.checkData(8, 1, True) + tdSql.checkData(9, 1, True) + tdSql.checkData(10, 1, False) + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(linear)") tdSql.checkRows(6) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') tdSql.checkData(1, 0, '2020-02-01 00:00:07.000') @@ -580,28 +722,47 @@ class TDTestCase: tdSql.checkData(4, 0, '2020-02-01 00:00:13.000') tdSql.checkData(5, 0, '2020-02-01 00:00:15.000') - # multiple _irowts - tdSql.query(f"select interp(c0),_irowts from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") - tdSql.checkRows(11) - tdSql.checkCols(2) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) - tdSql.checkData(0, 1, '2020-02-01 00:00:05.000') - tdSql.checkData(1, 1, '2020-02-01 00:00:06.000') - tdSql.checkData(2, 1, '2020-02-01 00:00:07.000') - tdSql.checkData(3, 1, '2020-02-01 00:00:08.000') - tdSql.checkData(4, 1, '2020-02-01 00:00:09.000') - tdSql.checkData(5, 1, '2020-02-01 00:00:10.000') - tdSql.checkData(6, 1, '2020-02-01 00:00:11.000') - tdSql.checkData(7, 1, '2020-02-01 00:00:12.000') - tdSql.checkData(8, 1, '2020-02-01 00:00:13.000') - tdSql.checkData(9, 1, '2020-02-01 00:00:14.000') + # multiple _irowts,_isfilled + tdSql.query(f"select interp(c0),_irowts,_isfilled from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") + tdSql.checkRows(11) + tdSql.checkCols(3) + + tdSql.checkData(0, 1, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 1, '2020-02-01 00:00:06.000') + tdSql.checkData(2, 1, '2020-02-01 00:00:07.000') + tdSql.checkData(3, 1, '2020-02-01 00:00:08.000') + tdSql.checkData(4, 1, '2020-02-01 00:00:09.000') + tdSql.checkData(5, 1, '2020-02-01 00:00:10.000') + tdSql.checkData(6, 1, '2020-02-01 00:00:11.000') + tdSql.checkData(7, 1, '2020-02-01 00:00:12.000') + tdSql.checkData(8, 1, '2020-02-01 00:00:13.000') + tdSql.checkData(9, 1, '2020-02-01 00:00:14.000') tdSql.checkData(10, 1, '2020-02-01 00:00:15.000') - tdSql.query(f"select _irowts, interp(c0), interp(c0), _irowts from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") + tdSql.checkData(0, 2, False) + tdSql.checkData(1, 2, True) + tdSql.checkData(2, 2, True) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, True) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, True) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, True) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, False) + + tdSql.query(f"select _irowts, _isfilled, interp(c0), interp(c0), _isfilled, _irowts from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") tdSql.checkRows(11) - tdSql.checkCols(4) + tdSql.checkCols(6) - cols = (0, 3) + cols = (0, 5) for i in cols: tdSql.checkData(0, i, '2020-02-01 00:00:05.000') tdSql.checkData(1, i, '2020-02-01 00:00:06.000') @@ -615,6 +776,20 @@ class TDTestCase: tdSql.checkData(9, i, '2020-02-01 00:00:14.000') tdSql.checkData(10, i, '2020-02-01 00:00:15.000') + cols = (1, 4) + for i in cols: + tdSql.checkData(0, i, False) + tdSql.checkData(1, i, True) + tdSql.checkData(2, i, True) + tdSql.checkData(3, i, True) + tdSql.checkData(4, i, True) + tdSql.checkData(5, i, False) + tdSql.checkData(6, i, True) + tdSql.checkData(7, i, True) + tdSql.checkData(8, i, True) + tdSql.checkData(9, i, True) + tdSql.checkData(10, i, False) + tdLog.printNoPrefix("==========step9:test intra block interpolation") tdSql.execute(f"drop database {dbname}"); diff --git a/tests/system-test/2-query/leastsquares.py b/tests/system-test/2-query/leastsquares.py index 8ece4c46f0d430ec540468467e8b9e89b35e1349..1718802a82a0ffb740918e1ff0e083a11168b3f4 100644 --- a/tests/system-test/2-query/leastsquares.py +++ b/tests/system-test/2-query/leastsquares.py @@ -19,12 +19,17 @@ BINARY_COL = "c8" NCHAR_COL = "c9" TS_COL = "c10" -NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] +UINT_COL = "c11" +UBINT_COL = "c12" +USINT_COL = "c13" +UTINT_COL = "c14" + +NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, UINT_COL, UBINT_COL, USINT_COL, UTINT_COL] CHAR_COL = [ BINARY_COL, NCHAR_COL, ] BOOLEAN_COL = [ BOOL_COL, ] TS_TYPE_COL = [ TS_COL, ] -ALL_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BOOL_COL, BINARY_COL, NCHAR_COL, TS_COL ] +ALL_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BOOL_COL, BINARY_COL, NCHAR_COL, TS_COL, UINT_COL, UBINT_COL, USINT_COL, UTINT_COL ] DBNAME = "db" class TDTestCase: @@ -208,6 +213,13 @@ class TDTestCase: tdLog.info(f"sql: {current_sqls[i]}") tdSql.query(current_sqls[i]) + def check_result(self): + for col in NUM_COL: + tdSql.query("select leastsquares(%s, 1, 9) from %s.stb1" % (col, DBNAME)) + tdSql.checkRows(1) + res = tdSql.getData(0, 0) + if res is None: + tdLog.exit("result is not correct") def __test_current(self): # tdSql.query("explain select c1 from {dbname}.ct1") @@ -236,6 +248,7 @@ class TDTestCase: def all_test(self): self.__test_error() self.__test_current() + self.check_result() def __create_tb(self, dbname=DBNAME): @@ -243,13 +256,15 @@ class TDTestCase: create_stb_sql = f'''create table {dbname}.stb1( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, - {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp, {UINT_COL} int unsigned, + {UBINT_COL} bigint unsigned, {USINT_COL} smallint unsigned, {UTINT_COL} tinyint unsigned ) tags (t1 int) ''' create_ntb_sql = f'''create table {dbname}.nt1( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, - {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp, {UINT_COL} int unsigned, + {UBINT_COL} bigint unsigned, {USINT_COL} smallint unsigned, {UTINT_COL} tinyint unsigned ) ''' tdSql.execute(create_stb_sql) @@ -262,49 +277,49 @@ class TDTestCase: now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) for i in range(rows): tdSql.execute( - f"insert into {dbname}.ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + f"insert into {dbname}.ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127} )" ) tdSql.execute( - f"insert into {dbname}.ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + f"insert into {dbname}.ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127} )" ) tdSql.execute( - f"insert into {dbname}.ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + f"insert into {dbname}.ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127} )" ) tdSql.execute( f'''insert into {dbname}.ct1 values - ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } ) - ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } ) + ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 }, 0, 0, 0, 0) + ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 }, 0, 0, 0, NULL ) ''' ) tdSql.execute( f'''insert into {dbname}.ct4 values - ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) ( { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127, - { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000} + { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000}, NULL, NULL, NULL, NULL ) ( { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126, - { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000} + { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000}, NULL, NULL, NULL, NULL ) ''' ) tdSql.execute( f'''insert into {dbname}.ct2 values - ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) + ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126, - { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } + { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }, NULL, NULL, NULL, NULL ) ( { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127, - { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } + { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }, NULL, NULL, NULL, NULL ) ''' ) @@ -312,22 +327,22 @@ class TDTestCase: for i in range(rows): insert_data = f'''insert into {dbname}.nt1 values ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, - "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } ) + "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i }, NULL, NULL, NULL, NULL ) ''' tdSql.execute(insert_data) tdSql.execute( f'''insert into {dbname}.nt1 values - ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127, { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, - "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } + "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }, NULL, NULL, NULL, NULL ) ( { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126, { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, - "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } + "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }, NULL, NULL, NULL, NULL ) ''' ) diff --git a/tests/system-test/2-query/nestedQuery.py b/tests/system-test/2-query/nestedQuery.py index 3d0db9a562b508234cb8897f178ec5dc73d944a6..6557aad05f7dc0d3021f1e0bbba8dfb08964ad0f 100755 --- a/tests/system-test/2-query/nestedQuery.py +++ b/tests/system-test/2-query/nestedQuery.py @@ -24,9 +24,9 @@ from util.dnodes import tdDnodes from util.dnodes import * class TDTestCase: - updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , - "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, - "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 131 ,"cDebugFlag":131,"uDebugFlag":131 ,"rpcDebugFlag":131 , "tmrDebugFlag":131 , + "jniDebugFlag":131 ,"simDebugFlag":131,"dDebugFlag":131, "dDebugFlag":131,"vDebugFlag":131,"mDebugFlag":131,"qDebugFlag":131, + "wDebugFlag":131,"sDebugFlag":131,"tsdbDebugFlag":131,"tqDebugFlag":131 ,"fsDebugFlag":131 ,"fnDebugFlag":131} def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index 78b633cf94259058b5a50eca2de66d6b2f922d26..ec6309c71ad295eb504d8f97b493a062a044fed1 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -21,7 +21,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), True) #tdSql.init(conn.cursor(), logSql) # output sql.txt file def checkFileContent(self, dbname="sml_db"): @@ -76,22 +76,26 @@ class TDTestCase: tdSql.checkData(0, 2, "web01") tdSql.query(f"select distinct tbname from {dbname}.`sys.cpu.nice`") - tdSql.checkRows(2) + tdSql.checkRows(3) tdSql.query(f"select * from {dbname}.`sys.cpu.nice` order by _ts") - tdSql.checkRows(2) - tdSql.checkData(0, 1, 9.000000000) - tdSql.checkData(0, 2, "lga") - tdSql.checkData(0, 3, "web02") - tdSql.checkData(0, 4, None) - tdSql.checkData(1, 1, 18.000000000) - tdSql.checkData(1, 2, "lga") - tdSql.checkData(1, 3, "web01") - tdSql.checkData(1, 4, "t1") + tdSql.checkRows(4) + tdSql.checkData(0, 1, 13.000000000) + tdSql.checkData(0, 2, "web01") + tdSql.checkData(0, 3, None) + tdSql.checkData(0, 4, "lga") + + tdSql.checkData(1, 1, 9.000000000) + tdSql.checkData(1, 2, "web02") + tdSql.checkData(3, 3, "t1") + tdSql.checkData(0, 4, "lga") tdSql.query(f"select * from {dbname}.macylr") tdSql.checkRows(2) + tdSql.query(f"select * from {dbname}.qelhxo") + tdSql.checkRows(5) + tdSql.query(f"desc {dbname}.macylr") tdSql.checkRows(25) return diff --git a/tests/system-test/2-query/stablity.py b/tests/system-test/2-query/stablity.py index ff026bf1202af2f3e69b2f5316f193c4cdc54296..5e4d5dcbaf0368cbc5f5955d4bd172131eddfacf 100755 --- a/tests/system-test/2-query/stablity.py +++ b/tests/system-test/2-query/stablity.py @@ -24,9 +24,9 @@ from util.dnodes import tdDnodes from util.dnodes import * class TDTestCase: - updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , - "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, - "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 131 ,"cDebugFlag":131,"uDebugFlag":131 ,"rpcDebugFlag":131 , "tmrDebugFlag":131 , + "jniDebugFlag":131 ,"simDebugFlag":131,"dDebugFlag":131, "dDebugFlag":131,"vDebugFlag":131,"mDebugFlag":131,"qDebugFlag":131, + "wDebugFlag":131,"sDebugFlag":131,"tsdbDebugFlag":131,"tqDebugFlag":131 ,"fsDebugFlag":131 ,"fnDebugFlag":131} def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) diff --git a/tests/system-test/6-cluster/5dnode1mnode.py b/tests/system-test/6-cluster/5dnode1mnode.py index d0157ec7c772182a2e24064dfec3cc29ed2a7ed6..ce87bced1b75d1994024586cf5084c5194befc22 100644 --- a/tests/system-test/6-cluster/5dnode1mnode.py +++ b/tests/system-test/6-cluster/5dnode1mnode.py @@ -137,11 +137,34 @@ class TDTestCase: config_dir = dnode.cfgDir return taos.connect(host=host, port=int(port), config=config_dir) + def check_alive(self): + # check cluster alive + tdLog.printNoPrefix("======== test cluster alive: ") + tdSql.checkDataLoop(0, 0, 1, "show cluster alive;", 20, 0.5) + + tdSql.query("show db.alive;") + tdSql.checkData(0, 0, 1) + + # stop 5 dnode + self.TDDnodes.stoptaosd(5) + tdSql.checkDataLoop(0, 0, 2, "show cluster alive;", 20, 0.5) + + tdSql.query("show db.alive;") + tdSql.checkData(0, 0, 2) + + # stop 2 dnode + self.TDDnodes.stoptaosd(2) + tdSql.checkDataLoop(0, 0, 0, "show cluster alive;", 20, 0.5) + + tdSql.query("show db.alive;") + tdSql.checkData(0, 0, 0) + def run(self): # print(self.master_dnode.cfgDict) self.five_dnode_one_mnode() - + # check cluster and db alive + self.check_alive() def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeDrop.py b/tests/system-test/6-cluster/5dnode3mnodeDrop.py index 4f3916a48730a92fbcd59ce6ea8543365ce73b5a..9dd3c568054948d7062c2f87f145149509b1b575 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeDrop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeDrop.py @@ -121,7 +121,7 @@ class TDTestCase: def check3mnode(self): count=0 while count < 10: - time.sleep(1) + time.sleep(0.1) tdSql.query("select * from information_schema.ins_mnodes;") if tdSql.checkRows(3) : tdLog.debug("mnode is three nodes") @@ -158,7 +158,7 @@ class TDTestCase: def check3mnode1off(self): count=0 while count < 10: - time.sleep(1) + time.sleep(0.1) tdSql.query("select * from information_schema.ins_mnodes;") if tdSql.checkRows(3) : tdLog.debug("mnode is three nodes") @@ -190,7 +190,7 @@ class TDTestCase: def check3mnode2off(self): count=0 while count < 40: - time.sleep(1) + time.sleep(0.1) tdSql.query("select * from information_schema.ins_mnodes;") if tdSql.checkRows(3) : tdLog.debug("mnode is three nodes") @@ -220,7 +220,7 @@ class TDTestCase: def check3mnode3off(self): count=0 while count < 10: - time.sleep(1) + time.sleep(0.1) tdSql.query("select * from information_schema.ins_mnodes;") if tdSql.checkRows(3) : tdLog.debug("mnode is three nodes") @@ -280,32 +280,32 @@ class TDTestCase: # drop follower of mnode dropcount =0 - while dropcount <= 10: + while dropcount <= 5: for i in range(1,3): tdLog.debug("drop mnode on dnode %d"%(i+1)) tdSql.execute("drop mnode on dnode %d"%(i+1)) tdSql.query("select * from information_schema.ins_mnodes;") count=0 while count<10: - time.sleep(1) + time.sleep(0.1) tdSql.query("select * from information_schema.ins_mnodes;") - if tdSql.checkRows(2): + if tdSql.queryRows == 2: tdLog.debug("drop mnode %d successfully"%(i+1)) break count+=1 - self.wait_for_transactions(20) + self.wait_for_transactions(100) tdLog.debug("create mnode on dnode %d"%(i+1)) tdSql.execute("create mnode on dnode %d"%(i+1)) count=0 while count<10: - time.sleep(1) + time.sleep(0.1) tdSql.query("select * from information_schema.ins_mnodes;") - if tdSql.checkRows(3): + if tdSql.queryRows == 3: tdLog.debug("create mnode %d successfully"%(i+1)) break count+=1 - self.wait_for_transactions(20) + self.wait_for_transactions(100) dropcount+=1 self.check3mnode() @@ -314,12 +314,13 @@ class TDTestCase: while count= timeout: - tdLog.debug("transactions not finished before timeout (%d secs)", timeout) + tdLog.debug("transactions not finished before timeout (%d secs)"%timeout) def getConnection(self, dnode): host = dnode.cfgDict["fqdn"] diff --git a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py index 94e02b77b307281975ff7393298f182f233a7c65..392b0d7764c4404a4e2282b7bf5708a79556acf8 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py @@ -161,6 +161,19 @@ class TDTestCase: stableName= '%s_%d'%(paraDict['stbName'],i) newTdSql=tdCom.newTdSql() threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]))) + + for i in range(5): + clusterComCreate.createUser(newTdSql,f"user{i}",f"pass{i}") + userTdSql=tdCom.newTdSql(user=f"user{i}",password=f"pass{i}") + clusterComCreate.alterUser(userTdSql,f"user{i}",f"pass{i+1}") + clusterComCreate.deleteUser(newTdSql,f"user{i}") + for j in range(5): + i=100 + clusterComCreate.createUser(newTdSql,f"user{i}",f"pass{i}") + userTdSql=tdCom.newTdSql(user=f"user{i}",password=f"pass{i}") + clusterComCreate.alterUser(userTdSql,f"user{i}",f"pass{i+1}") + clusterComCreate.deleteUser(newTdSql,f"user{i}") + for tr in threads: tr.start() for tr in threads: diff --git a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py index d6d06446a1ab60c0dabdf0a676f7067a3497c799..66c2fdd14fd5f9ad5ef4c8944dab481e434a5970 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py @@ -67,29 +67,11 @@ class TDTestCase: self._async_raise(thread.ident, SystemExit) - def insertData(self,countstart,countstop): - # fisrt add data : db\stable\childtable\general table - - for couti in range(countstart,countstop): - tdLog.debug("drop database if exists db%d" %couti) - tdSql.execute("drop database if exists db%d" %couti) - print("create database if not exists db%d replica 1 duration 300" %couti) - tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) - tdSql.execute("use db%d" %couti) - tdSql.execute( - '''create table stb1 - (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) - tags (t1 int) - ''' - ) - tdSql.execute( - ''' - create table t1 - (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) - ''' - ) - for i in range(4): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + def reCreateUser(self, tdsql, count, user, passwd): + clusterComCreate.createUser(tdsql,f"{user}{count}",f"{passwd}{count}") + userTdSql=tdCom.newTdSql(user=f"{user}{count}",password=f"{passwd}{count}") + clusterComCreate.alterUser(userTdSql,f"{user}{count}",f"{passwd}{count+1}") + clusterComCreate.deleteUser(tdsql,f"{user}{count}") def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): @@ -161,6 +143,8 @@ class TDTestCase: stableName= '%s_%d'%(paraDict['stbName'],i) newTdSql=tdCom.newTdSql() threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]))) + threads.append(threading.Thread(target=self.reCreateUser,args=(newTdSql,i,"user","passwd"))) + for tr in threads: tr.start() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py index 881f383b8c07b66987290e07cda6eda99d01dce5..b55c689eee88ef6f4f19893f6d6bf18a91d6acc4 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py @@ -147,6 +147,9 @@ class TDTestCase: # print(f"==================={dbNameIndex},{a11111}") threads.append(threading.Thread(target=clusterComCreate.createDeltedatabases, args=(newTdSql, dbNameIndex,repeatNumber,paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))) + redbNameIndex = '%s%d'%(paraDict["dbName"],i+100) + threads.append(threading.Thread(target=clusterComCreate.createDeltedatabases, args=(newTdSql, redbNameIndex,1,paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))) + for tr in threads: tr.start() @@ -199,7 +202,7 @@ class TDTestCase: def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=4,stopRole='dnode') + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=1,stopRole='dnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py index 265000bdc9d0c5530fdcbd7d4e6ed633c58b0a29..296e9daecaf16d3ef4016bafa7454ff0c7d62af9 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py @@ -97,7 +97,7 @@ class TDTestCase: 'dropFlag': 1, 'event': '', 'vgroups': 4, - 'replica': 1, + 'replica': 3, 'stbName': 'stb', 'stbNumbers': 2, 'colPrefix': 'c', @@ -105,9 +105,9 @@ class TDTestCase: 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], 'ctbPrefix': 'ctb', - 'ctbNum': 200, + 'ctbNum': 100, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - "rowsPerTbl": 10000, + "rowsPerTbl": 100000, "batchNum": 5000 } @@ -198,16 +198,16 @@ class TDTestCase: clusterComCheck.checkDbRows(dbNumbers) # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"]) - tdSql.execute("use %s" %(paraDict["dbName"])) - tdSql.query("show stables") + # tdSql.execute("use %s" %(paraDict["dbName"])) + tdSql.query("show %s.stables"%(paraDict["dbName"])) tdSql.checkRows(paraDict["stbNumbers"]) for i in range(paraDict['stbNumbers']): - stableName= '%s_%d'%(paraDict['stbName'],i) - tdSql.query("select * from %s"%stableName) - tdSql.checkRows(rowsPerStb) + stableName= '%s.%s_%d'%(paraDict["dbName"],paraDict['stbName'],i) + tdSql.query("select count(*) from %s"%stableName) + tdSql.checkData(0,0,rowsPerStb) def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=2,stopRole='dnode') + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=1,stopRole='dnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py new file mode 100644 index 0000000000000000000000000000000000000000..06d626b77cc8d9e67f1ecbe2fc524a1d9a69decc --- /dev/null +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py @@ -0,0 +1,202 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +from numpy import row_stack +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + +import time +import socket +import subprocess +from multiprocessing import Process +import threading +import time +import inspect +import ctypes + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + self.TDDnodes = None + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def _async_raise(self, tid, exctype): + """raises the exception, performs cleanup if needed""" + if not inspect.isclass(exctype): + exctype = type(exctype) + res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype)) + if res == 0: + raise ValueError("invalid thread id") + elif res != 1: + # """if it returns a number greater than one, you're in trouble, + # and you should call it again with exc=NULL to revert the effect""" + ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) + raise SystemError("PyThreadState_SetAsyncExc failed") + + def stopThread(self,thread): + self._async_raise(thread.ident, SystemExit) + + + def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db0_0', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 3, + 'stbName': 'stb', + 'stbNumbers': 2, + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 200, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + "rowsPerTbl": 1000, + "batchNum": 5000 + } + + dnodeNumbers=int(dnodeNumbers) + mnodeNums=int(mnodeNums) + vnodeNumbers = int(dnodeNumbers-mnodeNums) + allctbNumbers=(paraDict['stbNumbers']*paraDict["ctbNum"]) + rowsPerStb=paraDict["ctbNum"]*paraDict["rowsPerTbl"] + rowsall=rowsPerStb*paraDict['stbNumbers'] + dbNumbers = 1 + + tdLog.info("first check dnode and mnode") + tdSql.query("select * from information_schema.ins_dnodes;") + tdSql.checkData(0,1,'%s:6030'%self.host) + tdSql.checkData(4,1,'%s:6430'%self.host) + clusterComCheck.checkDnodes(dnodeNumbers) + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) + + # add some error operations and + tdLog.info("Confirm the status of the dnode again") + tdSql.error("create mnode on dnode 2") + tdSql.query("select * from information_schema.ins_dnodes;") + print(tdSql.queryResult) + clusterComCheck.checkDnodes(dnodeNumbers) + + # create database and stable + clusterComCreate.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']) + tdLog.info("Take turns stopping Mnodes ") + + tdDnodes=cluster.dnodes + stopcount =0 + threads=[] + + # create stable:stb_0 + stableName= paraDict['stbName'] + newTdSql=tdCom.newTdSql() + clusterComCreate.create_stables(newTdSql, paraDict["dbName"],stableName,paraDict['stbNumbers']) + #create child table:ctb_0 + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + newTdSql=tdCom.newTdSql() + clusterComCreate.create_ctable(newTdSql, paraDict["dbName"],stableName,stableName, paraDict['ctbNum']) + #insert date + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + newTdSql=tdCom.newTdSql() + threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]))) + for tr in threads: + tr.start() + for tr in threads: + tr.join() + + while stopcount < restartNumbers: + tdLog.info(" restart loop: %d"%stopcount ) + if stopRole == "mnode": + for i in range(mnodeNums): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + elif stopRole == "vnode": + for i in range(vnodeNumbers): + tdDnodes[i+mnodeNums].stoptaosd() + # sleep(10) + tdDnodes[i+mnodeNums].starttaosd() + # sleep(10) + elif stopRole == "dnode": + for i in range(dnodeNumbers): + tdDnodes[i].stoptaosd() + clusterComCheck.checkDbRows(dbNumbers) + if i == 0 : + stableName= '%s_%d'%(paraDict['stbName'],0) + newTdSql=tdCom.newTdSql() + clusterComCreate.alterStbMetaData(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"]) + # sleep(10) + tdDnodes[i].starttaosd() + + + # dnodeNumbers don't include database of schema + if clusterComCheck.checkDnodes(dnodeNumbers): + tdLog.info("123") + else: + print("456") + + self.stopThread(threads) + tdLog.exit("one or more of dnodes failed to start ") + # self.check3mnode() + stopcount+=1 + + + clusterComCheck.checkDnodes(dnodeNumbers) + clusterComCheck.checkDbRows(dbNumbers) + # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"]) + + # tdSql.execute("use %s" %(paraDict["dbName"])) + tdSql.query("show %s.stables"%(paraDict["dbName"])) + tdSql.checkRows(paraDict["stbNumbers"]) + for i in range(paraDict['stbNumbers']): + stableName= '%s.%s_%d'%(paraDict["dbName"],paraDict['stbName'],i) + tdSql.query("select count(*) from %s"%stableName) + if i == 0 : + tdSql.checkData(0,0,rowsPerStb*2) + else: + tdSql.checkData(0,0,rowsPerStb) + def run(self): + # print(self.master_dnode.cfgDict) + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=1,stopRole='dnode') + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeRCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeRCreateDb.py new file mode 100644 index 0000000000000000000000000000000000000000..9d99980b88081607d825f8f9198bb3d0487906df --- /dev/null +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeRCreateDb.py @@ -0,0 +1,209 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + +import time +import socket +import subprocess +from multiprocessing import Process +import threading +import time +import inspect +import ctypes + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + self.TDDnodes = None + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.replicaVar=int(replicaVar) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def _async_raise(self, tid, exctype): + """raises the exception, performs cleanup if needed""" + if not inspect.isclass(exctype): + exctype = type(exctype) + res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype)) + if res == 0: + raise ValueError("invalid thread id") + elif res != 1: + # """if it returns a number greater than one, you're in trouble, + # and you should call it again with exc=NULL to revert the effect""" + ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) + raise SystemError("PyThreadState_SetAsyncExc failed") + + def stopThread(self,thread): + self._async_raise(thread.ident, SystemExit) + + + def insertData(self,countstart,countstop): + # fisrt add data : db\stable\childtable\general table + + for couti in range(countstart,countstop): + tdLog.debug("drop database if exists db%d" %couti) + tdSql.execute("drop database if exists db%d" %couti) + print("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("use db%d" %couti) + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + + def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db', + 'dbNumbers': 4, + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 3, + 'stbName': 'stb', + 'stbNumbers': 100, + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 1, + } + + dnodeNumbers=int(dnodeNumbers) + dbNumbers=paraDict['dbNumbers'] + mnodeNums=int(mnodeNums) + repeatNumber = 2 + vnodeNumbers = int(dnodeNumbers-mnodeNums) + allDbNumbers=dbNumbers + allStbNumbers=(paraDict['stbNumbers']*restartNumbers) + paraDict['replica'] = self.replicaVar + + tdLog.info("first check dnode and mnode") + tdSql.query("select * from information_schema.ins_dnodes;") + tdSql.checkData(0,1,'%s:6030'%self.host) + tdSql.checkData(4,1,'%s:6430'%self.host) + clusterComCheck.checkDnodes(dnodeNumbers) + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) + + # add some error operations and + tdLog.info("Confirm the status of the dnode again") + tdSql.error("create mnode on dnode 2") + tdSql.query("select * from information_schema.ins_dnodes;") + print(tdSql.queryResult) + clusterComCheck.checkDnodes(dnodeNumbers) + + # create database and stable + + + tdDnodes=cluster.dnodes + stopcount =0 + threads=[] + for i in range(dbNumbers): + dbNameIndex = '%s%d'%(paraDict["dbName"],0) + newTdSql=tdCom.newTdSql() + # a11111=paraDict["dbNumbers"] + # print(f"==================={dbNameIndex},{a11111}") + clusterComCreate.createDeltedatabases(newTdSql, dbNameIndex,repeatNumber,paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']) + + redbNameIndex = '%s%d'%(paraDict["dbName"],100) + clusterComCreate.createDeltedatabases(newTdSql, redbNameIndex,1,paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']) + + + tdLog.info("Take turns stopping Mnodes ") + while stopcount < restartNumbers: + tdLog.info(" restart loop: %d"%stopcount ) + if stopRole == "mnode": + for i in range(mnodeNums): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + elif stopRole == "vnode": + for i in range(vnodeNumbers): + tdDnodes[i+mnodeNums].stoptaosd() + # sleep(10) + tdDnodes[i+mnodeNums].starttaosd() + # sleep(10) + elif stopRole == "dnode": + for i in range(dnodeNumbers): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + + # dnodeNumbers don't include database of schema + if clusterComCheck.checkDnodes(dnodeNumbers): + tdLog.info("check dnodes status is ready") + else: + tdLog.info("check dnodes status is not ready") + self.stopThread(threads) + tdLog.exit("one or more of dnodes failed to start ") + # self.check3mnode() + stopcount+=1 + + + tdLog.info("check dnode number:") + clusterComCheck.checkDnodes(dnodeNumbers) + tdSql.query("select * from information_schema.ins_databases") + tdLog.debug("we find %d databases but exepect to create %d databases "%(tdSql.queryRows-2,allDbNumbers)) + + # tdLog.info("check DB Rows:") + # clusterComCheck.checkDbRows(allDbNumbers) + # tdLog.info("check DB Status on by on") + # for i in range(restartNumbers): + # clusterComCheck.checkDb(paraDict['dbNumbers'],restartNumbers,dbNameIndex = '%s%d'%(paraDict["dbName"],i)) + + + + def run(self): + # print(self.master_dnode.cfgDict) + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=2,stopRole='dnode') + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeModifyMeta.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeModifyMeta.py new file mode 100644 index 0000000000000000000000000000000000000000..3e4dc2483f2222fe9f5cd04c2e4dc529d1e83c7d --- /dev/null +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeModifyMeta.py @@ -0,0 +1,207 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +from numpy import row_stack +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + +import time +import socket +import subprocess +from multiprocessing import Process +import threading +import time +import inspect +import ctypes + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + self.TDDnodes = None + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def _async_raise(self, tid, exctype): + """raises the exception, performs cleanup if needed""" + if not inspect.isclass(exctype): + exctype = type(exctype) + res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype)) + if res == 0: + raise ValueError("invalid thread id") + elif res != 1: + # """if it returns a number greater than one, you're in trouble, + # and you should call it again with exc=NULL to revert the effect""" + ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) + raise SystemError("PyThreadState_SetAsyncExc failed") + + def stopThread(self,thread): + self._async_raise(thread.ident, SystemExit) + + + def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db0_0', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 3, + 'stbName': 'stb', + 'stbNumbers': 2, + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 200, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + "rowsPerTbl": 1000, + "batchNum": 5000 + } + + dnodeNumbers=int(dnodeNumbers) + mnodeNums=int(mnodeNums) + vnodeNumbers = int(dnodeNumbers-mnodeNums) + allctbNumbers=(paraDict['stbNumbers']*paraDict["ctbNum"]) + rowsPerStb=paraDict["ctbNum"]*paraDict["rowsPerTbl"] + rowsall=rowsPerStb*paraDict['stbNumbers'] + dbNumbers = 1 + + tdLog.info("first check dnode and mnode") + tdSql.query("select * from information_schema.ins_dnodes;") + tdSql.checkData(0,1,'%s:6030'%self.host) + tdSql.checkData(4,1,'%s:6430'%self.host) + clusterComCheck.checkDnodes(dnodeNumbers) + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) + + # add some error operations and + tdLog.info("Confirm the status of the dnode again") + tdSql.error("create mnode on dnode 2") + tdSql.query("select * from information_schema.ins_dnodes;") + print(tdSql.queryResult) + clusterComCheck.checkDnodes(dnodeNumbers) + + # create database and stable + clusterComCreate.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']) + tdLog.info("Take turns stopping Mnodes ") + + tdDnodes=cluster.dnodes + stopcount =0 + threads=[] + + # create stable:stb_0 + stableName= paraDict['stbName'] + newTdSql=tdCom.newTdSql() + clusterComCreate.create_stables(newTdSql, paraDict["dbName"],stableName,paraDict['stbNumbers']) + #create child table:ctb_0 + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + newTdSql=tdCom.newTdSql() + clusterComCreate.create_ctable(newTdSql, paraDict["dbName"],stableName,stableName, paraDict['ctbNum']) + #insert date + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + newTdSql=tdCom.newTdSql() + threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]))) + for tr in threads: + tr.start() + for tr in threads: + tr.join() + + while stopcount < restartNumbers: + tdLog.info(" restart loop: %d"%stopcount ) + if stopRole == "mnode": + for i in range(mnodeNums): + tdDnodes[i].stoptaosd() + if i == 0 : + stableName= '%s_%d'%(paraDict['stbName'],0) + newTdSql=tdCom.newTdSql() + clusterComCreate.alterStbMetaData(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"]) + elif i == 1 : + tdSql.execute("ALTER TABLE db0_0.stb_0_0 SET TAG t1r=10000;") + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + elif stopRole == "vnode": + for i in range(vnodeNumbers): + tdDnodes[i+mnodeNums].stoptaosd() + # sleep(10) + tdDnodes[i+mnodeNums].starttaosd() + # sleep(10) + elif stopRole == "dnode": + for i in range(dnodeNumbers): + tdDnodes[i].stoptaosd() + clusterComCheck.checkDbRows(dbNumbers) + + # sleep(10) + tdDnodes[i].starttaosd() + + + # dnodeNumbers don't include database of schema + if clusterComCheck.checkDnodes(dnodeNumbers): + tdLog.info("123") + else: + print("456") + + self.stopThread(threads) + tdLog.exit("one or more of dnodes failed to start ") + # self.check3mnode() + stopcount+=1 + + + clusterComCheck.checkDnodes(dnodeNumbers) + clusterComCheck.checkDbRows(dbNumbers) + # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"]) + + # tdSql.execute("use %s" %(paraDict["dbName"])) + tdSql.query("select t1r from db0_0.stb_0_0 limit 1;") + tdSql.checkData(0,0,10000) + tdSql.query("show %s.stables"%(paraDict["dbName"])) + tdSql.checkRows(paraDict["stbNumbers"]) + for i in range(paraDict['stbNumbers']): + stableName= '%s.%s_%d'%(paraDict["dbName"],paraDict['stbName'],i) + tdSql.query("select count(*) from %s"%stableName) + if i == 0 : + tdSql.checkData(0,0,rowsPerStb*2) + else: + tdSql.checkData(0,0,rowsPerStb) + def run(self): + # print(self.master_dnode.cfgDict) + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=1,stopRole='mnode') + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeSepVnodeStopDnodeCreateUser.py b/tests/system-test/6-cluster/5dnode3mnodeSepVnodeStopDnodeCreateUser.py new file mode 100644 index 0000000000000000000000000000000000000000..94e02b77b307281975ff7393298f182f233a7c65 --- /dev/null +++ b/tests/system-test/6-cluster/5dnode3mnodeSepVnodeStopDnodeCreateUser.py @@ -0,0 +1,224 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +from numpy import row_stack +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + +import time +import socket +import subprocess +from multiprocessing import Process +import threading +import time +import inspect +import ctypes + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + self.TDDnodes = None + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.replicaVar = int(replicaVar) + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def _async_raise(self, tid, exctype): + """raises the exception, performs cleanup if needed""" + if not inspect.isclass(exctype): + exctype = type(exctype) + res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype)) + if res == 0: + raise ValueError("invalid thread id") + elif res != 1: + # """if it returns a number greater than one, you're in trouble, + # and you should call it again with exc=NULL to revert the effect""" + ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) + raise SystemError("PyThreadState_SetAsyncExc failed") + + def stopThread(self,thread): + self._async_raise(thread.ident, SystemExit) + + + def insertData(self,countstart,countstop): + # fisrt add data : db\stable\childtable\general table + + for couti in range(countstart,countstop): + tdLog.debug("drop database if exists db%d" %couti) + tdSql.execute("drop database if exists db%d" %couti) + print("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("use db%d" %couti) + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + + def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db0_0', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'stbNumbers': 2, + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 200, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + "rowsPerTbl": 100, + "batchNum": 5000 + } + + dnodeNumbers=int(dnodeNumbers) + mnodeNums=int(mnodeNums) + vnodeNumbers = int(dnodeNumbers-mnodeNums) + allctbNumbers=(paraDict['stbNumbers']*paraDict["ctbNum"]) + rowsPerStb=paraDict["ctbNum"]*paraDict["rowsPerTbl"] + rowsall=rowsPerStb*paraDict['stbNumbers'] + dbNumbers = 1 + paraDict['replica'] = self.replicaVar + + tdLog.info("first check dnode and mnode") + tdSql.query("select * from information_schema.ins_dnodes;") + tdSql.checkData(0,1,'%s:6030'%self.host) + tdSql.checkData(4,1,'%s:6430'%self.host) + clusterComCheck.checkDnodes(dnodeNumbers) + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) + + + # add some error operations and + tdLog.info("Confirm the status of the dnode again") + tdSql.error("create mnode on dnode 2") + tdSql.query("select * from information_schema.ins_dnodes;") + print(tdSql.queryResult) + clusterComCheck.checkDnodes(dnodeNumbers) + + # create database and stable + clusterComCreate.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']) + tdLog.info("Take turns stopping Mnodes ") + + tdDnodes=cluster.dnodes + stopcount =0 + threads=[] + + # create stable:stb_0 + stableName= paraDict['stbName'] + newTdSql=tdCom.newTdSql() + clusterComCreate.create_stables(newTdSql, paraDict["dbName"],stableName,paraDict['stbNumbers']) + #create child table:ctb_0 + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + newTdSql=tdCom.newTdSql() + clusterComCreate.create_ctable(newTdSql, paraDict["dbName"],stableName,stableName, paraDict['ctbNum']) + #insert data + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + newTdSql=tdCom.newTdSql() + threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]))) + for tr in threads: + tr.start() + for tr in threads: + tr.join() + + while stopcount < restartNumbers: + tdLog.info(" restart loop: %d"%stopcount ) + if stopRole == "mnode": + for i in range(mnodeNums): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + elif stopRole == "vnode": + for i in range(vnodeNumbers): + tdDnodes[i+mnodeNums].stoptaosd() + # sleep(10) + tdDnodes[i+mnodeNums].starttaosd() + # sleep(10) + elif stopRole == "dnode": + for i in range(dnodeNumbers): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + + # dnodeNumbers don't include database of schema + if clusterComCheck.checkDnodes(dnodeNumbers): + tdLog.info("dnode is ready") + else: + print("dnodes is not ready") + self.stopThread(threads) + tdLog.exit("one or more of dnodes failed to start ") + # self.check3mnode() + stopcount+=1 + + + clusterComCheck.checkDnodes(dnodeNumbers) + clusterComCheck.checkDbRows(dbNumbers) + # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"]) + + newTdSql=tdCom.newTdSql() + newTdSql.execute("reset query cache") + newTdSql.execute("use %s" %(paraDict["dbName"])) + newTdSql.query("show %s.stables"%(paraDict["dbName"])) + newTdSql.checkRows(paraDict["stbNumbers"]) + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + newTdSql.query("select * from %s"%stableName) + newTdSql.checkRows(rowsPerStb) + + def run(self): + # print(self.master_dnode.cfgDict) + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=2,stopRole='dnode') + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py b/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py deleted file mode 100644 index a34895ff42c94e5b4b201b0edbaf2d782eee3f60..0000000000000000000000000000000000000000 --- a/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py +++ /dev/null @@ -1,324 +0,0 @@ -from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE -import taos -import sys -import time -import os - -from util.log import * -from util.sql import * -from util.cases import * -from util.dnodes import TDDnodes -from util.dnodes import TDDnode -from util.cluster import * -from util.common import * -sys.path.append("./7-tmq") -from tmqCommon import * - -import time -import socket -import subprocess -from multiprocessing import Process -import threading -import time -import inspect -import ctypes - -class TDTestCase: - - def init(self, conn, logSql, replicaVar=1): - tdLog.debug(f"start to excute {__file__}") - # tdSql.init(conn.cursor()) - # self.host = socket.gethostname() - - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root) - len("/build/bin")] - break - return buildPath - - def _async_raise(self, tid, exctype): - """raises the exception, performs cleanup if needed""" - if not inspect.isclass(exctype): - exctype = type(exctype) - res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype)) - if res == 0: - raise ValueError("invalid thread id") - elif res != 1: - # """if it returns a number greater than one, you're in trouble, - # and you should call it again with exc=NULL to revert the effect""" - ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) - raise SystemError("PyThreadState_SetAsyncExc failed") - - def stop_thread(self,thread): - self._async_raise(thread.ident, SystemExit) - - - def insert_data(self,countstart,countstop): - # fisrt add data : db\stable\childtable\general table - - for couti in range(countstart,countstop): - tdLog.debug("drop database if exists db%d" %couti) - tdSql.execute("drop database if exists db%d" %couti) - print("create database if not exists db%d replica 1 duration 300" %couti) - tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) - tdSql.execute("use db%d" %couti) - tdSql.execute( - '''create table stb1 - (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) - tags (t1 int) - ''' - ) - tdSql.execute( - ''' - create table t1 - (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) - ''' - ) - for i in range(4): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') - - def checkData(self,dbname,stbname,stableCount,CtableCount,rowsPerSTable,): - tdSql.execute("use %s"%dbname) - tdSql.query("show stables") - tdSql.checkRows(stableCount) - tdSql.query("show tables") - tdSql.checkRows(CtableCount) - for i in range(stableCount): - tdSql.query("select count(*) from %s%d"%(stbname,i)) - tdSql.checkData(0,0,rowsPerSTable) - return - - def checkdnodes(self,dnodenumber): - count=0 - while count < 100: - time.sleep(1) - statusReadyBumber=0 - tdSql.query("select * from information_schema.ins_dnodes;") - if tdSql.checkRows(dnodenumber) : - print("dnode is %d nodes"%dnodenumber) - for i in range(dnodenumber): - if tdSql.queryResult[i][4] !='ready' : - status=tdSql.queryResult[i][4] - print("dnode:%d status is %s "%(i,status)) - break - else: - statusReadyBumber+=1 - print(statusReadyBumber) - if statusReadyBumber == dnodenumber : - print("all of %d mnodes is ready in 10s "%dnodenumber) - return True - break - count+=1 - else: - print("%d mnodes is not ready in 10s "%dnodenumber) - return False - - - def check3mnode(self): - count=0 - while count < 10: - time.sleep(1) - tdSql.query("select * from information_schema.ins_mnodes;") - if tdSql.checkRows(3) : - print("mnode is three nodes") - if tdSql.queryResult[0][2]=='leader' : - if tdSql.queryResult[1][2]=='follower': - if tdSql.queryResult[2][2]=='follower': - print("three mnodes is ready in 10s") - break - elif tdSql.queryResult[0][2]=='follower' : - if tdSql.queryResult[1][2]=='leader': - if tdSql.queryResult[2][2]=='follower': - print("three mnodes is ready in 10s") - break - elif tdSql.queryResult[0][2]=='follower' : - if tdSql.queryResult[1][2]=='follower': - if tdSql.queryResult[2][2]=='leader': - print("three mnodes is ready in 10s") - break - count+=1 - else: - print("three mnodes is not ready in 10s ") - return -1 - - tdSql.query("select * from information_schema.ins_mnodes;") - tdSql.checkRows(3) - tdSql.checkData(0,1,'%s:6030'%self.host) - tdSql.checkData(0,3,'ready') - tdSql.checkData(1,1,'%s:6130'%self.host) - tdSql.checkData(1,3,'ready') - tdSql.checkData(2,1,'%s:6230'%self.host) - tdSql.checkData(2,3,'ready') - - def check3mnode1off(self): - count=0 - while count < 10: - time.sleep(1) - tdSql.query("select * from information_schema.ins_mnodes;") - if tdSql.checkRows(3) : - print("mnode is three nodes") - if tdSql.queryResult[0][2]=='offline' : - if tdSql.queryResult[1][2]=='leader': - if tdSql.queryResult[2][2]=='follower': - print("stop mnodes on dnode 2 successfully in 10s") - break - elif tdSql.queryResult[1][2]=='follower': - if tdSql.queryResult[2][2]=='leader': - print("stop mnodes on dnode 2 successfully in 10s") - break - count+=1 - else: - print("stop mnodes on dnode 2 failed in 10s ") - return -1 - tdSql.error("drop mnode on dnode 1;") - - tdSql.query("select * from information_schema.ins_mnodes;") - tdSql.checkRows(3) - tdSql.checkData(0,1,'%s:6030'%self.host) - tdSql.checkData(0,2,'offline') - tdSql.checkData(0,3,'ready') - tdSql.checkData(1,1,'%s:6130'%self.host) - tdSql.checkData(1,3,'ready') - tdSql.checkData(2,1,'%s:6230'%self.host) - tdSql.checkData(2,3,'ready') - - def check3mnode2off(self): - count=0 - while count < 40: - time.sleep(1) - tdSql.query("select * from information_schema.ins_mnodes;") - if tdSql.checkRows(3) : - print("mnode is three nodes") - if tdSql.queryResult[0][2]=='leader' : - if tdSql.queryResult[1][2]=='offline': - if tdSql.queryResult[2][2]=='follower': - print("stop mnodes on dnode 2 successfully in 10s") - break - count+=1 - else: - print("stop mnodes on dnode 2 failed in 10s ") - return -1 - tdSql.error("drop mnode on dnode 2;") - - tdSql.query("select * from information_schema.ins_mnodes;") - tdSql.checkRows(3) - tdSql.checkData(0,1,'%s:6030'%self.host) - tdSql.checkData(0,2,'leader') - tdSql.checkData(0,3,'ready') - tdSql.checkData(1,1,'%s:6130'%self.host) - tdSql.checkData(1,2,'offline') - tdSql.checkData(1,3,'ready') - tdSql.checkData(2,1,'%s:6230'%self.host) - tdSql.checkData(2,2,'follower') - tdSql.checkData(2,3,'ready') - - def check3mnode3off(self): - count=0 - while count < 10: - time.sleep(1) - tdSql.query("select * from information_schema.ins_mnodes;") - if tdSql.checkRows(3) : - print("mnode is three nodes") - if tdSql.queryResult[0][2]=='leader' : - if tdSql.queryResult[2][2]=='offline': - if tdSql.queryResult[1][2]=='follower': - print("stop mnodes on dnode 3 successfully in 10s") - break - count+=1 - else: - print("stop mnodes on dnode 3 failed in 10s") - return -1 - tdSql.error("drop mnode on dnode 3;") - tdSql.query("select * from information_schema.ins_mnodes;") - tdSql.checkRows(3) - tdSql.checkData(0,1,'%s:6030'%self.host) - tdSql.checkData(0,2,'leader') - tdSql.checkData(0,3,'ready') - tdSql.checkData(1,1,'%s:6130'%self.host) - tdSql.checkData(1,2,'follower') - tdSql.checkData(1,3,'ready') - tdSql.checkData(2,1,'%s:6230'%self.host) - tdSql.checkData(2,2,'offline') - tdSql.checkData(2,3,'ready') - - - def check5dnode(self): - tdSql.query("select * from information_schema.ins_dnodes;") - tdSql.checkData(0,1,'%s:6030'%self.host) - tdSql.checkData(4,1,'%s:6430'%self.host) - tdSql.checkData(0,4,'ready') - tdSql.checkData(4,4,'ready') - - def five_dnode_three_mnode(self,dnodenumber): - tdSql.query("select * from information_schema.ins_dnodes;") - tdSql.checkData(0,1,'%s:6030'%self.host) - tdSql.checkData(4,1,'%s:6430'%self.host) - tdSql.checkData(0,4,'ready') - tdSql.checkData(4,4,'ready') - tdSql.query("select * from information_schema.ins_mnodes;") - tdSql.checkRows(1) - tdSql.checkData(0,1,'%s:6030'%self.host) - tdSql.checkData(0,2,'leader') - tdSql.checkData(0,3,'ready') - - # fisr add three mnodes; - tdSql.execute("create mnode on dnode 2") - tdSql.execute("create mnode on dnode 3") - - # fisrt check statut ready - self.check3mnode() - - tdSql.error("create mnode on dnode 2") - tdSql.query("select * from information_schema.ins_dnodes;") - print(tdSql.queryResult) - tdLog.debug("stop all of mnode ") - - # seperate vnode and mnode in different dnodes. - # create database and stable - stopcount =0 - while stopcount < 2: - for i in range(dnodenumber): - # threads=[] - # threads = MyThreadFunc(self.insert_data(i*2,i*2+2)) - threads=threading.Thread(target=self.insert_data, args=(i,i+1)) - threads.start() - self.TDDnodes.stoptaosd(i+1) - self.TDDnodes.starttaosd(i+1) - - if self.checkdnodes(5): - print("123") - threads.join() - else: - print("456") - threads.join() - self.stop_thread(threads) - assert 1 == 2 ,"some dnode started failed" - return False - # self.check3mnode() - self.check3mnode() - - - stopcount+=1 - self.check3mnode() - - - def run(self): - # print(self.master_dnode.cfgDict) - self.five_dnode_three_mnode(5) - - def stop(self): - tdSql.close() - tdLog.success(f"{__file__} successfully executed") - -tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/6-cluster/clusterCommonCheck.py b/tests/system-test/6-cluster/clusterCommonCheck.py index a69d9707e41ccc5f67f011999154cf59ac7de012..149c6d8ded3b1ce81dd80d7f4208b43c74acd78f 100644 --- a/tests/system-test/6-cluster/clusterCommonCheck.py +++ b/tests/system-test/6-cluster/clusterCommonCheck.py @@ -64,7 +64,7 @@ class ClusterComCheck: dbNumbers=int(dbNumbers) count=0 while count < 5: - tdSql.query("select * from information_schema.ins_databases;") + tdSql.query("select * from information_schema.ins_databases where name!='collectd' ;") count+=1 if tdSql.checkRows(dbNumbers+2): tdLog.success("we find %d databases and expect %d in clusters! " %(tdSql.queryRows,dbNumbers+2)) diff --git a/tests/system-test/6-cluster/clusterCommonCreate.py b/tests/system-test/6-cluster/clusterCommonCreate.py index 236708cf072b72fed0381311e770252f58b82889..6e699e2396ffba7955cd9f988ab6c6cd186ffc1d 100644 --- a/tests/system-test/6-cluster/clusterCommonCreate.py +++ b/tests/system-test/6-cluster/clusterCommonCreate.py @@ -137,12 +137,24 @@ class ClusterComCreate: # for i in range(dbNumbers): for i in range(dbNumbers): if dropFlag == 1: - tsql.execute("drop database if exists %s_%d"%(dbNameIndex,1)) - tdLog.debug("create database if not exists %s_%d vgroups %d replica %d"%(dbNameIndex,1, vgroups, replica)) - tsql.execute("create database if not exists %s_%d vgroups %d replica %d"%(dbNameIndex,1, vgroups, replica)) - tdLog.debug("complete to create database %s_%d"%(dbNameIndex,1)) + tsql.execute("drop database if exists %s_%d"%(dbNameIndex,i)) + tdLog.debug("create database if not exists %s_%d vgroups %d replica %d"%(dbNameIndex,i, vgroups, replica)) + tsql.execute("create database if not exists %s_%d vgroups %d replica %d"%(dbNameIndex,i, vgroups, replica)) + tdLog.debug("complete to create database %s_%d"%(dbNameIndex,i)) + def createUser(self,tsql,user,password): + tdLog.info(f"create new user f{user}") + tsql.execute(f"CREATE USER {user} PASS '{password}';") + def alterUser(self,tsql,user,password): + tdLog.info(f"alter user {user} pass '{password}'") + tsql.execute(f"alter USER {user} pass '{password}' ;") + + def deleteUser(self,tsql,user): + tdLog.info(f"drop user f{user}") + tsql.execute(f"DROP USER {user} ;") + + def create_stable(self,tsql, dbName,stbName): tsql.execute("create table if not exists %s.%s (ts timestamp, c1 int, c2 int, c3 binary(16)) tags(t1 int, t2 binary(32))"%(dbName, stbName)) tdLog.debug("complete to create %s.%s" %(dbName, stbName)) @@ -202,6 +214,52 @@ class ClusterComCreate: tdLog.debug("insert data ............ [OK]") return + def alterStbMetaData(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=None): + tdLog.debug("alter Stb column ............") + tdLog.debug(f"ALTER STABLE {dbName}.{stbName} MODIFY COLUMN c3 binary(20);") + tsql.execute(f" ALTER STABLE {dbName}.{stbName} MODIFY COLUMN c3 binary(20);") + tdLog.debug(f"ALTER STABLE {dbName}.{stbName} ADD COLUMN c4 DOUBLE;") + tsql.execute(f" ALTER STABLE {dbName}.{stbName} ADD COLUMN c4 DOUBLE;") + tdLog.debug(f"ALTER STABLE {dbName}.{stbName} DROP COLUMN c2;") + tsql.execute(f" ALTER STABLE {dbName}.{stbName} DROP COLUMN c2;") + tdLog.debug(f"ALTER STABLE {dbName}.{stbName} RENAME TAG t1 t1r;") + tsql.execute(f" ALTER STABLE {dbName}.{stbName} RENAME TAG t1 t1r;") + tdLog.debug(f"ALTER STABLE {dbName}.{stbName} DROP TAG t2;") + tsql.execute(f" ALTER STABLE {dbName}.{stbName} DROP TAG t2;") + tdLog.debug(f"ALTER STABLE {dbName}.{stbName} ADD TAG t2 binary(32) ;") + tsql.execute(f" ALTER STABLE {dbName}.{stbName} ADD TAG t2 binary(32);") + tdLog.debug(f"ALTER STABLE {dbName}.{stbName} MODIFY TAG t2 binary(34) ;") + tsql.execute(f" ALTER STABLE {dbName}.{stbName} MODIFY TAG t2 binary(34);") + tdLog.debug(f"ALTER STABLE {dbName}.{stbName} ADD TAG t3 double ;") + tsql.execute(f" ALTER STABLE {dbName}.{stbName} ADD TAG t3 double;") + tsql.error(f" ALTER STABLE {dbName}.{stbName} ADD TAG t2 double;") + + tdLog.debug("start to insert data ............") + # tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + if startTs is None: + t = time.time() + startTs = int(round(t * 1000)) + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + for i in range(ctbNum): + sql += " %s.%s_%d values "%(dbName,stbName,i) + for j in range(rowsPerTbl): + sql += "(%d, %d,'mnode_%d', %d ) "%(startTs + j, j, j,j) + if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + if j < rowsPerTbl - 1: + sql = "insert into %s.%s_%d values " %(dbName,stbName,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + def insert_data_1(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs): tdLog.debug("start to insert data ............") tsql.execute("use %s" %dbName) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb.py b/tests/system-test/7-tmq/tmqConsFromTsdb.py index 9bb8c4cc0d953d4c03fa772427523b477896a90d..8ed4a6df973b57f7302d5a2c193debffbf7286a1 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb.py @@ -130,7 +130,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) - tmqCom.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) time.sleep(10) for i in range(len(topicNameList)): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py index 009862137f8ccbb45f0f9bbfa134ae1a5b0479f1..4dcc0b963f8e883326132f5db3bc04a6edd349d1 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py @@ -116,7 +116,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) consumerId = 4 @@ -188,7 +188,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor 0") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py index 528b3a80887c3cc62b1092e5183249e962f5e8db..da8ac6c57deaf99d9871134489b290381b570306 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py @@ -116,7 +116,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) consumerId = 4 @@ -188,7 +188,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor 0") diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index f2fbd8486570e385332be5280f58f25af7ae4d75..7ca18f293cddf415a2b4ac3354f5848171e7b736 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -123,6 +123,11 @@ class TDTestCase: def checkData(self): tdSql.execute('use db_taosx') + tdSql.query("select * from tb1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 0) + tdSql.checkData(0, 2, 1) + tdSql.query("select * from ct3 order by c1 desc") tdSql.checkRows(2) tdSql.checkData(0, 1, 51) diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index 1e7d0ed1406d9fc251ffdb6319fede131febef78..342d6410d29f1b94d80b508c05507a0f5b81cdec 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -26,6 +26,10 @@ ELSE () SET(LINK_WEBSOCKET "") ENDIF () +IF (CUS_NAME OR CUS_PROMPT OR CUS_EMAIL) + ADD_DEFINITIONS(-I${CMAKE_CURRENT_SOURCE_DIR}/../../../enterprise/packaging) +ENDIF (CUS_NAME OR CUS_PROMPT OR CUS_EMAIL) + if(TD_WINDOWS) target_link_libraries(shell PUBLIC taos_static ${LINK_WEBSOCKET}) else() diff --git a/tools/shell/inc/shellAuto.h b/tools/shell/inc/shellAuto.h index b7bf5fa1019502acbeaefc8884d4553704f58702..f3ea87e4a540aa34084e0c3c5d91ad58bbae1adf 100644 --- a/tools/shell/inc/shellAuto.h +++ b/tools/shell/inc/shellAuto.h @@ -24,13 +24,13 @@ void pressTabKey(SShellCmd* cmd); // press othr key void pressOtherKey(char c); -// init shell auto funciton , shell start call once +// init shell auto function , shell start call once bool shellAutoInit(); // set conn void shellSetConn(TAOS* conn); -// exit shell auto funciton, shell exit call once +// exit shell auto function, shell exit call once void shellAutoExit(); // callback autotab module diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h index af724c1533177c2203d58b6f5cd1f02fc236f5c4..6b3bc56dc767712b3607075aa07fafb85530d39b 100644 --- a/tools/shell/inc/shellInt.h +++ b/tools/shell/inc/shellInt.h @@ -80,8 +80,9 @@ typedef struct { } SShellArgs; typedef struct { - const char* clientVersion; - const char* promptHeader; + const char *clientVersion; + char cusName[32]; + char promptHeader[32]; const char* promptContinue; const char* osname; int32_t promptSize; diff --git a/tools/shell/inc/shellTire.h b/tools/shell/inc/shellTire.h index bdcf7bcfb3310832b60cc66af5d07c16b7eb6134..e87c3ee4f33726171d107544a137bac7b649174a 100644 --- a/tools/shell/inc/shellTire.h +++ b/tools/shell/inc/shellTire.h @@ -19,7 +19,7 @@ // // The prefix search tree is a efficient storage words and search words tree, it support 95 visible ascii code character // -#define FIRST_ASCII 40 // first visiable char is '0' +#define FIRST_ASCII 40 // first visible char is '0' #define LAST_ASCII 122 // last visilbe char is 'z' // capacity save char is 95 diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 4d40de66bde2c8504369110c436275b27268ac08..d899249b97f0160d366db385653c1c65a9864d9a 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -19,10 +19,25 @@ #include "shellInt.h" -#define TAOS_CONSOLE_PROMPT_HEADER "taos> " +#ifndef CUS_NAME + char cusName[] = "TDengine"; +#endif + +#ifndef CUS_PROMPT + char cusPrompt[] = "taos"; +#endif + +#ifndef CUS_EMAIL + char cusEmail[] = ""; +#endif + +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#endif + #define TAOS_CONSOLE_PROMPT_CONTINUE " -> " -#define SHELL_HOST "TDengine server FQDN to connect. The default host is localhost." +#define SHELL_HOST "The server FQDN to connect. The default host is localhost." #define SHELL_PORT "The TCP/IP port number to use for the connection." #define SHELL_USER "The user name to use when connecting to the server." #define SHELL_PASSWORD "The password to use when connecting to the server." @@ -41,7 +56,6 @@ #define SHELL_PKT_LEN "Packet length used for net test, default is 1024 bytes." #define SHELL_PKT_NUM "Packet numbers used for net test, default is 100." #define SHELL_VERSION "Print program version." -#define SHELL_EMAIL "" #ifdef WEBSOCKET #define SHELL_DSN "The dsn to use when connecting to cloud server." @@ -78,7 +92,7 @@ void shellPrintHelp() { #endif printf("%s%s%s%s\r\n", indent, "-w,", indent, SHELL_WIDTH); printf("%s%s%s%s\r\n", indent, "-V,", indent, SHELL_VERSION); - printf("\r\n\r\nReport bugs to %s.\r\n", SHELL_EMAIL); + printf("\r\n\r\nReport bugs to %s.\r\n", cusEmail); } #ifdef LINUX @@ -86,7 +100,7 @@ void shellPrintHelp() { #include const char *argp_program_version = version; -const char *argp_program_bug_address = SHELL_EMAIL; +const char *argp_program_bug_address = cusEmail; static struct argp_option shellOptions[] = { {"host", 'h', "HOST", 0, SHELL_HOST}, @@ -388,12 +402,13 @@ static int32_t shellCheckArgs() { int32_t shellParseArgs(int32_t argc, char *argv[]) { shellInitArgs(argc, argv); - shell.info.clientVersion = - "Welcome to the TDengine Command Line Interface, Client Version:%s\r\n" - "Copyright (c) 2022 by TDengine, all rights reserved.\r\n\r\n"; - shell.info.promptHeader = TAOS_CONSOLE_PROMPT_HEADER; + shell.info.clientVersion = + "Welcome to the %s Command Line Interface, Client Version:%s\r\n" + "Copyright (c) 2022 by %s, all rights reserved.\r\n\r\n"; + strcpy(shell.info.cusName, cusName); + sprintf(shell.info.promptHeader, "%s> ", cusPrompt); shell.info.promptContinue = TAOS_CONSOLE_PROMPT_CONTINUE; - shell.info.promptSize = 6; + shell.info.promptSize = strlen(shell.info.promptHeader); snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), "version: %s", version); #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index b391d59725938295fcffba13dfac81821a40a0cb..81af5d7fe8e860a242256160aa133124d2cf2503 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -138,6 +138,7 @@ SWords shellCommands[] = { {"show create table \\G;", 0, 0, NULL}, {"show connections;", 0, 0, NULL}, {"show cluster;", 0, 0, NULL}, + {"show cluster alive;", 0, 0, NULL}, {"show databases;", 0, 0, NULL}, {"show dnodes;", 0, 0, NULL}, {"show dnode variables;", 0, 0, NULL}, @@ -263,7 +264,7 @@ char* key_tags[] = {"tags("}; char* key_select[] = {"select "}; // -// ------- gobal variant define --------- +// ------- global variant define --------- // int32_t firstMatchIndex = -1; // first match shellCommands index int32_t lastMatchIndex = -1; // last match shellCommands index @@ -328,7 +329,15 @@ int cntDel = 0; // delete byte count after next press tab // show auto tab introduction void printfIntroduction() { printf(" ****************************** Tab Completion **********************************\n"); - printf(" * The TDengine CLI supports tab completion for a variety of items, *\n"); + char secondLine[160] = "\0"; + sprintf(secondLine, " * The %s CLI supports tab completion for a variety of items, ", + shell.info.cusName); + printf("%s", secondLine); + int secondLineLen = strlen(secondLine); + while (84-(secondLineLen++) > 0) { + printf(" "); + } + printf("*\n"); printf(" * including database names, table names, function names and keywords. *\n"); printf(" * The full list of shortcut keys is as follows: *\n"); printf(" * [ TAB ] ...... complete the current word *\n"); @@ -343,7 +352,7 @@ void printfIntroduction() { } void showHelp() { - printf("\nThe TDengine CLI supports the following commands:"); + printf("\nThe %s CLI supports the following commands:", shell.info.cusName); printf( "\n\ ----- A ----- \n\ @@ -425,6 +434,7 @@ void showHelp() { show create table ;\n\ show connections;\n\ show cluster;\n\ + show cluster alive;\n\ show databases;\n\ show dnodes;\n\ show dnode variables;\n\ @@ -593,7 +603,7 @@ void GenerateVarType(int type, char** p, int count) { // -------------------- shell auto ---------------- // -// init shell auto funciton , shell start call once +// init shell auto function , shell start call once bool shellAutoInit() { // command int32_t count = SHELL_COMMAND_COUNT(); @@ -626,7 +636,7 @@ bool shellAutoInit() { // set conn void shellSetConn(TAOS* conn) { varCon = conn; } -// exit shell auto funciton, shell exit call once +// exit shell auto function, shell exit call once void shellAutoExit() { // free command int32_t count = SHELL_COMMAND_COUNT(); @@ -643,7 +653,7 @@ void shellAutoExit() { } } taosThreadMutexUnlock(&tiresMutex); - // destory + // destroy taosThreadMutexDestroy(&tiresMutex); // free threads @@ -664,7 +674,7 @@ void shellAutoExit() { // // ------------------- auto ptr for tires -------------------------- // -bool setNewAuotPtr(int type, STire* pNew) { +bool setNewAutoPtr(int type, STire* pNew) { if (pNew == NULL) return false; taosThreadMutexLock(&tiresMutex); @@ -707,13 +717,13 @@ void putBackAutoPtr(int type, STire* tire) { if (tires[type] != tire) { // update by out, can't put back , so free if (--tire->ref == 1) { - // support multi thread getAuotPtr + // support multi thread getAutoPtr freeTire(tire); } } else { tires[type]->ref--; - assert(tires[type]->ref > 0); + ASSERT(tires[type]->ref > 0); } taosThreadMutexUnlock(&tiresMutex); @@ -762,7 +772,7 @@ int writeVarNames(int type, TAOS_RES* tres) { } while (row != NULL); // replace old tire - setNewAuotPtr(type, tire); + setNewAutoPtr(type, tire); return numOfRows; } @@ -1030,7 +1040,7 @@ SWords* matchCommand(SWords* input, bool continueSearch) { for (int32_t i = 0; i < count; i++) { SWords* shellCommand = shellCommands + i; if (continueSearch && lastMatchIndex != -1 && i <= lastMatchIndex) { - // new match must greate than lastMatchIndex + // new match must greater than lastMatchIndex if (varMode && i == lastMatchIndex) { // do nothing, var match on lastMatchIndex } else { @@ -1159,7 +1169,7 @@ void createInputFromFirst(SWords* input, SWords* firstMatch) { for (int i = 0; i < firstMatch->matchIndex && word; i++) { // combine source from each word strncpy(input->source + input->source_len, word->word, word->len); - strcat(input->source, " "); // append blank splite + strcat(input->source, " "); // append blank space input->source_len += word->len + 1; // 1 is blank length // move next word = word->next; @@ -1388,7 +1398,7 @@ bool appendAfterSelect(TAOS* con, SShellCmd* cmd, char* sql, int32_t len) { return true; } - // fill funciton + // fill function if (fieldEnd) { // fields is end , need match keyword ret = fillWithType(con, cmd, last, WT_VAR_KEYWORD); @@ -1571,7 +1581,7 @@ bool matchCreateTable(TAOS* con, SShellCmd* cmd) { // tb options if (!ret) { - // find like create talbe st (...) tags(..) + // find like create table st (...) tags(..) char* p1 = strchr(ps, ')'); // first ')' end if (p1) { if (strchr(p1 + 1, ')')) { // second ')' end diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index 5235030cf91cac5d41b782544da9d8a7be4e1349..0e305f57e9c781a8149db1fecccd06a703bc1465 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -40,7 +40,7 @@ static void shellPositionCursorEnd(SShellCmd *cmd); static void shellPrintChar(char c, int32_t times); static void shellPositionCursor(int32_t step, int32_t direction); static void shellUpdateBuffer(SShellCmd *cmd); -static int32_t shellIsReadyGo(SShellCmd *cmd); +static bool shellIsReadyGo(SShellCmd *cmd); static void shellGetMbSizeInfo(const char *str, int32_t *size, int32_t *width); static void shellResetCommand(SShellCmd *cmd, const char s[]); void shellClearScreen(int32_t ecmd_pos, int32_t cursor_pos); @@ -62,7 +62,8 @@ int32_t shellCountPrefixOnes(uint8_t c) { } void shellGetPrevCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width) { - assert(pos > 0); + ASSERT(pos > 0); + if (pos <= 0) return; TdWchar wc; *size = 0; @@ -75,13 +76,14 @@ void shellGetPrevCharSize(const char *str, int32_t pos, int32_t *size, int32_t * } taosMbToWchar(&wc, str + pos, MB_CUR_MAX); - // assert(rc == *size); // it will be core, if str is encode by utf8 and taos charset is gbk + // ASSERT(rc == *size); // it will be core, if str is encode by utf8 and taos charset is gbk *width = taosWcharWidth(wc); } void shellGetNextCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width) { - assert(pos >= 0); + ASSERT(pos >= 0); + if(pos < 0) return; TdWchar wc; *size = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); @@ -89,7 +91,8 @@ void shellGetNextCharSize(const char *str, int32_t pos, int32_t *size, int32_t * } void shellInsertChar(SShellCmd *cmd, char *c, int32_t size) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; TdWchar wc; if (taosMbToWchar(&wc, c, size) < 0) return; @@ -135,7 +138,8 @@ void shellInsertStr(SShellCmd *cmd, char *str, int32_t size) { } void shellBackspaceChar(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset > 0) { shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -155,7 +159,8 @@ void shellBackspaceChar(SShellCmd *cmd) { } void shellClearLineBefore(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); memmove(cmd->command, cmd->command + cmd->cursorOffset, cmd->commandSize - cmd->cursorOffset); @@ -169,7 +174,8 @@ void shellClearLineBefore(SShellCmd *cmd) { } void shellClearLineAfter(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); cmd->commandSize -= cmd->endOffset - cmd->cursorOffset; @@ -178,7 +184,8 @@ void shellClearLineAfter(SShellCmd *cmd) { } void shellDeleteChar(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset < cmd->commandSize) { shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -196,7 +203,8 @@ void shellDeleteChar(SShellCmd *cmd) { } void shellMoveCursorLeft(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset > 0) { shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -210,7 +218,8 @@ void shellMoveCursorLeft(SShellCmd *cmd) { } void shellMoveCursorRight(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset < cmd->commandSize) { shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -224,7 +233,8 @@ void shellMoveCursorRight(SShellCmd *cmd) { } void shellPositionCursorHome(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset > 0) { shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -244,7 +254,8 @@ void positionCursorMiddle(SShellCmd *cmd) { } void shellPositionCursorEnd(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset < cmd->commandSize) { shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -279,7 +290,8 @@ void shellPositionCursor(int32_t step, int32_t direction) { } void shellUpdateBuffer(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (shellRegexMatch(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " "); strcat(cmd->buffer, cmd->command); @@ -293,8 +305,9 @@ void shellUpdateBuffer(SShellCmd *cmd) { shellShowOnScreen(cmd); } -int32_t shellIsReadyGo(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); +bool shellIsReadyGo(SShellCmd *cmd) { + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return false; char *total = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE); memset(cmd->command + cmd->commandSize, 0, SHELL_MAX_COMMAND_SIZE - cmd->commandSize); @@ -305,11 +318,11 @@ int32_t shellIsReadyGo(SShellCmd *cmd) { "\\s*clear\\s*$)"; if (shellRegexMatch(total, reg_str, REG_EXTENDED | REG_ICASE)) { taosMemoryFree(total); - return 1; + return true; } taosMemoryFree(total); - return 0; + return false; } void shellGetMbSizeInfo(const char *str, int32_t *size, int32_t *width) { @@ -321,7 +334,8 @@ void shellGetMbSizeInfo(const char *str, int32_t *size, int32_t *width) { } void shellResetCommand(SShellCmd *cmd, const char s[]) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); memset(cmd->buffer, 0, SHELL_MAX_COMMAND_SIZE); @@ -399,7 +413,7 @@ void shellShowOnScreen(SShellCmd *cmd) { int32_t ret = taosMbToWchar(&wc, str, MB_CUR_MAX); if (ret < 0) break; size += ret; - /* assert(size >= 0); */ + /* ASSERT(size >= 0); */ int32_t width = taosWcharWidth(wc); if (remain_column > width) { printf("%lc", wc); diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 986806fdd83a7b2ee9d2b72dbfcf332c713323df..4a9cf8343120654899a9080e4f6e219685c88cdf 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -713,7 +713,7 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) { } default: - assert(false); + ASSERT(false); } return 0; @@ -1072,7 +1072,8 @@ void *shellThreadLoop(void *arg) { } int32_t shellExecute() { - printf(shell.info.clientVersion, taos_get_client_info()); + printf(shell.info.clientVersion, shell.info.cusName, + taos_get_client_info(), shell.info.cusName); fflush(stdout); SShellArgs *pArgs = &shell.args; @@ -1127,7 +1128,7 @@ int32_t shellExecute() { } if (tsem_init(&shell.cancelSem, 0, 0) != 0) { - printf("failed to create cancel semphore\r\n"); + printf("failed to create cancel semaphore\r\n"); return -1; } diff --git a/tools/shell/src/shellUtil.c b/tools/shell/src/shellUtil.c index 8c47d165557317dbcf710dda7d72b77037340692..e15b49efcc35da2682d003243c0a19eb278acbc7 100644 --- a/tools/shell/src/shellUtil.c +++ b/tools/shell/src/shellUtil.c @@ -50,19 +50,19 @@ bool shellRegexMatch(const char *s, const char *reg, int32_t cflags) { int32_t shellCheckIntSize() { if (sizeof(int8_t) != 1) { - printf("taos int8 size is %d(!= 1)", (int)sizeof(int8_t)); + printf("int8 size is %d(!= 1)", (int)sizeof(int8_t)); return -1; } if (sizeof(int16_t) != 2) { - printf("taos int16 size is %d(!= 2)", (int)sizeof(int16_t)); + printf("int16 size is %d(!= 2)", (int)sizeof(int16_t)); return -1; } if (sizeof(int32_t) != 4) { - printf("taos int32 size is %d(!= 4)", (int)sizeof(int32_t)); + printf("int32 size is %d(!= 4)", (int)sizeof(int32_t)); return -1; } if (sizeof(int64_t) != 8) { - printf("taos int64 size is %d(!= 8)", (int)sizeof(int64_t)); + printf("int64 size is %d(!= 8)", (int)sizeof(int64_t)); return -1; } return 0; @@ -80,7 +80,7 @@ void shellGenerateAuth() { void shellDumpConfig() { SConfig *pCfg = taosGetCfg(); if (pCfg == NULL) { - printf("TDengine read global config failed!\r\n"); + printf("read global config failed!\r\n"); } else { cfgDumpCfg(pCfg, 1, true); } diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c index add3ccc51f730d9398c7f339f8e3a645cc5a0b0e..e3584b689009f9ca120af6e6a29e7ab8c643ec29 100644 --- a/tools/shell/src/shellWebsocket.c +++ b/tools/shell/src/shellWebsocket.c @@ -37,7 +37,9 @@ static int horizontalPrintWebsocket(WS_RES* wres, double* execute_time) { const void* data = NULL; int rows; ws_fetch_block(wres, &data, &rows); - *execute_time += (double)(ws_take_timing(wres)/1E6); + if (wres) { + *execute_time += (double)(ws_take_timing(wres)/1E6); + } if (!rows) { return 0; } @@ -77,7 +79,9 @@ static int verticalPrintWebsocket(WS_RES* wres, double* pexecute_time) { int rows = 0; const void* data = NULL; ws_fetch_block(wres, &data, &rows); - *pexecute_time += (double)(ws_take_timing(wres)/1E6); + if (wres) { + *pexecute_time += (double)(ws_take_timing(wres)/1E6); + } if (!rows) { return 0; } @@ -129,7 +133,9 @@ static int dumpWebsocketToFile(const char* fname, WS_RES* wres, double* pexecute int rows = 0; const void* data = NULL; ws_fetch_block(wres, &data, &rows); - *pexecute_time += (double)(ws_take_timing(wres)/1E6); + if (wres) { + *pexecute_time += (double)(ws_take_timing(wres)/1E6); + } if (!rows) { taosCloseFile(&pFile); return 0; @@ -223,17 +229,23 @@ void shellRunSingleCommandWebsocketImp(char *command) { if (code == TSDB_CODE_WS_SEND_TIMEOUT || code == TSDB_CODE_WS_RECV_TIMEOUT) { fprintf(stderr, "Hint: use -t to increase the timeout in seconds\n"); } else if (code == TSDB_CODE_WS_INTERNAL_ERRO || code == TSDB_CODE_WS_CLOSED) { - fprintf(stderr, "TDengine server is disconnected, will try to reconnect\n"); shell.ws_conn = NULL; } ws_free_result(res); - if (reconnectNum == 0) continue; + if (reconnectNum == 0) { + continue; + } else { + fprintf(stderr, "The server is disconnected, will try to reconnect\n"); + } return; } break; } - double execute_time = ws_take_timing(res)/1E6; + double execute_time = 0; + if (res) { + execute_time = ws_take_timing(res)/1E6; + } if (shellRegexMatch(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) { fprintf(stdout, "Database changed.\r\n\r\n"); diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index c6073541fdfadacebc4ef49e8fd00cd0a5ef0fbb..33fbbba1c175c4ac37d273274d993efb23580e94 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -19,8 +19,8 @@ #include #include #include "taos.h" -#include "types.h" #include "tlog.h" +#include "types.h" int smlProcess_influx_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -78,12 +78,23 @@ int smlProcess_telnet_Test() { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - const char *sql[] = {"sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0", - "sys.if.bytes.out 1479496101 1.3E1 interface=eth0 host=web01 ", - "sys.if.bytes.out 1479496102 1.3E3 network=tcp", - " sys.procs.running 1479496100 42 host=web01 "}; - - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, + // char *sql[4] = {0}; + // sql[0] = taosMemoryCalloc(1, 128); + // sql[1] = taosMemoryCalloc(1, 128); + // sql[2] = taosMemoryCalloc(1, 128); + // sql[3] = taosMemoryCalloc(1, 128); + const char *sql1[] = {"sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0", + "sys.if.bytes.out 1479496101 1.3E1 interface=eth0 host=web01 ", + "sys.if.bytes.out 1479496102 1.3E3 network=tcp", + " sys.procs.running 1479496100 42 host=web01 "}; + + // for(int i = 0; i < 4; i++){ + // strncpy(sql[i], sql1[i], 128); + // } + + // pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, + // TSDB_SML_TIMESTAMP_NANO_SECONDS); + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); @@ -103,41 +114,62 @@ int smlProcess_json1_Test() { taos_free_result(pRes); const char *sql[] = { - "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":0,\"value\":18,\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":\"lga\"}},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":\"lga\"}}]" - }; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":0,\"value\":18,\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":" + "\"lga\"}},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":" + "\"lga\"}}]"}; + + char *sql1[1] = {0}; + for (int i = 0; i < 1; i++) { + sql1[i] = taosMemoryCalloc(1, 1024); + strncpy(sql1[i], sql[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } taos_free_result(pRes); - taos_close(taos); - - return code; -} -int smlProcess_json2_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql1[i]); + } - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); + const char *sql2[] = { + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":\"web01\",\"dc\":\"lga\"}" + "},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":\"lga\"}" + "}]", + }; - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); + char *sql3[1] = {0}; + for (int i = 0; i < 1; i++) { + sql3[i] = taosMemoryCalloc(1, 1024); + strncpy(sql3[i], sql2[i], 1023); + } - const char *sql[] = { - "{\"metric\":\"meter_current0\",\"timestamp\":{\"value\":1662344042,\"type\":\"s\"},\"value\":{\"value\":10.3,\"type\":\"i64\"},\"tags\":{\"groupid\":{\"value\":2,\"type\":\"bigint\"},\"location\":{\"value\":\"北京\",\"type\":\"binary\"},\"id\":\"d1001\"}}" - }; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); + code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } taos_free_result(pRes); + + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql3[i]); + } + taos_close(taos); return code; } -int smlProcess_json3_Test() { +int smlProcess_json2_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); @@ -147,18 +179,33 @@ int smlProcess_json3_Test() { taos_free_result(pRes); const char *sql[] = { - "{\"metric\":\"meter_current1\",\"timestamp\":{\"value\":1662344042,\"type\":\"s\"},\"value\":{\"value\":10.3,\"type\":\"i64\"},\"tags\":{\"t1\":{\"value\":2,\"type\":\"bigint\"},\"t2\":{\"value\":2,\"type\":\"int\"},\"t3\":{\"value\":2,\"type\":\"i16\"},\"t4\":{\"value\":2,\"type\":\"i8\"},\"t5\":{\"value\":2,\"type\":\"f32\"},\"t6\":{\"value\":2,\"type\":\"double\"},\"t7\":{\"value\":\"8323\",\"type\":\"binary\"},\"t8\":{\"value\":\"北京\",\"type\":\"nchar\"},\"t9\":{\"value\":true,\"type\":\"bool\"},\"id\":\"d1001\"}}"}; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + "{\"metric\":\"meter_current0\",\"timestamp\":{\"value\":1662344042,\"type\":\"s\"},\"value\":{\"value\":10.3," + "\"type\":\"i64\"},\"tags\":{\"groupid\":{\"value\":2,\"type\":\"bigint\"},\"location\":{\"value\":\"北京\"," + "\"type\":\"binary\"},\"id\":\"d1001\"}}"}; + char *sql1[1] = {0}; + for (int i = 0; i < 1; i++) { + sql1[i] = taosMemoryCalloc(1, 1024); + strncpy(sql1[i], sql[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } taos_free_result(pRes); taos_close(taos); + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql1[i]); + } return code; } -int smlProcess_json4_Test() { +int smlProcess_json3_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); @@ -168,15 +215,28 @@ int smlProcess_json4_Test() { taos_free_result(pRes); const char *sql[] = { - "{\"metric\":\"meter_current2\",\"timestamp\":{\"value\":1662344042000,\"type\":\"ms\"},\"value\":\"ni\",\"tags\":{\"t1\":{\"value\":20,\"type\":\"i64\"},\"t2\":{\"value\":25,\"type\":\"i32\"},\"t3\":{\"value\":2,\"type\":\"smallint\"},\"t4\":{\"value\":2,\"type\":\"tinyint\"},\"t5\":{\"value\":2,\"type\":\"float\"},\"t6\":{\"value\":0.2,\"type\":\"f64\"},\"t7\":\"nsj\",\"t8\":{\"value\":\"北京\",\"type\":\"nchar\"},\"t9\":false,\"id\":\"d1001\"}}" - }; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + "[{\"metric\":\"sys.cpu.nice3\",\"timestamp\":0,\"value\":\"18\",\"tags\":{\"host\":\"web01\",\"id\":\"t1\"," + "\"dc\":\"lga\"}}]"}; + char *sql1[1] = {0}; + for (int i = 0; i < 1; i++) { + sql1[i] = taosMemoryCalloc(1, 1024); + strncpy(sql1[i], sql[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } taos_free_result(pRes); taos_close(taos); + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql1[i]); + } return code; } @@ -242,10 +302,10 @@ int sml_16384_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); - if(code) return code; + if (code) return code; const char *sql1[] = { - "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c0=f,c1=127i8,c11=L\"ncharColValue\",c10=t 1626006833639000000", + "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c0=f,c1=127i8,c11=L\"ncharColValue\",c10=t 1626006833631000000", }; pRes = taos_schemaless_insert(taos, (char **)sql1, 1, TSDB_SML_LINE_PROTOCOL, 0); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -677,52 +737,6 @@ int sml_oom_Test() { return code; } -int sml_16368_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "[{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639000, \"type\": \"us\"}, \"value\": 1, " - "\"tags\": {\"t1\": 3, \"t2\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t3\", \"type\": " - "\"binary\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833739000, \"type\": \"us\"}, \"value\": 2, " - "\"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " - "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}," - "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639100, \"type\": \"us\"}, \"value\": 3, " - "\"tags\": {\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t3\": {\"value\": \"ste\", \"type\": \"nchar\"}}}," - "{\"metric\": \"stf567890\", \"timestamp\": {\"value\": 1626006833639200, \"type\": \"us\"}, \"value\": 4, " - "\"tags\": {\"t1\": {\"value\": 4, \"type\": \"bigint\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " - "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639300, \"type\": \"us\"}, \"value\": " - "{\"value\": 5, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t2\": 5.0, " - "\"t3\": {\"value\": \"t4\", \"type\": \"binary\"}}}," - "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639400, \"type\": \"us\"}, \"value\": " - "{\"value\": 6, \"type\": \"double\"}, \"tags\": {\"t2\": 5.0, \"t3\": {\"value\": \"ste2\", \"type\": " - "\"nchar\"}}}," - "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006834639400, \"type\": \"us\"}, \"value\": " - "{\"value\": 7, \"type\": \"double\"}, \"tags\": {\"t2\": {\"value\": 5.0, \"type\": \"double\"}, \"t3\": " - "{\"value\": \"ste2\", \"type\": \"nchar\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833839006, \"type\": \"us\"}, \"value\": " - "{\"value\": 8, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": " - "{\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, " - "\"type\": \"double\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833939007, \"type\": \"us\"}, \"value\": " - "{\"value\": 9, \"type\": \"double\"}, \"tags\": {\"t1\": 4, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " - "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}]"}; - pRes = taos_schemaless_insert(taos, (char **)sql, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MICRO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - taos_close(taos); - - return code; -} - int sml_dup_time_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -732,7 +746,7 @@ int sml_dup_time_Test() { const char *sql[] = {//"test_ms,t0=t c0=f 1626006833641", "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=false,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "c0=f,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." "123456789f64,c7=\"xcxvwjvf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " @@ -762,214 +776,6 @@ int sml_dup_time_Test() { return code; } -int sml_16960_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "[" - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955000, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": 830525384, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955001, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -588348364, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955002, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -370310823, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955003, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -811250191, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955004, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -330340558, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}" - "]"}; - - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, - TSDB_SML_TIMESTAMP_MILLI_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - taos_close(taos); - - return code; -} - int sml_add_tag_col_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -977,8 +783,10 @@ int sml_add_tag_col_Test() { taos_free_result(pRes); const char *sql[] = { - "macylr,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000" - }; + "macylr,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000"}; pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); @@ -989,8 +797,10 @@ int sml_add_tag_col_Test() { if (code) return code; const char *sql1[] = { - "macylr,id=macylr_17875_1804,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\",t11=127i8,t10=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c8=L\"ncharColValue\",c9=7u64,c11=L\"ncharColValue\",c10=f 1626006833639000000" - }; + "macylr,id=macylr_17875_1804,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=" + "22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\",t11=127i8,t10=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c8=" + "L\"ncharColValue\",c9=7u64,c11=L\"ncharColValue\",c10=f 1626006833639000000"}; pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, 0); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -1004,10 +814,10 @@ int sml_add_tag_col_Test() { int smlProcess_18784_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + TAOS_RES *pRes = taos_query(taos, "create database if not exists db_18784 schemaless 1"); taos_free_result(pRes); - pRes = taos_query(taos, "use sml_db"); + pRes = taos_query(taos, "use db_18784"); taos_free_result(pRes); const char *sql[] = { @@ -1018,7 +828,7 @@ int smlProcess_18784_Test() { printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes)); int code = taos_errno(pRes); ASSERT(!code); - ASSERT(taos_affected_rows(pRes) == 2); + ASSERT(taos_affected_rows(pRes) == 1); taos_free_result(pRes); pRes = taos_query(taos, "select * from disk"); @@ -1027,26 +837,26 @@ int smlProcess_18784_Test() { ASSERT(fieldNum == 5); printf("fieldNum:%d\n", fieldNum); TAOS_ROW row = NULL; - int32_t rowIndex = 0; - while((row = taos_fetch_row(pRes)) != NULL) { - int64_t ts = *(int64_t*)row[0]; - int64_t used = *(int64_t*)row[1]; - int64_t total = *(int64_t*)row[2]; - int64_t freed = *(int64_t*)row[3]; - if(rowIndex == 0){ + int32_t rowIndex = 0; + while ((row = taos_fetch_row(pRes)) != NULL) { + int64_t ts = *(int64_t *)row[0]; + int64_t used = *(int64_t *)row[1]; + int64_t total = *(int64_t *)row[2]; + int64_t freed = *(int64_t *)row[3]; + if (rowIndex == 0) { ASSERT(ts == 1661943960000); ASSERT(used == 176059); ASSERT(total == 1081101176832); ASSERT(freed == 66932805); -// ASSERT_EQ(latitude, 24.5208); -// ASSERT_EQ(longitude, 28.09377); -// ASSERT_EQ(elevation, 428); -// ASSERT_EQ(velocity, 0); -// ASSERT_EQ(heading, 304); -// ASSERT_EQ(grade, 0); -// ASSERT_EQ(fuel_consumption, 25); - }else{ -// ASSERT(0); + // ASSERT_EQ(latitude, 24.5208); + // ASSERT_EQ(longitude, 28.09377); + // ASSERT_EQ(elevation, 428); + // ASSERT_EQ(velocity, 0); + // ASSERT_EQ(heading, 304); + // ASSERT_EQ(grade, 0); + // ASSERT_EQ(fuel_consumption, 25); + } else { + // ASSERT(0); } rowIndex++; } @@ -1063,17 +873,21 @@ int sml_19221_Test() { taos_free_result(pRes); const char *sql[] = { - "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000\nqelhxo,id=pnnhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000\n#comment\nqelhxo,id=pnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000", + "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 " + "1626006833632000000\nqelhxo,id=pnnhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 " + "1626006833633000000\n#comment\nqelhxo,id=pnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 " + "1626006833634000000", }; pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - char* tmp = (char*)taosMemoryCalloc(1024, 1); + char *tmp = (char *)taosMemoryCalloc(1024, 1); memcpy(tmp, sql[0], strlen(sql[0])); - *(char*)(tmp+44) = 0; + *(char *)(tmp + 44) = 0; int32_t totalRows = 0; - pRes = taos_schemaless_insert_raw(taos, tmp, strlen(sql[0]), &totalRows, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + pRes = taos_schemaless_insert_raw(taos, tmp, strlen(sql[0]), &totalRows, TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT(totalRows == 3); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -1088,19 +902,22 @@ int sml_19221_Test() { int sml_ts2164_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - TAOS_RES *pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS line_test BUFFER 384 MINROWS 1000 PAGES 256 PRECISION 'ns'"); + TAOS_RES *pRes = + taos_query(taos, "CREATE DATABASE IF NOT EXISTS line_test BUFFER 384 MINROWS 1000 PAGES 256 PRECISION 'ns'"); taos_free_result(pRes); const char *sql[] = { + // "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", + "meters,location=la,groupid=ca current=11.8,voltage=221", "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", - "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", - "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27", + // "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27", }; pRes = taos_query(taos, "use line_test"); taos_free_result(pRes); - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); @@ -1117,22 +934,35 @@ int sml_ttl_Test() { taos_free_result(pRes); const char *sql[] = { - "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" 1626006833739000000", + "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" " + "1626006833739000000", + }; + const char *sql1[] = { + "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" " + "1626006833339000000", }; pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - pRes = taos_schemaless_insert_ttl(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, 20); + pRes = taos_schemaless_insert_ttl(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS, 20); printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes)); taos_free_result(pRes); - pRes = taos_query(taos, "select `ttl` from information_schema.ins_tables where table_name='t_be97833a0e1f523fcdaeb6291d6fdf27'"); + pRes = taos_schemaless_insert_ttl(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS, 20); + + printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes)); + taos_free_result(pRes); + + pRes = taos_query( + taos, "select `ttl` from information_schema.ins_tables where table_name='t_be97833a0e1f523fcdaeb6291d6fdf27'"); printf("%s result2:%s\n", __FUNCTION__, taos_errstr(pRes)); TAOS_ROW row = taos_fetch_row(pRes); - if(row != NULL && row[0] != NULL){ - int32_t ttl = *(int32_t*)row[0]; + if (row != NULL && *row != NULL) { + int32_t ttl = *(int32_t *)row[0]; ASSERT(ttl == 20); } @@ -1143,23 +973,91 @@ int sml_ttl_Test() { return code; } +// char *str[] ={ +// "", +// "f64", +// "F64", +// "f32", +// "F32", +// "i", +// "I", +// "i64", +// "I64", +// "u", +// "U", +// "u64", +// "U64", +// "i32", +// "I32", +// "u32", +// "U32", +// "i16", +// "I16", +// "u16", +// "U16", +// "i8", +// "I8", +// "u8", +// "U8", +// }; +// uint8_t smlCalTypeSum(char* endptr, int32_t left){ +// uint8_t sum = 0; +// for(int i = 0; i < left; i++){ +// sum += endptr[i]; +// } +// return sum; +// } + int sml_ts2385_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS_RES *pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS ts2385"); taos_free_result(pRes); - const char *sql[] ={ - "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 s5=false,s18=false,k14=0,k2=0,k8=0,k10=0,s9=false,s19=false,k11=0,k13=0,s22=false,k15=0,m2=37.416671660000006,m8=600,m10=1532,m1=20.25,m13=0,s7=false,k7=0,m16=0,s17=false,k4=0,s11=false,s15=true,m7=600,m12=1490,s1=true,m14=0,s14=false,s16=true,k5=0,hex=\"7b3b00000001030301030200000000323231313233304339344b30002b01012a10028003000000070d05da025802580258025802580258045305fc05f505d200000000000000000afc7d\",k6=0,m3=600,s3=false,s24=false,k3=0,m6=600,m15=0,s12=false,k1=0,k16=0,s10=false,s21=false,k12=0,m5=600,s8=false,m4=600,m9=1107,s2=false,s13=false,s20=false,s23=false,k9=0,m11=1525,s4=false,s6=false 1672818929178749400", - "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 k2=0,k11=0,m3=600,m12=1506,s17=false,m5=600,s11=false,s22=false,k6=0,m13=0,s16=true,k5=0,s21=false,m4=600,m7=600,s9=false,s10=false,s18=false,k7=0,m8=600,k1=0,hex=\"7b3a00000001030301030200000000323231313233304339344b30002b01012a10028003000000071105e8025802580258025802580258044905eb05ef05e200000000000000000afc7d\",m11=1519,m16=0,s19=false,s23=false,s24=false,s14=false,s6=false,k10=0,k15=0,k14=0,s2=false,s4=false,s8=false,s13=false,s15=true,s20=false,m2=38.000005040000005,s3=false,s7=false,k3=0,k8=0,k13=0,m6=600,m14=0,m15=0,k4=0,m1=20.450000000000003,m9=1097,s1=true,m10=1515,s5=false,s12=false,k9=0,k12=0,k16=0 1672818919126971000", - "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 k7=0,k14=0,m3=600,m7=600,s5=false,k2=0,k3=0,k8=0,s3=false,s20=false,k15=0,m10=1482,s17=false,k1=0,k16=0,m15=0,s12=false,k9=0,m16=0,s11=false,m4=600,s10=false,s15=true,s24=false,m8=600,m13=0,s2=false,s18=false,k12=0,s14=false,s19=false,hex=\"7b3900000001030301030200000000323231313233304339344b30002b01012a10028003000000071505ef025802580258025802580258045005ca05b105d800000000000000000aa47d\",s1=true,s4=false,s7=false,s8=false,s13=false,m6=600,s6=false,s21=false,k11=0,m12=1496,m9=1104,s16=true,k5=0,s9=false,k10=0,k13=0,m2=38.291671730000004,s22=false,m5=600,m11=1457,m14=0,k4=0,m1=20.650000000000006,s23=false,k6=0 1672818909130866800", - "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 m7=600,k4=0,k14=0,s22=false,k13=0,s2=false,m11=1510,m14=0,s4=false,s10=false,m1=21,m16=0,m13=0,s9=false,s13=false,s14=false,k10=0,m3=600,m9=1107,s18=false,s19=false,k2=0,hex=\"7b3600000001030301030200000000323231313233304339344b30002b01012a10028003000000071c0619025802580258025802580258045305dc05e6058d00000000000000000ad27d\",m2=40.04167187,m8=600,k7=0,k8=0,m10=1500,s23=false,k5=0,s11=false,s21=false,k9=0,m15=0,m12=1421,s1=true,s5=false,s8=false,m5=600,k16=0,k15=0,m6=600,s3=false,s6=false,s7=false,s15=true,s20=false,s24=false,k11=0,k1=0,k6=0,k12=0,m4=600,s16=true,s17=false,k3=0,s12=false 1672818879189483200", - "DataRTU,deviceId=2106070C11M0_2,dataModelName=DataRTU_2106070C11M0_2 m1=5691,k14=0,m6=0,s14=false,k8=0,s19=false,s20=false,k12=0,s17=false,k3=0,m8=0,s8=false,m7=0,s9=false,s4=false,s11=false,s13=false,s16=false,k5=0,k15=0,k16=0,s10=false,s23=false,s1=false,s2=false,s3=false,s12=false,s24=false,k2=0,k10=0,hex=\"7b1400000001030301030200000000323130363037304331314d30002b01022a080400000000000008af0c000000000000000000000000000000000000000000000000000000000ad47d\",m2=0,s7=false,s18=false,s21=false,m3=0,m5=0,k4=0,k11=0,m4=0,k1=0,k6=0,k13=0,s6=false,s15=false,s5=false,s22=false,k7=0,k9=0 1672818779549848800" - }; + const char *sql[] = { + "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 " + "s5=false,s18=false,k14=0,k2=0,k8=0,k10=0,s9=false,s19=false,k11=0,k13=0,s22=false,k15=0,m2=37.416671660000006," + "m8=600,m10=1532,m1=20.25,m13=0,s7=false,k7=0,m16=0,s17=false,k4=0,s11=false,s15=true,m7=600,m12=1490,s1=true," + "m14=0,s14=false,s16=true,k5=0,hex=" + "\"7b3b00000001030301030200000000323231313233304339344b30002b01012a10028003000000070d05da025802580258025802580258" + "045305fc05f505d200000000000000000afc7d\",k6=0,m3=600,s3=false,s24=false,k3=0,m6=600,m15=0,s12=false,k1=0,k16=0," + "s10=false,s21=false,k12=0,m5=600,s8=false,m4=600,m9=1107,s2=false,s13=false,s20=false,s23=false,k9=0,m11=1525," + "s4=false,s6=false 1672818929178749400", + "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 " + "k2=0,k11=0,m3=600,m12=1506,s17=false,m5=600,s11=false,s22=false,k6=0,m13=0,s16=true,k5=0,s21=false,m4=600,m7=" + "600,s9=false,s10=false,s18=false,k7=0,m8=600,k1=0,hex=" + "\"7b3a00000001030301030200000000323231313233304339344b30002b01012a10028003000000071105e8025802580258025802580258" + "044905eb05ef05e200000000000000000afc7d\",m11=1519,m16=0,s19=false,s23=false,s24=false,s14=false,s6=false,k10=0," + "k15=0,k14=0,s2=false,s4=false,s8=false,s13=false,s15=true,s20=false,m2=38.000005040000005,s3=false,s7=false,k3=" + "0,k8=0,k13=0,m6=600,m14=0,m15=0,k4=0,m1=20.450000000000003,m9=1097,s1=true,m10=1515,s5=false,s12=false,k9=0,k12=" + "0,k16=0 1672818919126971000", + "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 " + "k7=0,k14=0,m3=600,m7=600,s5=false,k2=0,k3=0,k8=0,s3=false,s20=false,k15=0,m10=1482,s17=false,k1=0,k16=0,m15=0," + "s12=false,k9=0,m16=0,s11=false,m4=600,s10=false,s15=true,s24=false,m8=600,m13=0,s2=false,s18=false,k12=0,s14=" + "false,s19=false,hex=" + "\"7b3900000001030301030200000000323231313233304339344b30002b01012a10028003000000071505ef025802580258025802580258" + "045005ca05b105d800000000000000000aa47d\",s1=true,s4=false,s7=false,s8=false,s13=false,m6=600,s6=false,s21=false," + "k11=0,m12=1496,m9=1104,s16=true,k5=0,s9=false,k10=0,k13=0,m2=38.291671730000004,s22=false,m5=600,m11=1457,m14=0," + "k4=0,m1=20.650000000000006,s23=false,k6=0 1672818909130866800", + "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 " + "m7=600,k4=0,k14=0,s22=false,k13=0,s2=false,m11=1510,m14=0,s4=false,s10=false,m1=21,m16=0,m13=0,s9=false,s13=" + "false,s14=false,k10=0,m3=600,m9=1107,s18=false,s19=false,k2=0,hex=" + "\"7b3600000001030301030200000000323231313233304339344b30002b01012a10028003000000071c0619025802580258025802580258" + "045305dc05e6058d00000000000000000ad27d\",m2=40.04167187,m8=600,k7=0,k8=0,m10=1500,s23=false,k5=0,s11=false,s21=" + "false,k9=0,m15=0,m12=1421,s1=true,s5=false,s8=false,m5=600,k16=0,k15=0,m6=600,s3=false,s6=false,s7=false,s15=" + "true,s20=false,s24=false,k11=0,k1=0,k6=0,k12=0,m4=600,s16=true,s17=false,k3=0,s12=false 1672818879189483200", + "DataRTU,deviceId=2106070C11M0_2,dataModelName=DataRTU_2106070C11M0_2 " + "m1=5691,k14=0,m6=0,s14=false,k8=0,s19=false,s20=false,k12=0,s17=false,k3=0,m8=0,s8=false,m7=0,s9=false,s4=false," + "s11=false,s13=false,s16=false,k5=0,k15=0,k16=0,s10=false,s23=false,s1=false,s2=false,s3=false,s12=false,s24=" + "false,k2=0,k10=0,hex=" + "\"7b1400000001030301030200000000323130363037304331314d30002b01022a080400000000000008af0c000000000000000000000000" + "000000000000000000000000000000000ad47d\",m2=0,s7=false,s18=false,s21=false,m3=0,m5=0,k4=0,k11=0,m4=0,k1=0,k6=0," + "k13=0,s6=false,s15=false,s5=false,s22=false,k7=0,k9=0 1672818779549848800"}; pRes = taos_query(taos, "use ts2385"); taos_free_result(pRes); - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); @@ -1168,14 +1066,14 @@ int sml_ts2385_Test() { pRes = taos_query(taos, "select distinct tbname from `DataRTU` order by tbname"); printf("%s result2:%s\n", __FUNCTION__, taos_errstr(pRes)); - int num = 0; + int num = 0; TAOS_ROW row = NULL; - while((row = taos_fetch_row(pRes))){ - if(row[0] != NULL && num == 0){ + while ((row = taos_fetch_row(pRes))) { + if (row[0] != NULL && num == 0) { ASSERT(strncmp((char *)row[0], "DataRTU_2106070C11M0_2", sizeof("DataRTU_2106070C11M0_2") - 1) == 0); } - if(row[0] != NULL && num == 1){ + if (row[0] != NULL && num == 1) { ASSERT(strncmp((char *)row[0], "DataRTU_2211230C94K0_1", sizeof("DataRTU_2211230C94K0_1") - 1) == 0); } num++; @@ -1190,13 +1088,17 @@ int sml_ts2385_Test() { } int main(int argc, char *argv[]) { - if(argc == 2){ + if (argc == 2) { taos_options(TSDB_OPTION_CONFIGDIR, argv[1]); } int ret = 0; ret = sml_ts2385_Test(); ASSERT(!ret); + // for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){ + // printf("str:%s \t %d\n", str[i], smlCalTypeSum(str[i], strlen(str[i]))); + // } + // int ret = 0; ret = sml_ttl_Test(); ASSERT(!ret); ret = sml_ts2164_Test(); @@ -1211,8 +1113,6 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = smlProcess_json3_Test(); ASSERT(!ret); - ret = smlProcess_json4_Test(); - ASSERT(!ret); ret = sml_TD15662_Test(); ASSERT(!ret); ret = sml_TD15742_Test(); @@ -1221,12 +1121,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_oom_Test(); ASSERT(!ret); - ret = sml_16368_Test(); - ASSERT(!ret); ret = sml_dup_time_Test(); ASSERT(!ret); - ret = sml_16960_Test(); - ASSERT(!ret); ret = sml_add_tag_col_Test(); ASSERT(!ret); ret = smlProcess_18784_Test(); diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 024a253a2e2aad97b0639468b79ff04cb177b8a4..69bf52bb1ac1c80015e12a6ecf6701dc73f8d139 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -78,28 +78,27 @@ static void msg_process(TAOS_RES* msg) { int buildDatabase(TAOS* pConn, TAOS_RES* pRes){ /* test for TD-20612 start*/ -// pRes = taos_query(pConn,"create table tb1 (ts timestamp, c1 int, c2 int)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn,"insert into tb1 (ts, c1) values(1669092069069, 0)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn,"insert into tb1 (ts, c2) values(1669092069069, 1)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// return 0; + pRes = taos_query(pConn,"create table tb1 (ts timestamp, c1 int, c2 int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn,"insert into tb1 (ts, c1) values(1669092069069, 0)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn,"insert into tb1 (ts, c2) values(1669092069069, 1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + /* test for TD-20612 end*/ pRes = taos_query(pConn, @@ -614,6 +613,7 @@ void initLogFile() { } }else{ char *result[] = { + "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"tb1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":4}],\"tags\":[]}", "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":64},{\"name\":\"c4\",\"type\":5}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1},{\"name\":\"t2\",\"type\":8,\"length\":64}]}", "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}", @@ -652,6 +652,7 @@ void initLogFile() { } }else{ char *result[] = { + "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"tb1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":4}],\"tags\":[]}", "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}", "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}",