diff --git a/cmake/cmake.define b/cmake/cmake.define index d32200bb9145672e18af24fb158c68e802fcf45d..10f217254152db82518099560ef902b2ebabf499 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -1,6 +1,7 @@ cmake_minimum_required(VERSION 3.0) set(CMAKE_VERBOSE_MAKEFILE OFF) +set(TD_BUILD_TAOSA_INTERNAL FALSE) #set output directory SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib) @@ -117,12 +118,18 @@ ELSE () IF (${BUILD_SANITIZER}) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0") - MESSAGE(STATUS "Will compile with Address Sanitizer!") + MESSAGE(STATUS "Compile with Address Sanitizer!") ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") ENDIF () + # disable all assert + IF ((${DISABLE_ASSERT} MATCHES "true") OR (${DISABLE_ASSERTS} MATCHES "true")) + ADD_DEFINITIONS(-DDISABLE_ASSERT) + MESSAGE(STATUS "Disable all asserts") + ENDIF() + INCLUDE(CheckCCompilerFlag) IF (TD_ARM_64 OR TD_ARM_32) SET(COMPILER_SUPPORT_SSE42 false) @@ -155,7 +162,7 @@ ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2") ENDIF() - MESSAGE(STATUS "SIMD instructions (AVX/AVX2) is ACTIVATED") + MESSAGE(STATUS "SIMD instructions (FMA/AVX/AVX2) is ACTIVATED") ENDIF() ENDIF () diff --git a/cmake/cmake.install b/cmake/cmake.install index fd1e080ddab1478f73689e7cced405ae8404fbc2..67634625ce53c6ea48dc03746a39fdb03975c2af 100644 --- a/cmake/cmake.install +++ b/cmake/cmake.install @@ -21,7 +21,7 @@ IF (TD_LINUX) ELSEIF (TD_WINDOWS) SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.bat") INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") - INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} :needAdmin ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Windows ${TD_VER_NUMBER})") + INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} :needAdmin ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Windows ${TD_VER_NUMBER} ${TD_BUILD_TAOSA_INTERNAL})") ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.sh") INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") diff --git a/cmake/cmake.version b/cmake/cmake.version index f43465bf1de9722cd1c4243bd22e91fa457ddc05..ba85a3d99b8280a49d9d6e6475cbeb3807090d28 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.2.1") + SET(TD_VER_NUMBER "3.0.2.2") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 31ca6b30fa1ccc5058276ace1429354d26fb3941..13b247770ea7eef6b64209ca98787ff6d733bf85 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG 5662a6d + GIT_TAG 213f8b3 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index e23ebb104bdea622267ce94a30fc14d3b4d94f53..cddc5aee4ea1d33c98833d134a507587e7dd6bdf 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 11b60a4 + GIT_TAG 723f696 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/en/07-develop/07-tmq.mdx b/docs/en/07-develop/07-tmq.mdx index 17b3f5caa062eaacb4216b7153e899040e702cc1..94a9dbffbd2bc5199edf6f2a6e4561355d967705 100644 --- a/docs/en/07-develop/07-tmq.mdx +++ b/docs/en/07-develop/07-tmq.mdx @@ -117,19 +117,22 @@ class TaosConsumer(): ```go -func NewConsumer(conf *Config) (*Consumer, error) +func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error) -func (c *Consumer) Close() error - -func (c *Consumer) Commit(ctx context.Context, message unsafe.Pointer) error +// rebalanceCb is reserved for compatibility purpose +func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error -func (c *Consumer) FreeMessage(message unsafe.Pointer) +// rebalanceCb is reserved for compatibility purpose +func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error -func (c *Consumer) Poll(timeout time.Duration) (*Result, error) +func (c *Consumer) Poll(timeoutMs int) tmq.Event -func (c *Consumer) Subscribe(topics []string) error +// tmq.TopicPartition is reserved for compatibility purpose +func (c *Consumer) Commit() ([]tmq.TopicPartition, error) func (c *Consumer) Unsubscribe() error + +func (c *Consumer) Close() error ``` @@ -357,50 +360,20 @@ public class MetersDeserializer extends ReferenceDeserializer { ```go -config := tmq.NewConfig() -defer config.Destroy() -err = config.SetGroupID("test") -if err != nil { - panic(err) -} -err = config.SetAutoOffsetReset("earliest") -if err != nil { - panic(err) -} -err = config.SetConnectIP("127.0.0.1") -if err != nil { - panic(err) -} -err = config.SetConnectUser("root") -if err != nil { - panic(err) -} -err = config.SetConnectPass("taosdata") -if err != nil { - panic(err) -} -err = config.SetConnectPort("6030") -if err != nil { - panic(err) -} -err = config.SetMsgWithTableName(true) -if err != nil { - panic(err) -} -err = config.EnableHeartBeat() -if err != nil { - panic(err) -} -err = config.EnableAutoCommit(func(result *wrapper.TMQCommitCallbackResult) { - if result.ErrCode != 0 { - errStr := wrapper.TMQErr2Str(result.ErrCode) - err := errors.NewError(int(result.ErrCode), errStr) - panic(err) - } -}) -if err != nil { - panic(err) +conf := &tmq.ConfigMap{ + "group.id": "test", + "auto.offset.reset": "earliest", + "td.connect.ip": "127.0.0.1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "td.connect.port": "6030", + "client.id": "test_tmq_c", + "enable.auto.commit": "false", + "enable.heartbeat.background": "true", + "experimental.snapshot.enable": "true", + "msg.with.table.name": "true", } +consumer, err := NewConsumer(conf) ``` @@ -523,11 +496,7 @@ consumer.subscribe(topics); ```go -consumer, err := tmq.NewConsumer(config) -if err != nil { - panic(err) -} -err = consumer.Subscribe([]string{"example_tmq_topic"}) +err = consumer.Subscribe("example_tmq_topic", nil) if err != nil { panic(err) } @@ -611,13 +580,17 @@ while(running){ ```go for { - result, err := consumer.Poll(time.Second) - if err != nil { - panic(err) + ev := consumer.Poll(0) + if ev != nil { + switch e := ev.(type) { + case *tmqcommon.DataMessage: + fmt.Println(e.Value()) + case tmqcommon.Error: + fmt.Fprintf(os.Stderr, "%% Error: %v: %v\n", e.Code(), e) + panic(e) + } + consumer.Commit() } - fmt.Println(result) - consumer.Commit(context.Background(), result.Message) - consumer.FreeMessage(result.Message) } ``` @@ -729,7 +702,11 @@ consumer.close(); ```go -consumer.Close() +/* Unsubscribe */ +_ = consumer.Unsubscribe() + +/* Close consumer */ +_ = consumer.Close() ``` diff --git a/docs/en/12-taos-sql/02-database.md b/docs/en/12-taos-sql/02-database.md index c3c7e5928bb6705939dfae7d4e0096b202025520..476df0a60d593f5215432c08ba4d7560021693e6 100644 --- a/docs/en/12-taos-sql/02-database.md +++ b/docs/en/12-taos-sql/02-database.md @@ -30,8 +30,10 @@ database_option: { | WAL_LEVEL {1 | 2} | VGROUPS value | SINGLE_STABLE {0 | 1} + | STT_TRIGGER value | TABLE_PREFIX value | TABLE_SUFFIX value + | TSDB_PAGESIZE value | WAL_RETENTION_PERIOD value | WAL_ROLL_PERIOD value | WAL_RETENTION_SIZE value @@ -69,8 +71,10 @@ database_option: { - SINGLE_STABLE: specifies whether the database can contain more than one supertable. - 0: The database can contain multiple supertables. - 1: The database can contain only one supertable. +- STT_TRIGGER: specifies the number of file merges triggered by flushed files. The default is 8, ranging from 1 to 16. For high-frequency scenarios with few tables, it is recommended to use the default configuration or a smaller value for this parameter; For multi-table low-frequency scenarios, it is recommended to configure this parameter with a larger value. - TABLE_PREFIX：The prefix length in the table name that is ignored when distributing table to vnode based on table name. - TABLE_SUFFIX：The suffix length in the table name that is ignored when distributing table to vnode based on table name. +- TSDB_PAGESIZE: The page size of the data storage engine in a vnode. The unit is KB. The default is 4 KB. The range is 1 to 16384, that is, 1 KB to 16 MB. - WAL_RETENTION_PERIOD: specifies the time after which WAL files are deleted. This parameter is used for data subscription. Enter a time in seconds. The default value of single copy is 0. A value of 0 indicates that each WAL file is deleted immediately after its contents are written to disk. -1: WAL files are never deleted. The default value of multiple copy is 4 days. - WAL_RETENTION_SIZE: specifies the size at which WAL files are deleted. This parameter is used for data subscription. Enter a size in KB. The default value of single copy is 0. A value of 0 indicates that each WAL file is deleted immediately after its contents are written to disk. -1: WAL files are never deleted. The default value of multiple copy is -1. - WAL_ROLL_PERIOD: specifies the time after which WAL files are rotated. After this period elapses, a new WAL file is created. The default value of single copy is 0. A value of 0 indicates that a new WAL file is created only after the previous WAL file was written to disk. The default values of multiple copy is 1 day. @@ -112,6 +116,10 @@ alter_database_options: alter_database_option: { CACHEMODEL {'none' | 'last_row' | 'last_value' | 'both'} | CACHESIZE value + | BUFFER value + | PAGES value + | REPLICA value + | STT_TRIGGER value | WAL_LEVEL value | WAL_FSYNC_PERIOD value | KEEP value @@ -154,3 +162,19 @@ TRIM DATABASE db_name; ``` The preceding SQL statement deletes data that has expired and orders the remaining data in accordance with the storage configuration. + +## Redistribute Vgroup + +```sql +REDISTRIBUTE VGROUP vgroup_no DNODE dnode_id1 [DNODE dnode_id2] [DNODE dnode_id3] +``` + +Adjust the distribution of vnodes in the vgroup according to the given list of dnodes. + +## Balance Vgroup + +```sql +BALANCE VGROUP +``` + +Automatically adjusts the distribution of vnodes in all vgroups of the cluster, which is equivalent to load balancing the data of the cluster at the vnode level. diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index c087a9e9fb2f0af921aa031d41d124c66fbb0ae7..ee06a7be2d3172210bf35302d5bffbf7a49adabc 100644 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -350,9 +350,9 @@ SELECT AVG(CASE WHEN voltage < 200 or voltage > 250 THEN 220 ELSE voltage END) F ## JOIN -TDengine supports natural joins between supertables, between standard tables, and between subqueries. The difference between natural joins and inner joins is that natural joins require that the fields being joined in the supertables or standard tables must have the same name. Data or tag columns must be joined with the equivalent column in another table. +TDengine supports the `INTER JOIN` based on the timestamp primary key, that is, the `JOIN` condition must contain the timestamp primary key. As long as the requirement of timestamp-based primary key is met, `INTER JOIN` can be made between normal tables, sub-tables, super tables and sub-queries at will, and there is no limit on the number of tables. -For standard tables, only the timestamp (primary key) can be used in join operations. For example: +For standard tables: ```sql SELECT * @@ -360,7 +360,7 @@ FROM temp_tb_1 t1, pressure_tb_1 t2 WHERE t1.ts = t2.ts ``` -For supertables, tags as well as timestamps can be used in join operations. For example: +For supertables: ```sql SELECT * @@ -368,20 +368,15 @@ FROM temp_stable t1, temp_stable t2 WHERE t1.ts = t2.ts AND t1.deviceid = t2.deviceid AND t1.status=0; ``` -Similarly, join operations can be performed on the result sets of multiple subqueries. - -:::note - -The following restriction apply to JOIN statements: +For sub-table and super table： -- The number of tables or supertables in a single join operation cannot exceed 10. -- `FILL` cannot be used in a JOIN statement. -- Arithmetic operations cannot be performed on the result sets of join operation. -- `GROUP BY` is not allowed on a segment of the tables that participate in a join operation. -- `OR` cannot be used in the conditions for join operation -- Join operation can be performed only on tags or timestamps. You cannot perform a join operation on data columns. +```sql +SELECT * +FROM temp_ctable t1, temp_stable t2 +WHERE t1.ts = t2.ts AND t1.deviceid = t2.deviceid AND t1.status=0; +``` -::: +Similarly, join operations can be performed on the result sets of multiple subqueries. ## Nested Query diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 30422ca20cf44af4e1808eae2912e1591502d4c8..802eb259bf72687f9c75cdb34e3520040d3c9010 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -877,7 +877,7 @@ INTERP(expr) - Interpolation is performed based on `FILL` parameter. - `INTERP` can only be used to interpolate in single timeline. So it must be used with `partition by tbname` when it's used on a STable. - Pseudocolumn `_irowts` can be used along with `INTERP` to return the timestamps associated with interpolation points(support after version 3.0.1.4). -- Pseudocolumn `_isfilled` can be used along with `INTERP` to indicate whether the results are original records or data points generated by interpolation algorithm(support after version 3.0.2.1). +- Pseudocolumn `_isfilled` can be used along with `INTERP` to indicate whether the results are original records or data points generated by interpolation algorithm(support after version 3.0.2.3). ### LAST diff --git a/docs/en/12-taos-sql/20-keywords.md b/docs/en/12-taos-sql/20-keywords.md index 4b479b866b77e1e354d20376ccb869755af76d00..23f85947e30ae2876cf631124e6485c09f2ff8da 100644 --- a/docs/en/12-taos-sql/20-keywords.md +++ b/docs/en/12-taos-sql/20-keywords.md @@ -17,6 +17,7 @@ The following list shows all reserved keywords: - ADD - AFTER - AGGREGATE +- ALIVE - ALL - ALTER - ANALYZE diff --git a/docs/en/12-taos-sql/24-show.md b/docs/en/12-taos-sql/24-show.md index 08e8df6252e406a6910d979657c2c231ad43a71a..2db3e7cb31463e20f024f48e62d06422519ba0e7 100644 --- a/docs/en/12-taos-sql/24-show.md +++ b/docs/en/12-taos-sql/24-show.md @@ -178,76 +178,138 @@ SHOW TABLE DISTRIBUTED table_name; Shows how table data is distributed. -Examples： show table distributed d0\G; Display the block distribution of table `d0` in detailed format. +Examples： Below is an example of this command to display the block distribution of table `d0` in detailed format. +```sql +show table distributed d0\G; +``` + +
+ Show Example + 

 *************************** 1.row ***************************
 _block_dist: Total_Blocks=[5] Total_Size=[93.65 Kb] Average_size=[18.73 Kb] Compression_Ratio=[23.98 %]
 
 Total_Blocks :  Table `d0` contains total 5 blocks
+
 Total_Size:  The total size of all the data blocks in table `d0` is 93.65 KB 
+
 Average_size:  The average size of each block is 18.73 KB
+
 Compression_Ratio: The data compression rate is 23.98%
  
 *************************** 2.row ***************************
 _block_dist: Total_Rows=[20000] Inmem_Rows=[0] MinRows=[3616] MaxRows=[4096] Average_Rows=[4000]
 
 Total_Rows: Table `d0` contains 20,000 rows
+
 Inmem_Rows： The rows still in memory, i.e. not committed in disk, is 0, i.e. none such rows
+
 MinRows：  The minimum number of rows in a block is 3,616 
+
 MaxRows： The maximum number of rows in a block is 4,096B
+
 Average_Rows： The average number of rows in a block is 4,000
 
 *************************** 3.row ***************************
 _block_dist: Total_Tables=[1] Total_Files=[2]
 
 Total_Tables:  The number of child tables, 1 in this example
+
 Total_Files：   The number of files storing the table's data, 2 in this example
 
 *************************** 4.row ***************************
+
 _block_dist: --------------------------------------------------------------------------------
+
 *************************** 5.row ***************************
+
 _block_dist: 0100 |
+
 *************************** 6.row ***************************
+
 _block_dist: 0299 |
+
 *************************** 7.row ***************************
+
 _block_dist: 0498 |
+
 *************************** 8.row ***************************
+
 _block_dist: 0697 |
+
 *************************** 9.row ***************************
+
 _block_dist: 0896 |
+
 *************************** 10.row ***************************
+
 _block_dist: 1095 |
+
 *************************** 11.row ***************************
+
 _block_dist: 1294 |
+
 *************************** 12.row ***************************
+
 _block_dist: 1493 |
+
 *************************** 13.row ***************************
+
 _block_dist: 1692 |
+
 *************************** 14.row ***************************
+
 _block_dist: 1891 |
+
 *************************** 15.row ***************************
+
 _block_dist: 2090 |
+
 *************************** 16.row ***************************
+
 _block_dist: 2289 |
+
 *************************** 17.row ***************************
+
 _block_dist: 2488 |
+
 *************************** 18.row ***************************
+
 _block_dist: 2687 |
+
 *************************** 19.row ***************************
+
 _block_dist: 2886 |
+
 *************************** 20.row ***************************
+
 _block_dist: 3085 |
+
 *************************** 21.row ***************************
+
 _block_dist: 3284 |
+
 *************************** 22.row ***************************
+
 _block_dist: 3483 |||||||||||||||||  1 (20.00%)
+
 *************************** 23.row ***************************
+
 _block_dist: 3682 |
+
 *************************** 24.row ***************************
+
 _block_dist: 3881 |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||  4 (80.00%)
+
 Query OK, 24 row(s) in set (0.002444s)
 
-  The above show the block distribution percentage according to the number of rows in each block. In the above example, `_block_dist: 3483 |||||||||||||||||  1 (20.00%)` means there is one block whose rows is between 3,483 and 3,681. `_block_dist: 3881 |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||  4 (80.00%)` means there are 4 blocks whose rows is between 3,881 and 4,096. The number of blocks whose rows fall in other range is zero.
+
+
+ + The above show the block distribution percentage according to the number of rows in each block. In the above example, we can get below information: + - `_block_dist: 3483 ||||||||||||||||| 1 (20.00%)` means there is one block whose rows is between 3,483 and 3,681. + - `_block_dist: 3881 ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| 4 (80.00%)` means there are 4 blocks whose rows is between 3,881 and 4,096. - The number of blocks whose rows fall in other range is zero. ## SHOW TAGS diff --git a/docs/en/12-taos-sql/29-changes.md b/docs/en/12-taos-sql/29-changes.md index 78b6d5fc05b9b03e1e8b3af268bc357dfaa401bc..f288cd7545c1de25f6cf8b003e084ba3622524a7 100644 --- a/docs/en/12-taos-sql/29-changes.md +++ b/docs/en/12-taos-sql/29-changes.md @@ -54,7 +54,6 @@ The following data types can be used in the schema for standard tables. | 27 | GRANT | Added | Grants permissions to a user. | 28 | KILL TRANSACTION | Added | Terminates an mnode transaction. | 29 | KILL STREAM | Deprecated | Terminated a continuous query. The continuous query feature has been replaced with the stream processing feature. -| 30 | MERGE VGROUP | Added | Merges vgroups. | 31 | REVOKE | Added | Revokes permissions from a user. | 32 | SELECT | Modified | | 33 | SHOW ACCOUNTS | Deprecated | This Enterprise Edition-only statement has been removed. It returns the error "This statement is no longer supported." @@ -76,8 +75,9 @@ The following data types can be used in the schema for standard tables. | 49 | SHOW TRANSACTIONS | Added | Shows all running transactions in the system. | 50 | SHOW DNODE VARIABLES | Added | Shows the configuration of the specified dnode. | 51 | SHOW VNODES | Not supported | Shows information about vnodes in the system. Not supported. -| 52 | SPLIT VGROUP | Added | Splits a vgroup into two vgroups. -| 53 | TRIM DATABASE | Added | Deletes data that has expired and orders the remaining data in accordance with the storage configuration. +| 52 | TRIM DATABASE | Added | Deletes data that has expired and orders the remaining data in accordance with the storage configuration. +| 53 | REDISTRIBUTE VGROUP | Added | Adjust the distribution of VNODES in VGROUP. +| 54 | BALANCE VGROUP | Added | Auto adjust the distribution of VNODES in VGROUP. ## SQL Functions diff --git a/docs/en/14-reference/03-connector/05-go.mdx b/docs/en/14-reference/03-connector/05-go.mdx index df5b129cea552144d5833190d46e8a78f2fd2fa5..60407c0735bf9bcb42ae54bddcc9afa639a02fcc 100644 --- a/docs/en/14-reference/03-connector/05-go.mdx +++ b/docs/en/14-reference/03-connector/05-go.mdx @@ -355,26 +355,29 @@ The `af` package encapsulates TDengine advanced functions such as connection man #### Subscribe -* `func NewConsumer(conf *Config) (*Consumer, error)` +* `func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error)` Creates consumer group. -* `func (c *Consumer) Subscribe(topics []string) error` +* `func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error` +Note: `rebalanceCb` is reserved for compatibility purpose + +Subscribes a topic. + +* `func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error` +Note: `rebalanceCb` is reserved for compatibility purpose Subscribes to topics. -* `func (c *Consumer) Poll(timeout time.Duration) (*Result, error)` +* `func (c *Consumer) Poll(timeoutMs int) tmq.Event` Polling information. -* `func (c *Consumer) Commit(ctx context.Context, message unsafe.Pointer) error` +* `func (c *Consumer) Commit() ([]tmq.TopicPartition, error)` +Note: `tmq.TopicPartition` is reserved for compatibility purpose Commit information. -* `func (c *Consumer) FreeMessage(message unsafe.Pointer)` - -Free information. - * `func (c *Consumer) Unsubscribe() error` Unsubscribe. @@ -441,25 +444,36 @@ Close consumer. ### Subscribe via WebSocket -* `func NewConsumer(config *Config) (*Consumer, error)` +* `func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error)` - Creates consumer group. +Creates consumer group. + +* `func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error` +Note: `rebalanceCb` is reserved for compatibility purpose -* `func (c *Consumer) Subscribe(topic []string) error` +Subscribes a topic. - Subscribes to topics. +* `func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error` +Note: `rebalanceCb` is reserved for compatibility purpose -* `func (c *Consumer) Poll(timeout time.Duration) (*Result, error)` +Subscribes to topics. - Polling information. +* `func (c *Consumer) Poll(timeoutMs int) tmq.Event` -* `func (c *Consumer) Commit(messageID uint64) error` +Polling information. - Commit information. +* `func (c *Consumer) Commit() ([]tmq.TopicPartition, error)` +Note: `tmq.TopicPartition` is reserved for compatibility purpose + +Commit information. + +* `func (c *Consumer) Unsubscribe() error` + +Unsubscribe. * `func (c *Consumer) Close() error` - Close consumer. +Close consumer. For a complete example see [GitHub sample file](https://github.com/taosdata/driver-go/blob/3.0/examples/tmqoverws/main.go) diff --git a/docs/en/14-reference/03-connector/09-csharp.mdx b/docs/en/14-reference/03-connector/09-csharp.mdx index 85514f58ac1a19c7ae1a725e9b055f10280ebbb6..b73e1c262758df77d5b84c692f7098dacac70c80 100644 --- a/docs/en/14-reference/03-connector/09-csharp.mdx +++ b/docs/en/14-reference/03-connector/09-csharp.mdx @@ -252,14 +252,14 @@ ws://localhost:6041/test |Sample program |Sample program description | |--------------------------------------------------------------------------------------------------------------------|--------------------------------------------| -| [CURD](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/Query/Query.cs) | Table creation, data insertion, and query examples with TDengine.Connector | -| [JSON Tag](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/JSONTag) | Writing and querying JSON tag data with TDengine Connector | -| [stmt](https://github.com/taosdata/taos-connector-dotnet/tree/3.0/examples/Stmt) | Parameter binding with TDengine Connector | -| [schemaless](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/schemaless) | Schemaless writes with TDengine Connector | -| [async query](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/AsyncQuery/QueryAsync.cs) | Asynchronous queries with TDengine Connector | -| [Subscription](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/TMQ/TMQ.cs) | Subscription example with TDengine Connector | -| [Basic WebSocket Usage](https://github.com/taosdata/taos-connector-dotnet/blob/5a4a7cd0dbcda114447cdc6d0c6dedd8e84a52da/examples/WS/WebSocketSample.cs) | WebSocket basic data in and out with TDengine connector | -| [WebSocket Parameter Binding](https://github.com/taosdata/taos-connector-dotnet/blob/5a4a7cd0dbcda114447cdc6d0c6dedd8e84a52da/examples/WS/WebSocketSTMT.cs) | WebSocket parameter binding example | +| [CURD](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/Query/Query.cs) | Table creation, data insertion, and query examples with TDengine.Connector | +| [JSON Tag](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/JSONTag) | Writing and querying JSON tag data with TDengine Connector | +| [stmt](https://github.com/taosdata/taos-connector-dotnet/tree/3.0/examples/NET6Examples/Stmt) | Parameter binding with TDengine Connector | +| [schemaless](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/schemaless) | Schemaless writes with TDengine Connector | +| [async query](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/AsyncQuery/QueryAsync.cs) | Asynchronous queries with TDengine Connector | +| [Subscription](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/TMQ/TMQ.cs) | Subscription example with TDengine Connector | +| [Basic WebSocket Usage](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/FrameWork45/WS/WebSocketSample.cs) | WebSocket basic data in and out with TDengine connector | +| [WebSocket Parameter Binding](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/FrameWork45/WS/WebSocketSTMT.cs) | WebSocket parameter binding example | ## Important update records diff --git a/docs/en/14-reference/04-taosadapter.md b/docs/en/14-reference/04-taosadapter.md index 870cebb1033f0f0cddaec19b616228c2ac46d87c..9eb6cb921391df3ce494fddaae8caa23cb0f4adc 100644 --- a/docs/en/14-reference/04-taosadapter.md +++ b/docs/en/14-reference/04-taosadapter.md @@ -21,6 +21,7 @@ taosAdapter provides the following features. - Seamless connection to collectd - Seamless connection to StatsD - Supports Prometheus remote_read and remote_write +- Get table's VGroup ID ## taosAdapter architecture diagram @@ -178,6 +179,7 @@ See [example/config/taosadapter.toml](https://github.com/taosdata/taosadapter/bl node_export is an exporter for machine metrics. Please visit [https://github.com/prometheus/node_exporter](https://github.com/prometheus/node_exporter) for more information. - Support for Prometheus remote_read and remote_write remote_read and remote_write are interfaces for Prometheus data read and write from/to other data storage solution. Please visit [https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis) for more information. +- Get table's VGroup ID. For more information about VGroup, please refer to [primary-logic-unit](/tdinternal/arch/#primary-logic-unit). ## Interfaces @@ -199,7 +201,7 @@ Support InfluxDB query parameters as follows. - `precision` The time precision used by TDengine - `u` TDengine user name - `p` TDengine password -- `ttl` The time to live of automatically created sub-table. This value cannot be updated. TDengine will use the ttl value of the frist data of sub-table to create sub-table. For more information, please refer [Create Table](/taos-sql/table/#create-table) +- `ttl` The time to live of automatically created sub-table. This value cannot be updated. TDengine will use the ttl value of the first data of sub-table to create sub-table. For more information, please refer [Create Table](/taos-sql/table/#create-table) Note: InfluxDB token authorization is not supported at present. Only Basic authorization and query parameter validation are supported. Example: curl --request POST http://127.0.0.1:6041/influxdb/v1/write?db=test --user "root:taosdata" --data-binary "measurement,host=host1 field1=2i,field2=2.0 1577836800000000000" @@ -241,6 +243,10 @@ node_export is an exporter of hardware and OS metrics exposed by the \*NIX kerne +### Get table's VGroup ID + +You can call `http://:6041/rest/vgid?db=&table=` to get table's VGroup ID. For more information about VGroup, please refer to [primary-logic-unit](/tdinternal/arch/#primary-logic-unit). + ## Memory usage optimization methods taosAdapter will monitor its memory usage during operation and adjust it with two thresholds. Valid values are integers between 1 to 100, and represent a percentage of the system's physical memory. diff --git a/docs/en/14-reference/05-taosbenchmark.md b/docs/en/14-reference/05-taosbenchmark.md index 19feeb674060cbe0e7ec13ed4e47bb3fd85836cc..4017b12be923a9bcb5696c8b4b57c2d67b5c1378 100644 --- a/docs/en/14-reference/05-taosbenchmark.md +++ b/docs/en/14-reference/05-taosbenchmark.md @@ -92,7 +92,7 @@ taosBenchmark -f -## Command-line argument in detailed +## Command-line argument in detail - **-f/--file ** : specify the configuration file to use. This file includes All parameters. Users should not use this parameter with other parameters on the command-line. There is no default value. @@ -198,7 +198,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) - **-R/--disorder-range ** : Specify the timestamp range for the disordered data. It leads the resulting disorder timestamp as the ordered timestamp minus a random value in this range. Valid only if the percentage of disordered data specified by `-O/--disorder` is greater than 0. -- **-F/--prepare_rand ** : +- **-F/--prepared_rand ** : Specify the number of unique values in the generated random data. A value of 1 means that all data are equal. The default value is 10000. - **-a/--replica ** : @@ -216,7 +216,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) - **-? /--help** : Show help information and exit. Users should not use it with other parameters. -## Configuration file parameters in detailed +## Configuration file parameters in detail ### General configuration parameters @@ -380,7 +380,7 @@ The configuration parameters for specifying super table tag columns and data col - **num_of_records_per_req** : Writing the number of rows of records per request to TDengine, the default value is 30000. When it is set too large, the TDengine client driver will return the corresponding error message, so you need to lower the setting of this parameter to meet the writing requirements. -- **prepare_rand**: The number of unique values in the generated random data. A value of 1 means that all data are equal. The default value is 10000. +- **prepared_rand**: The number of unique values in the generated random data. A value of 1 means that all data are equal. The default value is 10000. ### Query scenario configuration parameters diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index b6bfa4bc7d57a9139992a0f1aab528b267e5bd03..c3d5de02137b61796cb9361721bf51fa6ac9dfb2 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -142,6 +142,15 @@ The parameters described in this document by the effect that they have on the sy | Meaning | Switch for allowing TDengine to collect and report service usage information | | Value Range | 0: Not allowed; 1: Allowed | | Default Value | 1 | +### crashReporting + +| Attribute | Description | +| -------- | -------------------------------------------- | +| Applicable | Server Only | +| Meaning |Switch for allowing TDengine to collect and report crash related information | +| Value Range | 0,1 0: Not allowed；1：allowed | +| Default Value | 1 | + ## Query Parameters diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index bcfcaf9ffb37206b92f265435f577b47aa452807..4e0239eb80974647ac5ab5e2f3e14bedb831859e 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -10,6 +10,14 @@ For TDengine 2.x installation packages by version, please visit [here](https://w import Release from "/components/ReleaseV3"; +## 3.0.2.3 + + + +## 3.0.2.2 + + + ## 3.0.2.1 diff --git a/docs/en/28-releases/02-tools.md b/docs/en/28-releases/02-tools.md index 6013aacc35692dc75c15de4fd2427bfaee36ea34..2d20c26c56237fd1834167f0bdf6fb2b42ffba33 100644 --- a/docs/en/28-releases/02-tools.md +++ b/docs/en/28-releases/02-tools.md @@ -10,6 +10,14 @@ For other historical version installers, please visit [here](https://www.taosdat import Release from "/components/ReleaseV3"; +## 2.4.1 + + + +## 2.4.0 + + + ## 2.3.3 diff --git a/docs/examples/csharp/stmtInsert/Program.cs b/docs/examples/csharp/stmtInsert/Program.cs index 87e1971feb8499c515206f05a1e916070ac57f4c..80cadb2ff8b596a0484d05ff15aeaa50f22ff859 100644 --- a/docs/examples/csharp/stmtInsert/Program.cs +++ b/docs/examples/csharp/stmtInsert/Program.cs @@ -42,7 +42,7 @@ namespace TDengineExample // 5. execute res = TDengine.StmtExecute(stmt); - CheckStmtRes(res, "faild to execute"); + CheckStmtRes(res, "failed to execute"); // 6. free TaosMultiBind.FreeTaosBind(tags); @@ -92,7 +92,7 @@ namespace TDengineExample int code = TDengine.StmtClose(stmt); if (code != 0) { - throw new Exception($"falied to close stmt, {code} reason: {TDengine.StmtErrorStr(stmt)} "); + throw new Exception($"failed to close stmt, {code} reason: {TDengine.StmtErrorStr(stmt)} "); } } } diff --git a/docs/examples/go/go.mod b/docs/examples/go/go.mod index 2bc1a74cb6ef14221fa384701773dc73fe3b161d..6696852312f39b93df189beba89f08c1945ef9f1 100644 --- a/docs/examples/go/go.mod +++ b/docs/examples/go/go.mod @@ -2,5 +2,5 @@ module goexample go 1.17 -require github.com/taosdata/driver-go/v3 3.0 +require github.com/taosdata/driver-go/v3 3.1.0 diff --git a/docs/examples/go/sub/main.go b/docs/examples/go/sub/main.go index a13d394a2c5009c1ad88684109b6f16b4d8a0540..1f7218936fbe457615562ded1b938daca95225cb 100644 --- a/docs/examples/go/sub/main.go +++ b/docs/examples/go/sub/main.go @@ -1,17 +1,12 @@ package main import ( - "context" - "encoding/json" "fmt" - "strconv" - "time" + "os" "github.com/taosdata/driver-go/v3/af" "github.com/taosdata/driver-go/v3/af/tmq" - "github.com/taosdata/driver-go/v3/common" - "github.com/taosdata/driver-go/v3/errors" - "github.com/taosdata/driver-go/v3/wrapper" + tmqcommon "github.com/taosdata/driver-go/v3/common/tmq" ) func main() { @@ -28,79 +23,56 @@ func main() { if err != nil { panic(err) } - config := tmq.NewConfig() - defer config.Destroy() - err = config.SetGroupID("test") if err != nil { panic(err) } - err = config.SetAutoOffsetReset("earliest") - if err != nil { - panic(err) - } - err = config.SetConnectIP("127.0.0.1") - if err != nil { - panic(err) - } - err = config.SetConnectUser("root") - if err != nil { - panic(err) - } - err = config.SetConnectPass("taosdata") + consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{ + "group.id": "test", + "auto.offset.reset": "earliest", + "td.connect.ip": "127.0.0.1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "td.connect.port": "6030", + "client.id": "test_tmq_client", + "enable.auto.commit": "false", + "enable.heartbeat.background": "true", + "experimental.snapshot.enable": "true", + "msg.with.table.name": "true", + }) if err != nil { panic(err) } - err = config.SetConnectPort("6030") + err = consumer.Subscribe("example_tmq_topic", nil) if err != nil { panic(err) } - err = config.SetMsgWithTableName(true) + _, err = db.Exec("create table example_tmq.t1 (ts timestamp,v int)") if err != nil { panic(err) } - err = config.EnableHeartBeat() + _, err = db.Exec("insert into example_tmq.t1 values(now,1)") if err != nil { panic(err) } - err = config.EnableAutoCommit(func(result *wrapper.TMQCommitCallbackResult) { - if result.ErrCode != 0 { - errStr := wrapper.TMQErr2Str(result.ErrCode) - err := errors.NewError(int(result.ErrCode), errStr) - panic(err) + for i := 0; i < 5; i++ { + ev := consumer.Poll(0) + if ev != nil { + switch e := ev.(type) { + case *tmqcommon.DataMessage: + fmt.Println(e.String()) + case tmqcommon.Error: + fmt.Fprintf(os.Stderr, "%% Error: %v: %v\n", e.Code(), e) + panic(e) + } + consumer.Commit() } - }) - if err != nil { - panic(err) } - consumer, err := tmq.NewConsumer(config) + err = consumer.Unsubscribe() if err != nil { panic(err) } - err = consumer.Subscribe([]string{"example_tmq_topic"}) + err = consumer.Close() if err != nil { panic(err) } - _, err = db.Exec("create table example_tmq.t1 (ts timestamp,v int)") - if err != nil { - panic(err) - } - _, err = db.Exec("insert into example_tmq.t1 values(now,1)") - if err != nil { - panic(err) - } - for { - result, err := consumer.Poll(time.Second) - if err != nil { - panic(err) - } - if result.Type != common.TMQ_RES_DATA { - panic("want message type 1 got " + strconv.Itoa(int(result.Type))) - } - data, _ := json.Marshal(result.Data) - fmt.Println(string(data)) - consumer.Commit(context.Background(), result.Message) - consumer.FreeMessage(result.Message) - break - } - consumer.Close() } diff --git a/docs/examples/python/connect_rest_examples.py b/docs/examples/python/connect_rest_examples.py index 900ec1022ec81ac2db761d918d1ec11c9bb26852..dba00b5a8279a3cbb3cab0a2d8b26bb312364479 100644 --- a/docs/examples/python/connect_rest_examples.py +++ b/docs/examples/python/connect_rest_examples.py @@ -15,10 +15,10 @@ cursor.execute("CREATE DATABASE power") cursor.execute("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)") # insert data -cursor.execute("""INSERT INTO power.d1001 USING power.meters TAGS(California.SanFrancisco, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) - power.d1002 USING power.meters TAGS(California.SanFrancisco, 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) - power.d1003 USING power.meters TAGS(California.LosAngeles, 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) - power.d1004 USING power.meters TAGS(California.LosAngeles, 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)""") +cursor.execute("""INSERT INTO power.d1001 USING power.meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) + power.d1002 USING power.meters TAGS('California.SanFrancisco', 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) + power.d1003 USING power.meters TAGS('California.LosAngeles', 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) + power.d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)""") print("inserted row count:", cursor.rowcount) # query data diff --git a/docs/examples/python/native_insert_example.py b/docs/examples/python/native_insert_example.py index 94fd00a6e9d1dcd2119693c4b5c862d36c219a3d..cdde7d23d24d12e11c67b6c6acc0e0b089fb5335 100644 --- a/docs/examples/python/native_insert_example.py +++ b/docs/examples/python/native_insert_example.py @@ -25,10 +25,10 @@ def create_stable(conn: taos.TaosConnection): # The generated SQL is: -# INSERT INTO d1001 USING meters TAGS(California.SanFrancisco, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) -# d1002 USING meters TAGS(California.SanFrancisco, 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) -# d1003 USING meters TAGS(California.LosAngeles, 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) -# d1004 USING meters TAGS(California.LosAngeles, 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000) +# INSERT INTO d1001 USING meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) +# d1002 USING meters TAGS('California.SanFrancisco', 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) +# d1003 USING meters TAGS('California.LosAngeles', 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) +# d1004 USING meters TAGS('California.LosAngeles', 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000) def get_sql(): global lines diff --git a/docs/zh/05-get-started/index.md b/docs/zh/05-get-started/index.md index 832310aa7c677940c7e4ca13be5f31c2d98a64dc..dd4cb8793dc4aa4c55a9bdf28ed8c4843a2cc286 100644 --- a/docs/zh/05-get-started/index.md +++ b/docs/zh/05-get-started/index.md @@ -35,13 +35,13 @@ TDengine 知识地图中涵盖了 TDengine 的各种知识点，揭示了各概
- + - - - + + +
小 T 的二维码小 T 的二维码 TDengine 微信视频号 TDengine 微信公众号
加入“物联网大数据技术前沿群”
与大家进行技术交流		关注 TDengine 微信视频号
收看技术直播与教学视频		关注 TDengine 微信公众号
阅读核心技术与行业案例文章		加入“物联网大数据技术群”
与大家进行技术交流		关注 TDengine 视频号
收看技术直播与教学视频		关注 TDengine 公众号
阅读技术文章与行业案例
diff --git a/docs/zh/05-get-started/xiaot-new.webp b/docs/zh/05-get-started/xiaot-new.webp new file mode 100644 index 0000000000000000000000000000000000000000..483b54d2ef3d8894527aa154a42cf6cd2463c579 Binary files /dev/null and b/docs/zh/05-get-started/xiaot-new.webp differ diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx index 1f5a089aaa2a051e238eedc0315c37cad643b33f..039e9eb6356f55d2cadd83cd4c3ce5e3082c04d1 100644 --- a/docs/zh/07-develop/07-tmq.mdx +++ b/docs/zh/07-develop/07-tmq.mdx @@ -115,19 +115,22 @@ class TaosConsumer(): ```go -func NewConsumer(conf *Config) (*Consumer, error) +func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error) -func (c *Consumer) Close() error - -func (c *Consumer) Commit(ctx context.Context, message unsafe.Pointer) error +// 出于兼容目的保留 rebalanceCb 参数，当前未使用 +func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error -func (c *Consumer) FreeMessage(message unsafe.Pointer) +// 出于兼容目的保留 rebalanceCb 参数，当前未使用 +func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error -func (c *Consumer) Poll(timeout time.Duration) (*Result, error) +func (c *Consumer) Poll(timeoutMs int) tmq.Event -func (c *Consumer) Subscribe(topics []string) error +// 出于兼容目的保留 tmq.TopicPartition 参数，当前未使用 +func (c *Consumer) Commit() ([]tmq.TopicPartition, error) func (c *Consumer) Unsubscribe() error + +func (c *Consumer) Close() error ``` @@ -355,50 +358,20 @@ public class MetersDeserializer extends ReferenceDeserializer { ```go -config := tmq.NewConfig() -defer config.Destroy() -err = config.SetGroupID("test") -if err != nil { - panic(err) -} -err = config.SetAutoOffsetReset("earliest") -if err != nil { - panic(err) -} -err = config.SetConnectIP("127.0.0.1") -if err != nil { - panic(err) -} -err = config.SetConnectUser("root") -if err != nil { - panic(err) -} -err = config.SetConnectPass("taosdata") -if err != nil { - panic(err) -} -err = config.SetConnectPort("6030") -if err != nil { - panic(err) -} -err = config.SetMsgWithTableName(true) -if err != nil { - panic(err) -} -err = config.EnableHeartBeat() -if err != nil { - panic(err) -} -err = config.EnableAutoCommit(func(result *wrapper.TMQCommitCallbackResult) { - if result.ErrCode != 0 { - errStr := wrapper.TMQErr2Str(result.ErrCode) - err := errors.NewError(int(result.ErrCode), errStr) - panic(err) - } -}) -if err != nil { - panic(err) +conf := &tmq.ConfigMap{ + "group.id": "test", + "auto.offset.reset": "earliest", + "td.connect.ip": "127.0.0.1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "td.connect.port": "6030", + "client.id": "test_tmq_c", + "enable.auto.commit": "false", + "enable.heartbeat.background": "true", + "experimental.snapshot.enable": "true", + "msg.with.table.name": "true", } +consumer, err := NewConsumer(conf) ``` @@ -532,11 +505,7 @@ consumer.subscribe(topics); ```go -consumer, err := tmq.NewConsumer(config) -if err != nil { - panic(err) -} -err = consumer.Subscribe([]string{"example_tmq_topic"}) +err = consumer.Subscribe("example_tmq_topic", nil) if err != nil { panic(err) } @@ -620,13 +589,17 @@ while(running){ ```go for { - result, err := consumer.Poll(time.Second) - if err != nil { - panic(err) + ev := consumer.Poll(0) + if ev != nil { + switch e := ev.(type) { + case *tmqcommon.DataMessage: + fmt.Println(e.Value()) + case tmqcommon.Error: + fmt.Fprintf(os.Stderr, "%% Error: %v: %v\n", e.Code(), e) + panic(e) + } + consumer.Commit() } - fmt.Println(result) - consumer.Commit(context.Background(), result.Message) - consumer.FreeMessage(result.Message) } ``` @@ -738,7 +711,11 @@ consumer.close(); ```go -consumer.Close() +/* Unsubscribe */ +_ = consumer.Unsubscribe() + +/* Close consumer */ +_ = consumer.Close() ``` diff --git a/docs/zh/08-connector/20-go.mdx b/docs/zh/08-connector/20-go.mdx index 0fc4007f6362697222b425c8c2c803b911b9ac8a..2aa1a58e49f34b412f12bd0d67586dc6e56cf0bc 100644 --- a/docs/zh/08-connector/20-go.mdx +++ b/docs/zh/08-connector/20-go.mdx @@ -15,7 +15,7 @@ import GoOpenTSDBTelnet from "../07-develop/03-insert-data/_go_opts_telnet.mdx" import GoOpenTSDBJson from "../07-develop/03-insert-data/_go_opts_json.mdx" import GoQuery from "../07-develop/04-query-data/_go.mdx" -`driver-go` 是 TDengine 的官方 Go 语言连接器，实现了 Go 语言[ database/sql ](https://golang.org/pkg/database/sql/) 包的接口。Go 开发人员可以通过它开发存取 TDengine 集群数据的应用软件。 +`driver-go` 是 TDengine 的官方 Go 语言连接器，实现了 Go 语言 [database/sql](https://golang.org/pkg/database/sql/) 包的接口。Go 开发人员可以通过它开发存取 TDengine 集群数据的应用软件。 `driver-go` 提供两种建立连接的方式。一种是**原生连接**，它通过 TDengine 客户端驱动程序（taosc）原生连接 TDengine 运行实例，支持数据写入、查询、订阅、schemaless 接口和参数绑定接口等功能。另外一种是 **REST 连接**，它通过 taosAdapter 提供的 REST 接口连接 TDengine 运行实例。REST 连接实现的功能特性集合和原生连接有少量不同。 @@ -112,6 +112,7 @@ REST 连接支持所有能运行 Go 的平台。 ```text username:password@protocol(address)/dbname?param=value ``` + ### 使用连接器进行连接 @@ -176,6 +177,7 @@ func main() { } } ``` + @@ -207,6 +209,7 @@ func main() { } } ``` + @@ -357,33 +360,32 @@ func main() { #### 订阅 -* `func NewConsumer(conf *Config) (*Consumer, error)` - -创建消费者。 +* `func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error)` -* `func (c *Consumer) Subscribe(topics []string) error` + 创建消费者。 -订阅主题。 +* `func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error` +注意：出于兼容目的保留 `rebalanceCb` 参数，当前未使用 -* `func (c *Consumer) Poll(timeout time.Duration) (*Result, error)` + 订阅单个主题。 -轮询消息。 +* `func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error` +注意：出于兼容目的保留 `rebalanceCb` 参数，当前未使用 -* `func (c *Consumer) Commit(ctx context.Context, message unsafe.Pointer) error` + 订阅主题。 -提交消息。 +* `func (c *Consumer) Poll(timeoutMs int) tmq.Event` -* `func (c *Consumer) FreeMessage(message unsafe.Pointer)` + 轮询消息。 -释放消息。 +* `func (c *Consumer) Commit() ([]tmq.TopicPartition, error)` +注意：出于兼容目的保留 `tmq.TopicPartition` 参数，当前未使用 -* `func (c *Consumer) Unsubscribe() error` - -取消订阅。 + 提交消息。 * `func (c *Consumer) Close() error` -关闭消费者。 + 关闭连接。 #### schemaless @@ -443,25 +445,32 @@ func main() { ### 通过 WebSocket 订阅 -* `func NewConsumer(config *Config) (*Consumer, error)` +* `func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error)` + + 创建消费者。 + +* `func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error` +注意：出于兼容目的保留 `rebalanceCb` 参数，当前未使用 - 创建消费者。 + 订阅单个主题。 -* `func (c *Consumer) Subscribe(topic []string) error` +* `func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error` +注意：出于兼容目的保留 `rebalanceCb` 参数，当前未使用 - 订阅主题。 + 订阅主题。 -* `func (c *Consumer) Poll(timeout time.Duration) (*Result, error)` +* `func (c *Consumer) Poll(timeoutMs int) tmq.Event` - 轮询消息。 + 轮询消息。 -* `func (c *Consumer) Commit(messageID uint64) error` +* `func (c *Consumer) Commit() ([]tmq.TopicPartition, error)` +注意：出于兼容目的保留 `tmq.TopicPartition` 参数，当前未使用 - 提交消息。 + 提交消息。 * `func (c *Consumer) Close() error` - 关闭消费者。 + 关闭连接。 完整订阅示例参见 [GitHub 示例文件](https://github.com/taosdata/driver-go/blob/3.0/examples/tmqoverws/main.go) diff --git a/docs/zh/08-connector/40-csharp.mdx b/docs/zh/08-connector/40-csharp.mdx index a1a161d4eea9e2534ebb3a573211dcfae5dbb21f..c70a8a633e4df46df208c829cce772912cb36f06 100644 --- a/docs/zh/08-connector/40-csharp.mdx +++ b/docs/zh/08-connector/40-csharp.mdx @@ -253,14 +253,14 @@ namespace TDengineExample |示例程序 | 示例程序描述 | |--------------------------------------------------------------------------------------------------------------------|--------------------------------------------| -| [CURD](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/Query/Query.cs) | 使用 TDengine.Connector 实现的建表、插入、查询示例 | -| [JSON Tag](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/JSONTag) | 使用 TDengine.Connector 实现的写入和查询 JSON tag 类型数据的示例 | -| [stmt](https://github.com/taosdata/taos-connector-dotnet/tree/3.0/examples/Stmt) | 使用 TDengine.Connector 实现的参数绑定插入和查询的示例 | -| [schemaless](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/schemaless) | 使用 TDengine.Connector 实现的使用 schemaless 写入的示例 | -| [async query](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/AsyncQuery/QueryAsync.cs) | 使用 TDengine.Connector 实现的异步查询的示例 | -| [数据订阅（TMQ）](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/TMQ/TMQ.cs) | 使用 TDengine.Connector 实现的订阅数据的示例 | -| [Basic WebSocket Usage](https://github.com/taosdata/taos-connector-dotnet/blob/5a4a7cd0dbcda114447cdc6d0c6dedd8e84a52da/examples/WS/WebSocketSample.cs) | 使用 TDengine.Connector 的 WebSocket 基本的示例 | -| [Basic WebSocket STMT](https://github.com/taosdata/taos-connector-dotnet/blob/5a4a7cd0dbcda114447cdc6d0c6dedd8e84a52da/examples/WS/WebSocketSTMT.cs) | 使用 TDengine.Connector 的 WebSocket STMT 基本的示例 | +| [CURD](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/Query/Query.cs) | 使用 TDengine.Connector 实现的建表、插入、查询示例 | +| [JSON Tag](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/JSONTag) | 使用 TDengine.Connector 实现的写入和查询 JSON tag 类型数据的示例 | +| [stmt](https://github.com/taosdata/taos-connector-dotnet/tree/3.0/examples/NET6Examples/Stmt) | 使用 TDengine.Connector 实现的参数绑定插入和查询的示例 | +| [schemaless](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/schemaless) | 使用 TDengine.Connector 实现的使用 schemaless 写入的示例 | +| [async query](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/AsyncQuery/QueryAsync.cs) | 使用 TDengine.Connector 实现的异步查询的示例 | +| [数据订阅（TMQ）](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/TMQ/TMQ.cs) | 使用 TDengine.Connector 实现的订阅数据的示例 | +| [Basic WebSocket Usage](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/FrameWork45/WS/WebSocketSample.cs) | 使用 TDengine.Connector 的 WebSocket 基本的示例 | +| [Basic WebSocket STMT](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/FrameWork45/WS/WebSocketSTMT.cs) | 使用 TDengine.Connector 的 WebSocket STMT 基本的示例 | ## 重要更新记录 diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index df52a0890b6ff091b8d5cfb051618c88e8195cf0..7d9566f4f257ed7a0ffdcdfb06d0d7116bf24043 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -30,8 +30,10 @@ database_option: { | WAL_LEVEL {1 | 2} | VGROUPS value | SINGLE_STABLE {0 | 1} + | STT_TRIGGER value | TABLE_PREFIX value | TABLE_SUFFIX value + | TSDB_PAGESIZE value | WAL_RETENTION_PERIOD value | WAL_ROLL_PERIOD value | WAL_RETENTION_SIZE value @@ -69,8 +71,10 @@ database_option: { - SINGLE_STABLE：表示此数据库中是否只可以创建一个超级表，用于超级表列非常多的情况。 - 0：表示可以创建多张超级表。 - 1：表示只可以创建一张超级表。 +- STT_TRIGGER：表示落盘文件触发文件合并的个数。默认为 1，范围 1 到 16。对于少表高频场景，此参数建议使用默认配置，或较小的值；而对于多表低频场景，此参数建议配置较大的值。 - TABLE_PREFIX：内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的前缀的长度。 - TABLE_SUFFIX：内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的后缀的长度。 +- TSDB_PAGESIZE：一个 VNODE 中时序数据存储引擎的页大小，单位为 KB，默认为 4 KB。范围为 1 到 16384，即 1 KB到 16 MB。 - WAL_RETENTION_PERIOD：wal 文件的额外保留策略，用于数据订阅。wal 的保存时长，单位为 s。单副本默认为 0，即落盘后立即删除。-1 表示不删除。多副本默认为 4 天。 - WAL_RETENTION_SIZE：wal 文件的额外保留策略，用于数据订阅。wal 的保存的最大上限，单位为 KB。单副本默认为 0，即落盘后立即删除。多副本默认为-1，表示不删除。 - WAL_ROLL_PERIOD：wal 文件切换时长，单位为 s。当 wal 文件创建并写入后，经过该时间，会自动创建一个新的 wal 文件。单副本默认为 0，即仅在落盘时创建新文件。多副本默认为 1 天。 @@ -112,6 +116,10 @@ alter_database_options: alter_database_option: { CACHEMODEL {'none' | 'last_row' | 'last_value' | 'both'} | CACHESIZE value + | BUFFER value + | PAGES value + | REPLICA value + | STT_TRIGGER value | WAL_LEVEL value | WAL_FSYNC_PERIOD value | KEEP value @@ -154,3 +162,19 @@ TRIM DATABASE db_name; ``` 删除过期数据，并根据多级存储的配置归整数据。 + +## 调整VGROUP中VNODE的分布 + +```sql +REDISTRIBUTE VGROUP vgroup_no DNODE dnode_id1 [DNODE dnode_id2] [DNODE dnode_id3] +``` + +按照给定的dnode列表，调整vgroup中的vnode分布。因为副本数目最大为3，所以最多输入3个dnode。 + +## 自动调整VGROUP中VNODE的分布 + +```sql +BALANCE VGROUP +``` + +自动调整集群所有vgroup中的vnode分布，相当于在vnode级别对集群进行数据的负载均衡操作。 \ No newline at end of file diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index 3b681f401c826a8b93ba446ad2804cb37c8c7bf7..8244c0f27a271347938e8b55cd5d0f4938df0ea7 100644 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -350,9 +350,9 @@ SELECT AVG(CASE WHEN voltage < 200 or voltage > 250 THEN 220 ELSE voltage END) F ## JOIN 子句 -TDengine 支持“普通表与普通表之间”、“超级表与超级表之间”、“子查询与子查询之间” 进行自然连接。自然连接与内连接的主要区别是，自然连接要求参与连接的字段在不同的表/超级表中必须是同名字段。也即，TDengine 在连接关系的表达中，要求必须使用同名数据列/标签列的相等关系。 +TDengine 支持基于时间戳主键的内连接，即 JOIN 条件必须包含时间戳主键。只要满足基于时间戳主键这个要求，普通表、子表、超级表和子查询之间可以随意的进行内连接，且对表个数没有限制。 -在普通表与普通表之间的 JOIN 操作中，只能使用主键时间戳之间的相等关系。例如： +普通表与普通表之间的 JOIN 操作： ```sql SELECT * @@ -360,7 +360,7 @@ FROM temp_tb_1 t1, pressure_tb_1 t2 WHERE t1.ts = t2.ts ``` -在超级表与超级表之间的 JOIN 操作中，除了主键时间戳一致的条件外，还要求引入能实现一一对应的标签列的相等关系。例如： +超级表与超级表之间的 JOIN 操作： ```sql SELECT * @@ -368,20 +368,15 @@ FROM temp_stable t1, temp_stable t2 WHERE t1.ts = t2.ts AND t1.deviceid = t2.deviceid AND t1.status=0; ``` -类似地，也可以对多个子查询的查询结果进行 JOIN 操作。 - -:::note - -JOIN 语句存在如下限制要求： +子表与超级表之间的 JOIN 操作： -- 参与一条语句中 JOIN 操作的表/超级表最多可以有 10 个。 -- 在包含 JOIN 操作的查询语句中不支持 FILL。 -- 暂不支持参与 JOIN 操作的表之间聚合后的四则运算。 -- 不支持只对其中一部分表做 GROUP BY。 -- JOIN 查询的不同表的过滤条件之间不能为 OR。 -- JOIN 查询要求连接条件不能是普通列，只能针对标签和主时间字段列（第一列）。 +```sql +SELECT * +FROM temp_ctable t1, temp_stable t2 +WHERE t1.ts = t2.ts AND t1.deviceid = t2.deviceid AND t1.status=0; +``` -::: +类似地，也可以对多个子查询的查询结果进行 JOIN 操作。 ## 嵌套查询 diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index cb99c83cc59aede3061ddc4a2c43278388961d8b..81fdb46f25a609f829401509aa87fb6c29350f90 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -880,7 +880,7 @@ INTERP(expr) - INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。 - INTERP 只能在一个时间序列内进行插值，因此当作用于超级表时必须跟 partition by tbname 一起使用。 - INTERP 可以与伪列 _irowts 一起使用，返回插值点所对应的时间戳(3.0.1.4版本以后支持)。 -- INTERP 可以与伪列 _isfilled 一起使用，显示返回结果是否为原始记录或插值算法产生的数据(3.0.2.1版本以后支持)。 +- INTERP 可以与伪列 _isfilled 一起使用，显示返回结果是否为原始记录或插值算法产生的数据(3.0.2.3版本以后支持)。 ### LAST diff --git a/docs/zh/12-taos-sql/20-keywords.md b/docs/zh/12-taos-sql/20-keywords.md index 8013698fced2f31ca21dfa220066b027e71cb856..8fd704ef551c9ebda61678bd129917e67cbfbef8 100644 --- a/docs/zh/12-taos-sql/20-keywords.md +++ b/docs/zh/12-taos-sql/20-keywords.md @@ -18,6 +18,7 @@ description: TDengine 保留关键字的详细列表 - ADD - AFTER - AGGREGATE +- ALIVE - ALL - ALTER - ANALYZE diff --git a/docs/zh/12-taos-sql/24-show.md b/docs/zh/12-taos-sql/24-show.md index a65746e7f93f438404703886127b56a325486daf..2b875199b501136afef4ce0ba1cbcdd1e0abc933 100644 --- a/docs/zh/12-taos-sql/24-show.md +++ b/docs/zh/12-taos-sql/24-show.md @@ -183,6 +183,10 @@ SHOW TABLE DISTRIBUTED table_name; 语句： show table distributed d0\G; 竖行显示表 d0 的 BLOCK 分布情况 +
+ 显示示例 + 

+
 *************************** 1.row ***************************
 
 _block_dist: Total_Blocks=[5] Total_Size=[93.65 Kb] Average_size=[18.73 Kb] Compression_Ratio=[23.98 %]
@@ -244,6 +248,8 @@ _block_dist: 3881 ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
 
 Query OK, 24 row(s) in set (0.002444s)
 
+
+
上面是块中包含数据行数的块儿分布情况图，这里的 0100 0299 0498 … 表示的是每个块中包含的数据行数，上面的意思就是这个表的 5 个块，分布在 3483 ~3681 行的块有 1 个，占整个块的 20%，分布在 3881 ~ 4096（最大行数）的块数为 4 个，占整个块的 80%， 其它区域内分布块数为 0。 diff --git a/docs/zh/12-taos-sql/29-changes.md b/docs/zh/12-taos-sql/29-changes.md index 1ab7f4f0169932576edf9087a825367573713b03..9d67533cdea5c055c8d3ff58000fb80ac640e271 100644 --- a/docs/zh/12-taos-sql/29-changes.md +++ b/docs/zh/12-taos-sql/29-changes.md @@ -54,7 +54,6 @@ description: "TDengine 3.0 版本的语法变更说明" | 27 | GRANT | 新增 | 授予用户权限。 | 28 | KILL TRANSACTION | 新增 | 终止管理节点的事务。 | 29 | KILL STREAM | 废除 | 终止连续查询。3.0版本不再支持连续查询，而是用更通用的流计算来代替。 -| 30 | MERGE VGROUP | 新增 | 合并VGROUP。 | 31 | REVOKE | 新增 | 回收用户权限。 | 32 | SELECT | 调整 |
  • SELECT关闭隐式结果列，输出列均需要由SELECT子句来指定。
  • DISTINCT功能全面支持。2.x版本只支持对标签列去重，并且不可以和JOIN、GROUP BY等子句混用。
  • JOIN功能增强。增加支持：JOIN后WHERE条件中有OR条件；JOIN后的多表运算；JOIN后的多表GROUP BY。
  • FROM后子查询功能大幅增强。不限制子查询嵌套层数；支持子查询和UNION ALL混合使用；移除其他一些之前版本的语法限制。
  • WHERE后可以使用任意的标量表达式。
  • GROUP BY功能增强。支持任意标量表达式及其组合的分组。
  • SESSION可以用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。
  • STATE_WINDOW可以用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。
  • ORDER BY功能大幅增强。不再必须和GROUP BY子句一起使用；不再有排序表达式个数的限制；增加支持NULLS FIRST/LAST语法功能；支持符合语法语义的任意表达式。
  • 新增PARTITION BY语法。替代原来的GROUP BY tags。
| 33 | SHOW ACCOUNTS | 废除 | 2.x中为企业版功能，3.0不再支持。语法暂时保留了，执行报“This statement is no longer supported”错误。 @@ -76,8 +75,9 @@ description: "TDengine 3.0 版本的语法变更说明" | 49 | SHOW TRANSACTIONS | 新增 | 显示当前系统中正在执行的事务的信息。 | 50 | SHOW DNODE VARIABLES | 新增 |显示指定DNODE的配置参数。 | 51 | SHOW VNODES | 暂不支持 | 显示当前系统中VNODE的信息。3.0.0版本暂不支持。 -| 52 | SPLIT VGROUP | 新增 | 拆分VGROUP。 -| 53 | TRIM DATABASE | 新增 | 删除过期数据，并根据多级存储的配置归整数据。 +| 52 | TRIM DATABASE | 新增 | 删除过期数据，并根据多级存储的配置归整数据。 +| 53 | REDISTRIBUTE VGROUP | 新增 | 调整VGROUP中VNODE的分布。 +| 54 | BALANCE VGROUP | 新增 | 自动调整VGROUP中VNODE的分布。 ## SQL 函数变更 @@ -94,6 +94,7 @@ description: "TDengine 3.0 版本的语法变更说明" | 9 | SAMPLE | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 | 10 | STATECOUNT | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 | 11 | STATEDURATION | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 +| 12 | TIMETRUNCATE | 增强 | 增加ignore_timezone参数，可选是否使用，默认值为1. ## SCHEMALESS 变更 diff --git a/docs/zh/14-reference/04-taosadapter.md b/docs/zh/14-reference/04-taosadapter.md index 0909ddf639b621b39af26e7bd378f8bf7e3b99e7..5c155bdd6ebe9f1a4a38886f30b753cbda4ab6ab 100644 --- a/docs/zh/14-reference/04-taosadapter.md +++ b/docs/zh/14-reference/04-taosadapter.md @@ -21,6 +21,7 @@ taosAdapter 提供以下功能： - 无缝连接到 collectd - 无缝连接到 StatsD - 支持 Prometheus remote_read 和 remote_write +- 获取 table 所在的虚拟节点组（VGroup）的 VGroup ID ## taosAdapter 架构图 @@ -178,6 +179,7 @@ AllowWebSockets node_export 是一个机器指标的导出器。请访问 [https://github.com/prometheus/node_exporter](https://github.com/prometheus/node_exporter) 了解更多信息。 - 支持 Prometheus remote_read 和 remote_write remote_read 和 remote_write 是 Prometheus 数据读写分离的集群方案。请访问[https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis) 了解更多信息。 +- 获取 table 所在的虚拟节点组（VGroup）的 VGroup ID。关于虚拟节点组（VGroup）的更多信息，请访问[整体架构文档](/tdinternal/arch/#主要逻辑单元) 。 ## 接口 @@ -240,6 +242,10 @@ Prometheus 使用的由 \*NIX 内核暴露的硬件和操作系统指标的输 +### 获取 table 的 VGroup ID + +可以访问 http 接口 `http://:6041/rest/vgid?db=&table=` 获取 table 的 VGroup ID。关于虚拟节点组（VGroup）的更多信息，请访问[整体架构文档](/tdinternal/arch/#主要逻辑单元) 。 + ## 内存使用优化方法 taosAdapter 将监测自身运行过程中内存使用率并通过两个阈值进行调节。有效值范围为 -1 到 100 的整数，单位为系统物理内存的百分比。 @@ -282,7 +288,7 @@ http 返回内容： ## taosAdapter 监控指标 -taosAdapter 采集 http 相关指标、cpu 百分比和内存百分比。 +taosAdapter 采集 http 相关指标、CPU 百分比和内存百分比。 ### http 接口 @@ -294,13 +300,13 @@ http://:6041/metrics ### 写入 TDengine -taosAdapter 支持将 http 监控、cpu 百分比和内存百分比写入 TDengine。 +taosAdapter 支持将 http 监控、CPU 百分比和内存百分比写入 TDengine。 有关配置参数 | **配置项** | **描述** | **默认值** | |-------------------------|--------------------------------------------|----------| -| monitor.collectDuration | cpu 和内存采集间隔 | 3s | +| monitor.collectDuration | CPU 和内存采集间隔 | 3s | | monitor.identity | 当前taosadapter 的标识符如果不设置将使用 'hostname:port' | | | monitor.incgroup | 是否是 cgroup 中运行(容器中运行设置为 true) | false | | monitor.writeToTD | 是否写入到 TDengine | false | diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index bc62a536e535b4de642d66849de7e6b10373a191..135d97e8fbe44daa07a1461f0146bc80bd9c66c8 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -134,6 +134,24 @@ taos --dump-config | 取值范围 | 1-200000 | | 缺省值 | 30 | +### telemetryReporting + +| 属性 | 说明 | +| -------- | -------------------------------------------- | +| 适用范围 | 仅服务端适用 | +| 含义 |是否上传 telemetry | +| 取值范围 | 0,1 0: 不上传；1：上传 | +| 缺省值 | 1 | + +### crashReporting + +| 属性 | 说明 | +| -------- | -------------------------------------------- | +| 适用范围 | 仅服务端适用 | +| 含义 |是否上传 crash 信息 | +| 取值范围 | 0,1 0: 不上传；1：上传 | +| 缺省值 | 1 | + ## 查询相关 ### queryPolicy @@ -698,7 +716,7 @@ charset 的有效值是 UTF-8。 | 2 | numOfThreadsPerCore | 是 | 否 | 有其它参数设置多种线程池的大小 | | 3 | numOfMnodes | 是 | 否 | 通过 create mnode 命令动态创建 mnode | | 4 | vnodeBak | 是 | 否 | 3.0 行为未知 | -| 5 | balance | 是 | 否 | 负载均衡功能由 split/merge vgroups 实现 | +| 5 | balance | 是 | 否 | 负载均衡功能由 split/merge vgroups 实现 （暂不支持） | | 6 | balanceInterval | 是 | 否 | 随着 balance 参数失效 | | 7 | offlineThreshold | 是 | 否 | 3.0 行为未知 | | 8 | role | 是 | 否 | 由 supportVnode 决定是否能够创建 | diff --git a/docs/zh/27-train-faq/01-faq.md b/docs/zh/27-train-faq/01-faq.md index 595b69b08b87ee33e27937fb89b84adc41c89d08..c6ecbe471a5c0863bac80ed6edfa6abd1e13c010 100644 --- a/docs/zh/27-train-faq/01-faq.md +++ b/docs/zh/27-train-faq/01-faq.md @@ -243,3 +243,8 @@ sudo launchctl load -w /Library/LaunchDaemons/limit.maxfiles.plist ``` launchctl limit maxfiles ``` +### 19 建库时提示Out of dnode +该提示是创建db的vnode数量不够了，需要的vnode不能超过了dnode中vnode的上限。因为系统默认是一个dnode中有cpu核数两倍的vnode，也可以通过配置文件中的参数supportVnodes控制。 +正常调大taos.cfg种这个supportVnodes参数即可。 + + diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index 0fe6555162dc2b043288bfba882dab621916f35b..29403f3874970ad32431ae54446ce08350959e5c 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,6 +10,14 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.0.2.3 + + + +## 3.0.2.2 + + + ## 3.0.2.1 diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md index 331f6832c00888941f4c1a22565a3d5d62512ade..f10d97ebb9e2a316e55963c8c6efa81ecf72e3c2 100644 --- a/docs/zh/28-releases/02-tools.md +++ b/docs/zh/28-releases/02-tools.md @@ -10,6 +10,14 @@ taosTools 各版本安装包下载链接如下： import Release from "/components/ReleaseV3"; +## 2.4.1 + + + +## 2.4.0 + + + ## 2.3.3 diff --git a/include/client/taos.h b/include/client/taos.h index 647f906d4fc8490f44b025cda30c3cf86867835c..cf410a42daf1e9c401af767497a603aa12c7a536 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -161,6 +161,8 @@ DLL_EXPORT int taos_stmt_set_tags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags) DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name); DLL_EXPORT int taos_stmt_get_tag_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields); DLL_EXPORT int taos_stmt_get_col_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields); +// let stmt to reclaim TAOS_FIELD_E that was allocated by `taos_stmt_get_tag_fields`/`taos_stmt_get_col_fields` +DLL_EXPORT void taos_stmt_reclaim_fields(TAOS_STMT *stmt, TAOS_FIELD_E *fields); DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); @@ -206,6 +208,7 @@ DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res); DLL_EXPORT const char *taos_get_server_info(TAOS *taos); DLL_EXPORT const char *taos_get_client_info(); +DLL_EXPORT int taos_get_current_db(TAOS *taos, char *database, int len, int *required); DLL_EXPORT const char *taos_errstr(TAOS_RES *res); DLL_EXPORT int taos_errno(TAOS_RES *res); @@ -218,6 +221,7 @@ DLL_EXPORT const void *taos_get_raw_block(TAOS_RES *res); DLL_EXPORT int taos_get_db_route_info(TAOS *taos, const char *db, TAOS_DB_ROUTE_INFO *dbInfo); DLL_EXPORT int taos_get_table_vgId(TAOS *taos, const char *db, const char *table, int *vgId); +DLL_EXPORT int taos_get_tables_vgId(TAOS *taos, const char *db, const char *table[], int tableNum, int *vgId); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); diff --git a/include/common/systable.h b/include/common/systable.h index 6f65c1e8b870d4a42427173bf3ea17ae7ade0ce1..9b5f66f64c6fb41c7479c726ab02dc96d08e8ef5 100644 --- a/include/common/systable.h +++ b/include/common/systable.h @@ -36,6 +36,7 @@ extern "C" { #define TSDB_INS_TABLE_STABLES "ins_stables" #define TSDB_INS_TABLE_TABLES "ins_tables" #define TSDB_INS_TABLE_TAGS "ins_tags" +#define TSDB_INS_TABLE_COLS "ins_columns" #define TSDB_INS_TABLE_TABLE_DISTRIBUTED "ins_table_distributed" #define TSDB_INS_TABLE_USERS "ins_users" #define TSDB_INS_TABLE_LICENCES "ins_grants" diff --git a/include/common/taosdef.h b/include/common/taosdef.h index bf4de9d4ded1d0955bef05b1e3000be0bf34d8aa..d1ca446904594fa57d54031d5b39e529795d70da 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -30,6 +30,11 @@ typedef int64_t tb_uid_t; #define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX)) #define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) +//define show cluster alive and show db.alive +#define SHOW_STATUS_NOT_AVAILABLE 0 +#define SHOW_STATUS_AVAILABLE 1 +#define SHOW_STATUS_HALF_AVAILABLE 2 + typedef enum { TSDB_SUPER_TABLE = 1, // super table TSDB_CHILD_TABLE = 2, // table created from super table diff --git a/include/common/tcommon.h b/include/common/tcommon.h index aad69862f0ec5591fccbf0e0f8e024c34e426be2..c6e21af644d1054ce91d31b137fac030abcc749a 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -162,6 +162,7 @@ typedef enum EStreamType { STREAM_PULL_DATA, STREAM_PULL_OVER, STREAM_FILL_OVER, + STREAM_CREATE_CHILD_TABLE, } EStreamType; #pragma pack(push, 1) @@ -205,8 +206,6 @@ typedef struct SDataBlockInfo { TSKEY watermark; // used for stream char parTbName[TSDB_TABLE_NAME_LEN]; // used for stream partition - int32_t tagLen; - void* pTag; // used for stream partition } SDataBlockInfo; typedef struct SSDataBlock { @@ -292,6 +291,7 @@ typedef struct STableBlockDistInfo { uint16_t numOfFiles; uint32_t numOfTables; uint32_t numOfBlocks; + uint32_t numOfVgroups; uint64_t totalSize; uint64_t totalRows; int32_t maxRows; @@ -341,7 +341,7 @@ typedef struct SExprInfo { typedef struct { const char* key; - int32_t keyLen; + size_t keyLen; uint8_t type; union { const char* value; @@ -350,7 +350,7 @@ typedef struct { double d; float f; }; - int32_t length; + size_t length; } SSmlKv; #define QUERY_ASC_FORWARD_STEP 1 @@ -378,6 +378,11 @@ typedef struct SSortExecInfo { #define CALCULATE_END_TS_COLUMN_INDEX 5 #define TABLE_NAME_COLUMN_INDEX 6 +// stream create table block column +#define UD_TABLE_NAME_COLUMN_INDEX 0 +#define UD_GROUPID_COLUMN_INDEX 1 +#define UD_TAG_COLUMN_INDEX 2 + #ifdef __cplusplus } #endif diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index e1d3b016113e6a117f81d2ceaf2034da38e7879c..20ffb48ab0173c8f811b2ab620d977e96256425d 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -41,6 +41,12 @@ typedef struct SBlockOrderInfo { BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \ } while (0) +#define colDataSetNull_f_s(c_, r_) \ + do { \ + colDataSetNull_f((c_)->nullbitmap, r_); \ + memset(((char*)(c_)->pData) + (c_)->info.bytes * (r_), 0, (c_)->info.bytes); \ + } while (0) + #define colDataClearNull_f(bm_, r_) \ do { \ BMCharPos(bm_, r_) &= ((char)(~(1u << (7u - BitPos(r_))))); \ @@ -136,7 +142,7 @@ static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uin if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { colDataSetNull_var(pColumnInfoData, currentRow); // it is a null value of VAR type. } else { - colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow); + colDataSetNull_f_s(pColumnInfoData, currentRow); } pColumnInfoData->hasNull = true; @@ -151,6 +157,7 @@ static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, ui for (int32_t i = start; i < start + nRows; ++i) { colDataSetNull_f(pColumnInfoData->nullbitmap, i); } + memset(pColumnInfoData->pData + start * pColumnInfoData->info.bytes, 0, pColumnInfoData->info.bytes * nRows); } pColumnInfoData->hasNull = true; @@ -231,7 +238,6 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows, bool clearPayload); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); -int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRows); void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows); void blockDataCleanup(SSDataBlock* pDataBlock); @@ -265,7 +271,7 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag); // for debug char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf); -int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataBlocks, STSchema* pTSchema, int32_t vgId, +int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pDataBlocks, const STSchema* pTSchema, int64_t uid, int32_t vgId, tb_uid_t suid); char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 6855287fb2d354f52998342d368c46b3a852bf89..e0aacbfec9db8804b4003417689f404b78549afb 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -44,18 +44,38 @@ typedef struct SColData SColData; #define HAS_VALUE ((uint8_t)0x4) // bitmap ================================ -const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, - {0b00000000, 0b00000100, 0b00001000, 2}, - {0b00000000, 0b00010000, 0b00100000, 4}, - {0b00000000, 0b01000000, 0b10000000, 6}}; - -#define N1(n) ((((uint8_t)1) << (n)) - 1) -#define BIT1_SIZE(n) ((((n)-1) >> 3) + 1) -#define BIT2_SIZE(n) ((((n)-1) >> 2) + 1) -#define SET_BIT1(p, i, v) ((p)[(i) >> 3] = (p)[(i) >> 3] & N1((i)&7) | (((uint8_t)(v)) << ((i)&7))) -#define GET_BIT1(p, i) (((p)[(i) >> 3] >> ((i)&7)) & ((uint8_t)1)) -#define SET_BIT2(p, i, v) ((p)[(i) >> 2] = (p)[(i) >> 2] & N1(BIT2_MAP[(i)&3][3]) | BIT2_MAP[(i)&3][(v)]) -#define GET_BIT2(p, i) (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3)) +const static uint8_t BIT1_MAP[8] = {0b11111110, 0b11111101, 0b11111011, 0b11110111, + 0b11101111, 0b11011111, 0b10111111, 0b01111111}; + +const static uint8_t BIT2_MAP[4] = {0b11111100, 0b11110011, 0b11001111, 0b00111111}; + +#define ONE ((uint8_t)1) +#define THREE ((uint8_t)3) +#define DIV_8(i) ((i) >> 3) +#define MOD_8(i) ((i)&7) +#define DIV_4(i) ((i) >> 2) +#define MOD_4(i) ((i)&3) +#define MOD_4_TIME_2(i) (MOD_4(i) << 1) +#define BIT1_SIZE(n) (DIV_8((n)-1) + 1) +#define BIT2_SIZE(n) (DIV_4((n)-1) + 1) +#define SET_BIT1(p, i, v) ((p)[DIV_8(i)] = (p)[DIV_8(i)] & BIT1_MAP[MOD_8(i)] | ((v) << MOD_8(i))) +#define SET_BIT1_EX(p, i, v) \ + do { \ + if (MOD_8(i) == 0) { \ + (p)[DIV_8(i)] = 0; \ + } \ + SET_BIT1(p, i, v); \ + } while (0) +#define GET_BIT1(p, i) (((p)[DIV_8(i)] >> MOD_8(i)) & ONE) +#define SET_BIT2(p, i, v) ((p)[DIV_4(i)] = (p)[DIV_4(i)] & BIT2_MAP[MOD_4(i)] | ((v) << MOD_4_TIME_2(i))) +#define SET_BIT2_EX(p, i, v) \ + do { \ + if (MOD_4(i) == 0) { \ + (p)[DIV_4(i)] = 0; \ + } \ + SET_BIT2(p, i, v); \ + } while (0) +#define GET_BIT2(p, i) (((p)[DIV_4(i)] >> MOD_4_TIME_2(i)) & THREE) // SBuffer ================================ struct SBuffer { @@ -70,9 +90,6 @@ int32_t tBufferInit(SBuffer *pBuffer, int64_t size); int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData); int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData); -// STSchema ================================ -void tDestroyTSchema(STSchema *pTSchema); - // SColVal ================================ #define CV_FLAG_VALUE ((int8_t)0x0) #define CV_FLAG_NONE ((int8_t)0x1) @@ -87,8 +104,12 @@ void tDestroyTSchema(STSchema *pTSchema); #define COL_VAL_IS_VALUE(CV) ((CV)->flag == CV_FLAG_VALUE) // SRow ================================ -int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer); +int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow); void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); +void tRowDestroy(SRow *pRow); +void tRowSort(SArray *aRowP); +int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag); +int32_t tRowAppendToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData); // SRowIter ================================ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter); @@ -110,15 +131,28 @@ void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remov int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf); // SColData ================================ +typedef void *(*xMallocFn)(void *, int32_t); void tColDataDestroy(void *ph); void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn); void tColDataClear(SColData *pColData); +void tColDataDeepClear(SColData *pColData); int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal); void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal); uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal); -int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest); +int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMalloc, void *arg); extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull); +// for stmt bind +int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind); +void tColDataSortMerge(SArray *colDataArr); + +//for raw block +int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t bytes, + int32_t nRows, char* lengthOrbitmap, char *data); +// for encode/decode +int32_t tPutColData(uint8_t *pBuf, SColData *pColData); +int32_t tGetColData(uint8_t *pBuf, SColData *pColData); + // STRUCT ================================ struct STColumn { col_id_t colId; @@ -225,23 +259,9 @@ struct STag { memcpy(varDataVal(x), (str), (_size)); \ } while (0); -// ----------------- SCHEMA BUILDER DEFINITION -typedef struct { - int32_t tCols; - int32_t nCols; - schema_ver_t version; - uint16_t flen; - int32_t tlen; - STColumn *columns; -} STSchemaBuilder; - -int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); -void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); -void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); -int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes); -STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); - +// STSchema ================================ STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version); +void tDestroyTSchema(STSchema *pTSchema); #endif diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 9e8a139b31abe31d2a382fe1c6c30282c5bf2a27..2331f0b23c52d6b826796549a1964d4eb970a9fd 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -55,7 +55,7 @@ extern int32_t tsNumOfMnodeQueryThreads; extern int32_t tsNumOfMnodeFetchThreads; extern int32_t tsNumOfMnodeReadThreads; extern int32_t tsNumOfVnodeQueryThreads; -extern int32_t tsNumOfVnodeStreamThreads; +extern float tsRatioOfVnodeStreamThreads; extern int32_t tsNumOfVnodeFetchThreads; extern int32_t tsNumOfVnodeRsmaThreads; extern int32_t tsNumOfQnodeQueryThreads; @@ -69,6 +69,9 @@ extern int32_t tsElectInterval; extern int32_t tsHeartbeatInterval; extern int32_t tsHeartbeatTimeout; +// vnode +extern int64_t tsVndCommitMaxIntervalMs; + // monitor extern bool tsEnableMonitor; extern int32_t tsMonitorInterval; @@ -82,6 +85,10 @@ extern bool tsEnableTelem; extern int32_t tsTelemInterval; extern char tsTelemServer[]; extern uint16_t tsTelemPort; +extern bool tsEnableCrashReport; +extern char* tsTelemUri; +extern char* tsClientCrashReportUri; +extern char* tsSvrCrashReportUri; // query buffer management extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ad6077db098b18d2b10d95058831d4f8c25d046a..1ee6c043ee769fba9112f165ece9ab30330c5034 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -115,6 +115,7 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_STREAMS, TSDB_MGMT_TABLE_TABLE, TSDB_MGMT_TABLE_TAG, + TSDB_MGMT_TABLE_COL, TSDB_MGMT_TABLE_USER, TSDB_MGMT_TABLE_GRANTS, TSDB_MGMT_TABLE_VGROUP, @@ -343,7 +344,8 @@ void tFreeSSubmitRsp(SSubmitRsp* pRsp); #define COL_IS_SET(FLG) (((FLG) & (COL_SET_VAL | COL_SET_NULL)) != 0) #define COL_CLR_SET(FLG) ((FLG) &= (~(COL_SET_VAL | COL_SET_NULL))) -#define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON) +#define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON) +#define IS_SET_NULL(s) (((s)->flags & COL_SET_NULL) == COL_SET_NULL) #define SSCHMEA_TYPE(s) ((s)->type) #define SSCHMEA_FLAGS(s) ((s)->flags) @@ -381,6 +383,13 @@ static FORCE_INLINE void tDeleteSSchemaWrapper(SSchemaWrapper* pSchemaWrapper) { } } +static FORCE_INLINE void tDeleteSSchemaWrapperForHash(void* pSchemaWrapper) { + if (pSchemaWrapper != NULL && *(SSchemaWrapper**)pSchemaWrapper != NULL) { + taosMemoryFree((*(SSchemaWrapper**)pSchemaWrapper)->pSchema); + taosMemoryFree(*(SSchemaWrapper**)pSchemaWrapper); + } +} + static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema) { int32_t tlen = 0; tlen += taosEncodeFixedI8(buf, pSchema->type); @@ -482,8 +491,6 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapperEx(SDecoder* pDecoder, SSchemaW return 0; } -STSchema* tdGetSTSChemaFromSSChema(SSchema* pSchema, int32_t nCols, int32_t sver); - typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igExists; @@ -1394,6 +1401,7 @@ typedef struct { char db[TSDB_DB_FNAME_LEN]; char tb[TSDB_TABLE_NAME_LEN]; char user[TSDB_USER_LEN]; + char filterTb[TSDB_TABLE_NAME_LEN]; int64_t showId; } SRetrieveTableReq; @@ -1734,6 +1742,8 @@ typedef struct { int32_t execId; } STaskDropReq; +int32_t tSerializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); +int32_t tDeserializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); int32_t tSerializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); int32_t tDeserializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); @@ -1751,6 +1761,8 @@ typedef struct { #define STREAM_FILL_HISTORY_ON 1 #define STREAM_FILL_HISTORY_OFF 0 #define STREAM_DEFAULT_FILL_HISTORY STREAM_FILL_HISTORY_OFF +#define STREAM_CREATE_STABLE_TRUE 1 +#define STREAM_CREATE_STABLE_FALSE 0 typedef struct { char name[TSDB_STREAM_FNAME_LEN]; @@ -1768,6 +1780,10 @@ typedef struct { SArray* pTags; // array of SField // 3.0.20 int64_t checkpointFreq; // ms + // 3.0.2.3 + int8_t createStb; + uint64_t targetStbUid; + SArray* fillNullCols; } SCMCreateStreamReq; typedef struct { @@ -2081,10 +2097,15 @@ typedef struct SVCreateTbReq { }; } SVCreateTbReq; -int tEncodeSVCreateTbReq(SEncoder* pCoder, const SVCreateTbReq* pReq); -int tDecodeSVCreateTbReq(SDecoder* pCoder, SVCreateTbReq* pReq); +int tEncodeSVCreateTbReq(SEncoder* pCoder, const SVCreateTbReq* pReq); +int tDecodeSVCreateTbReq(SDecoder* pCoder, SVCreateTbReq* pReq); +void tDestroySVCreateTbReq(SVCreateTbReq* pReq, int32_t flags); static FORCE_INLINE void tdDestroySVCreateTbReq(SVCreateTbReq* req) { + if (NULL == req) { + return; + } + taosMemoryFreeClear(req->name); taosMemoryFreeClear(req->comment); if (req->type == TSDB_CHILD_TABLE) { @@ -3232,6 +3253,57 @@ int32_t tSerializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); int32_t tDeserializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); int32_t tSerializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); int32_t tDeserializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); +int32_t tSerializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); +int32_t tDeserializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); +int32_t tSerializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); +int32_t tDeserializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); + +#define SUBMIT_REQ_AUTO_CREATE_TABLE 0x1 +#define SUBMIT_REQ_COLUMN_DATA_FORMAT 0x2 + +typedef struct { + int32_t flags; + SVCreateTbReq* pCreateTbReq; + int64_t suid; + int64_t uid; + int32_t sver; + union { + SArray* aRowP; + SArray* aCol; + }; +} SSubmitTbData; + +typedef struct { + SArray* aSubmitTbData; // SArray +} SSubmitReq2; + +int32_t tEncodeSSubmitReq2(SEncoder* pCoder, const SSubmitReq2* pReq); +int32_t tDecodeSSubmitReq2(SDecoder* pCoder, SSubmitReq2* pReq); +void tDestroySSubmitTbData(SSubmitTbData* pTbData, int32_t flag); +void tDestroySSubmitReq2(SSubmitReq2* pReq, int32_t flag); + +typedef struct { + int32_t affectedRows; + SArray* aCreateTbRsp; // SArray +} SSubmitRsp2; + +int32_t tEncodeSSubmitRsp2(SEncoder* pCoder, const SSubmitRsp2* pRsp); +int32_t tDecodeSSubmitRsp2(SDecoder* pCoder, SSubmitRsp2* pRsp); +void tDestroySSubmitRsp2(SSubmitRsp2* pRsp, int32_t flag); + +#define TSDB_MSG_FLG_ENCODE 0x1 +#define TSDB_MSG_FLG_DECODE 0x2 + +typedef struct { + union { + struct { + void* msgStr; + int32_t msgLen; + int64_t ver; + }; + void* pDataBlock; + }; +} SPackedData; #pragma pack(pop) diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index 32d00bb4227c8b153e4112851ba6004b5ab7c88d..7cf092b14417186d8e36f23d6d870809779a2638 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -39,6 +39,7 @@ typedef enum { QUEUE_MAX, } EQueueType; +typedef void (*UpdateDnodeInfoFp)(void* pData, int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port); typedef int32_t (*PutToQueueFp)(void* pMgmt, EQueueType qtype, SRpcMsg* pMsg); typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype); typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg); @@ -48,6 +49,7 @@ typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type); typedef void (*ReportStartup)(const char* name, const char* desc); typedef struct { + void* data; void* mgmt; void* clientRpc; PutToQueueFp putToQueueFp; @@ -57,6 +59,7 @@ typedef struct { RegisterBrokenLinkArgFp registerBrokenLinkArgFp; ReleaseHandleFp releaseHandleFp; ReportStartup reportStartupFp; + UpdateDnodeInfoFp updateDnodeInfoFp; } SMsgCb; void tmsgSetDefault(const SMsgCb* msgcb); @@ -67,6 +70,7 @@ void tmsgSendRsp(SRpcMsg* pMsg); void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg); void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type); void tmsgReportStartup(const char* name, const char* desc); +void tmsgUpdateDnodeInfo(int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port); #ifdef __cplusplus } diff --git a/include/common/tname.h b/include/common/tname.h index 666a25303ebae0d9098c50e10d4a8c4df0e7624f..6a89d2a6be398c0fd3afd13fd780c9a25631e473 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -78,7 +78,7 @@ typedef struct { // output char* ctbShortName; // must have size of TSDB_TABLE_NAME_LEN; - uint64_t uid; // child table uid, may be useful +// uint64_t uid; // child table uid, may be useful } RandTableName; void buildChildTableName(RandTableName* rName); diff --git a/include/common/trow.h b/include/common/trow.h index 6a71a8844ed54baa45836d16f2c3ccf9b114cc0a..8332c10ed278936ca77453e485d8dbb3147b4850 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -20,7 +20,6 @@ #include "talgo.h" #include "taosdef.h" #include "taoserror.h" -#include "tbuffer.h" #include "tdataformat.h" #include "tdef.h" #include "ttypes.h" diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 04970ccc349984b3d08f0a5709e5025f0ee28798..597371d9d1a50fce133f8a78c16a72d031119978 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -175,171 +175,172 @@ #define TK_CONSUMERS 157 #define TK_SUBSCRIPTIONS 158 #define TK_VNODES 159 -#define TK_LIKE 160 -#define TK_TBNAME 161 -#define TK_QTAGS 162 -#define TK_AS 163 -#define TK_INDEX 164 -#define TK_FUNCTION 165 -#define TK_INTERVAL 166 -#define TK_COUNT 167 -#define TK_LAST_ROW 168 -#define TK_TOPIC 169 -#define TK_WITH 170 -#define TK_META 171 -#define TK_CONSUMER 172 -#define TK_GROUP 173 -#define TK_DESC 174 -#define TK_DESCRIBE 175 -#define TK_RESET 176 -#define TK_QUERY 177 -#define TK_CACHE 178 -#define TK_EXPLAIN 179 -#define TK_ANALYZE 180 -#define TK_VERBOSE 181 -#define TK_NK_BOOL 182 -#define TK_RATIO 183 -#define TK_NK_FLOAT 184 -#define TK_OUTPUTTYPE 185 -#define TK_AGGREGATE 186 -#define TK_BUFSIZE 187 -#define TK_STREAM 188 -#define TK_INTO 189 -#define TK_TRIGGER 190 -#define TK_AT_ONCE 191 -#define TK_WINDOW_CLOSE 192 -#define TK_IGNORE 193 -#define TK_EXPIRED 194 -#define TK_FILL_HISTORY 195 -#define TK_SUBTABLE 196 -#define TK_KILL 197 -#define TK_CONNECTION 198 -#define TK_TRANSACTION 199 -#define TK_BALANCE 200 -#define TK_VGROUP 201 -#define TK_MERGE 202 -#define TK_REDISTRIBUTE 203 -#define TK_SPLIT 204 -#define TK_DELETE 205 -#define TK_INSERT 206 -#define TK_NULL 207 -#define TK_NK_QUESTION 208 -#define TK_NK_ARROW 209 -#define TK_ROWTS 210 -#define TK_QSTART 211 -#define TK_QEND 212 -#define TK_QDURATION 213 -#define TK_WSTART 214 -#define TK_WEND 215 -#define TK_WDURATION 216 -#define TK_IROWTS 217 -#define TK_ISFILLED 218 -#define TK_CAST 219 -#define TK_NOW 220 -#define TK_TODAY 221 -#define TK_TIMEZONE 222 -#define TK_CLIENT_VERSION 223 -#define TK_SERVER_VERSION 224 -#define TK_SERVER_STATUS 225 -#define TK_CURRENT_USER 226 -#define TK_CASE 227 -#define TK_END 228 -#define TK_WHEN 229 -#define TK_THEN 230 -#define TK_ELSE 231 -#define TK_BETWEEN 232 -#define TK_IS 233 -#define TK_NK_LT 234 -#define TK_NK_GT 235 -#define TK_NK_LE 236 -#define TK_NK_GE 237 -#define TK_NK_NE 238 -#define TK_MATCH 239 -#define TK_NMATCH 240 -#define TK_CONTAINS 241 -#define TK_IN 242 -#define TK_JOIN 243 -#define TK_INNER 244 -#define TK_SELECT 245 -#define TK_DISTINCT 246 -#define TK_WHERE 247 -#define TK_PARTITION 248 -#define TK_BY 249 -#define TK_SESSION 250 -#define TK_STATE_WINDOW 251 -#define TK_EVENT_WINDOW 252 -#define TK_START 253 -#define TK_SLIDING 254 -#define TK_FILL 255 -#define TK_VALUE 256 -#define TK_NONE 257 -#define TK_PREV 258 -#define TK_LINEAR 259 -#define TK_NEXT 260 -#define TK_HAVING 261 -#define TK_RANGE 262 -#define TK_EVERY 263 -#define TK_ORDER 264 -#define TK_SLIMIT 265 -#define TK_SOFFSET 266 -#define TK_LIMIT 267 -#define TK_OFFSET 268 -#define TK_ASC 269 -#define TK_NULLS 270 -#define TK_ABORT 271 -#define TK_AFTER 272 -#define TK_ATTACH 273 -#define TK_BEFORE 274 -#define TK_BEGIN 275 -#define TK_BITAND 276 -#define TK_BITNOT 277 -#define TK_BITOR 278 -#define TK_BLOCKS 279 -#define TK_CHANGE 280 -#define TK_COMMA 281 -#define TK_COMPACT 282 -#define TK_CONCAT 283 -#define TK_CONFLICT 284 -#define TK_COPY 285 -#define TK_DEFERRED 286 -#define TK_DELIMITERS 287 -#define TK_DETACH 288 -#define TK_DIVIDE 289 -#define TK_DOT 290 -#define TK_EACH 291 -#define TK_FAIL 292 -#define TK_FILE 293 -#define TK_FOR 294 -#define TK_GLOB 295 -#define TK_ID 296 -#define TK_IMMEDIATE 297 -#define TK_IMPORT 298 -#define TK_INITIALLY 299 -#define TK_INSTEAD 300 -#define TK_ISNULL 301 -#define TK_KEY 302 -#define TK_MODULES 303 -#define TK_NK_BITNOT 304 -#define TK_NK_SEMI 305 -#define TK_NOTNULL 306 -#define TK_OF 307 -#define TK_PLUS 308 -#define TK_PRIVILEGE 309 -#define TK_RAISE 310 -#define TK_REPLACE 311 -#define TK_RESTRICT 312 -#define TK_ROW 313 -#define TK_SEMI 314 -#define TK_STAR 315 -#define TK_STATEMENT 316 -#define TK_STRICT 317 -#define TK_STRING 318 -#define TK_TIMES 319 -#define TK_UPDATE 320 -#define TK_VALUES 321 -#define TK_VARIABLE 322 -#define TK_VIEW 323 -#define TK_WAL 324 +#define TK_ALIVE 160 +#define TK_LIKE 161 +#define TK_TBNAME 162 +#define TK_QTAGS 163 +#define TK_AS 164 +#define TK_INDEX 165 +#define TK_FUNCTION 166 +#define TK_INTERVAL 167 +#define TK_COUNT 168 +#define TK_LAST_ROW 169 +#define TK_TOPIC 170 +#define TK_WITH 171 +#define TK_META 172 +#define TK_CONSUMER 173 +#define TK_GROUP 174 +#define TK_DESC 175 +#define TK_DESCRIBE 176 +#define TK_RESET 177 +#define TK_QUERY 178 +#define TK_CACHE 179 +#define TK_EXPLAIN 180 +#define TK_ANALYZE 181 +#define TK_VERBOSE 182 +#define TK_NK_BOOL 183 +#define TK_RATIO 184 +#define TK_NK_FLOAT 185 +#define TK_OUTPUTTYPE 186 +#define TK_AGGREGATE 187 +#define TK_BUFSIZE 188 +#define TK_STREAM 189 +#define TK_INTO 190 +#define TK_TRIGGER 191 +#define TK_AT_ONCE 192 +#define TK_WINDOW_CLOSE 193 +#define TK_IGNORE 194 +#define TK_EXPIRED 195 +#define TK_FILL_HISTORY 196 +#define TK_SUBTABLE 197 +#define TK_KILL 198 +#define TK_CONNECTION 199 +#define TK_TRANSACTION 200 +#define TK_BALANCE 201 +#define TK_VGROUP 202 +#define TK_MERGE 203 +#define TK_REDISTRIBUTE 204 +#define TK_SPLIT 205 +#define TK_DELETE 206 +#define TK_INSERT 207 +#define TK_NULL 208 +#define TK_NK_QUESTION 209 +#define TK_NK_ARROW 210 +#define TK_ROWTS 211 +#define TK_QSTART 212 +#define TK_QEND 213 +#define TK_QDURATION 214 +#define TK_WSTART 215 +#define TK_WEND 216 +#define TK_WDURATION 217 +#define TK_IROWTS 218 +#define TK_ISFILLED 219 +#define TK_CAST 220 +#define TK_NOW 221 +#define TK_TODAY 222 +#define TK_TIMEZONE 223 +#define TK_CLIENT_VERSION 224 +#define TK_SERVER_VERSION 225 +#define TK_SERVER_STATUS 226 +#define TK_CURRENT_USER 227 +#define TK_CASE 228 +#define TK_END 229 +#define TK_WHEN 230 +#define TK_THEN 231 +#define TK_ELSE 232 +#define TK_BETWEEN 233 +#define TK_IS 234 +#define TK_NK_LT 235 +#define TK_NK_GT 236 +#define TK_NK_LE 237 +#define TK_NK_GE 238 +#define TK_NK_NE 239 +#define TK_MATCH 240 +#define TK_NMATCH 241 +#define TK_CONTAINS 242 +#define TK_IN 243 +#define TK_JOIN 244 +#define TK_INNER 245 +#define TK_SELECT 246 +#define TK_DISTINCT 247 +#define TK_WHERE 248 +#define TK_PARTITION 249 +#define TK_BY 250 +#define TK_SESSION 251 +#define TK_STATE_WINDOW 252 +#define TK_EVENT_WINDOW 253 +#define TK_START 254 +#define TK_SLIDING 255 +#define TK_FILL 256 +#define TK_VALUE 257 +#define TK_NONE 258 +#define TK_PREV 259 +#define TK_LINEAR 260 +#define TK_NEXT 261 +#define TK_HAVING 262 +#define TK_RANGE 263 +#define TK_EVERY 264 +#define TK_ORDER 265 +#define TK_SLIMIT 266 +#define TK_SOFFSET 267 +#define TK_LIMIT 268 +#define TK_OFFSET 269 +#define TK_ASC 270 +#define TK_NULLS 271 +#define TK_ABORT 272 +#define TK_AFTER 273 +#define TK_ATTACH 274 +#define TK_BEFORE 275 +#define TK_BEGIN 276 +#define TK_BITAND 277 +#define TK_BITNOT 278 +#define TK_BITOR 279 +#define TK_BLOCKS 280 +#define TK_CHANGE 281 +#define TK_COMMA 282 +#define TK_COMPACT 283 +#define TK_CONCAT 284 +#define TK_CONFLICT 285 +#define TK_COPY 286 +#define TK_DEFERRED 287 +#define TK_DELIMITERS 288 +#define TK_DETACH 289 +#define TK_DIVIDE 290 +#define TK_DOT 291 +#define TK_EACH 292 +#define TK_FAIL 293 +#define TK_FILE 294 +#define TK_FOR 295 +#define TK_GLOB 296 +#define TK_ID 297 +#define TK_IMMEDIATE 298 +#define TK_IMPORT 299 +#define TK_INITIALLY 300 +#define TK_INSTEAD 301 +#define TK_ISNULL 302 +#define TK_KEY 303 +#define TK_MODULES 304 +#define TK_NK_BITNOT 305 +#define TK_NK_SEMI 306 +#define TK_NOTNULL 307 +#define TK_OF 308 +#define TK_PLUS 309 +#define TK_PRIVILEGE 310 +#define TK_RAISE 311 +#define TK_REPLACE 312 +#define TK_RESTRICT 313 +#define TK_ROW 314 +#define TK_SEMI 315 +#define TK_STAR 316 +#define TK_STATEMENT 317 +#define TK_STRICT 318 +#define TK_STRING 319 +#define TK_TIMES 320 +#define TK_UPDATE 321 +#define TK_VALUES 322 +#define TK_VARIABLE 323 +#define TK_VIEW 324 +#define TK_WAL 325 #define TK_NK_SPACE 600 #define TK_NK_COMMENT 601 diff --git a/include/common/ttypes.h b/include/common/ttypes.h index d0f72fbfe59bccabe1d32474c252658fe8e85fdd..97ae151b7a8ccedb84dd113fa9c9560cfe5ffe14 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -343,8 +343,8 @@ typedef struct tDataTypeDescriptor { extern tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX]; bool isValidDataType(int32_t type); +int32_t operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type); void assignVal(char *val, const char *src, int32_t len, int32_t type); -void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type); void *getDataMin(int32_t type, void* value); void *getDataMax(int32_t type, void* value); diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 0e2d0431743385a280627160fa85521420148d73..fbb24d28623d99932966e625d6499ebbc836ac0a 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -210,6 +210,9 @@ int32_t catalogGetCachedTableMeta(SCatalog* pCtg, const SName* pTableName, STabl int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta); +int32_t catalogGetTablesHashVgId(SCatalog* pCtg, SRequestConnInfo* pConn, int32_t acctId, const char* pDb, const char* pTableName[], + int32_t tableNum, int32_t *vgId); + int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists); int32_t catalogGetCachedTableVgMeta(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, STableMeta** pTableMeta); diff --git a/include/libs/command/command.h b/include/libs/command/command.h index b3339a417ba463212c3abc163b57519194953c10..a8b1a0902acac276080d905554a3773f68e66b1c 100644 --- a/include/libs/command/command.h +++ b/include/libs/command/command.h @@ -22,7 +22,7 @@ typedef struct SExplainCtx SExplainCtx; -int32_t qExecCommand(bool sysInfoUser, SNode *pStmt, SRetrieveTableRsp **pRsp); +int32_t qExecCommand(int64_t* pConnId, bool sysInfoUser, SNode *pStmt, SRetrieveTableRsp **pRsp); int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp); int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int64_t startTs); diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index cfd5bd1ed77bc1a0c4f158f00cb8c2462ac70443..095d2f6d10dbe6faa0460f5532c569b543fb4ac0 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -154,6 +154,8 @@ void qCleanExecTaskBlockBuf(qTaskInfo_t tinfo); */ int32_t qAsyncKillTask(qTaskInfo_t tinfo, int32_t rspCode); +bool qTaskIsExecuting(qTaskInfo_t qinfo); + /** * destroy query info structure * @param qHandle @@ -190,7 +192,9 @@ int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts); int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType); -int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq); +// int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq, int64_t ver); +// +int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit); int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset); @@ -214,6 +218,7 @@ int32_t qStreamSourceRecoverStep2(qTaskInfo_t tinfo, int64_t ver); int32_t qStreamRecoverFinish(qTaskInfo_t tinfo); int32_t qStreamRestoreParam(qTaskInfo_t tinfo); bool qStreamRecoverScanFinished(qTaskInfo_t tinfo); +void qStreamCloseTsdbReader(void* task); #ifdef __cplusplus } diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 32b8cc7389b96e7479697b2c864e98543d92e9a1..32db6773e07ce61ad1ef100713b3bb696a36f09d 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -20,7 +20,6 @@ extern "C" { #endif -#include "tbuffer.h" #include "tcommon.h" #include "tvariant.h" @@ -138,7 +137,7 @@ typedef struct SqlFunctionCtx { char *pOutput; // final result output buffer, point to sdata->data int32_t numOfParams; // input parameter, e.g., top(k, 20), the number of results of top query is kept in param - SFunctParam *param; + SFunctParam *param; // corresponding output buffer for timestamp of each result, e.g., diff/csum SColumnInfoData *pTsOutput; int32_t offset; @@ -152,6 +151,7 @@ typedef struct SqlFunctionCtx { struct SSDataBlock *pSrcBlock; struct SSDataBlock *pDstBlock; // used by indefinite rows function to set selectivity SSerializeDataHandle saveHandle; + int32_t exprIdx; char udfName[TSDB_FUNC_NAME_LEN]; } SqlFunctionCtx; @@ -182,9 +182,9 @@ struct SScalarParam { int32_t numOfQualified; // number of qualified elements in the final results }; -void cleanupResultRowEntry(struct SResultRowEntryInfo *pCell); -bool isRowEntryCompleted(struct SResultRowEntryInfo *pEntry); -bool isRowEntryInitialized(struct SResultRowEntryInfo *pEntry); +void cleanupResultRowEntry(struct SResultRowEntryInfo *pCell); +bool isRowEntryCompleted(struct SResultRowEntryInfo *pEntry); +bool isRowEntryInitialized(struct SResultRowEntryInfo *pEntry); typedef struct SPoint { int64_t key; diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 66988ff135a59918e200d5138f6f5142351227b6..d77f2b62332ee071747676145b5dcba0b4e48371 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -40,6 +40,7 @@ extern "C" { #define SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE) #define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) +#define SHOW_ALIVE_RESULT_COLS 1 #define PRIVILEGE_TYPE_MASK(n) (1 << n) #define PRIVILEGE_TYPE_ALL PRIVILEGE_TYPE_MASK(0) @@ -262,6 +263,11 @@ typedef struct SShowCreateDatabaseStmt { void* pCfg; // SDbCfgInfo } SShowCreateDatabaseStmt; +typedef struct SShowAliveStmt { + ENodeType type; + char dbName[TSDB_DB_NAME_LEN]; +} SShowAliveStmt; + typedef struct SShowCreateTableStmt { ENodeType type; char dbName[TSDB_DB_NAME_LEN]; @@ -295,7 +301,7 @@ typedef struct SShowTableTagsStmt { SNodeList* pTags; } SShowTableTagsStmt; -typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT } EIndexType; +typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT, INDEX_TYPE_NORMAL } EIndexType; typedef struct SIndexOptions { ENodeType type; @@ -401,6 +407,7 @@ typedef struct SCreateStreamStmt { SNode* pQuery; SNodeList* pTags; SNode* pSubtable; + SNodeList* pCols; } SCreateStreamStmt; typedef struct SDropStreamStmt { diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index e111f36077575adc381c59ba1de62ccbab07af24..24b436ec994f903cd09c4c6bd918c813b192254d 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -208,6 +208,8 @@ typedef enum ENodeType { QUERY_NODE_DELETE_STMT, QUERY_NODE_INSERT_STMT, QUERY_NODE_QUERY, + QUERY_NODE_SHOW_DB_ALIVE_STMT, + QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT, // logic plan node QUERY_NODE_LOGIC_PLAN_SCAN = 1000, diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 6f700c75edf82a9eddb23ef82f0dade1c89680b2..3602225789f5d2b5586e592646763e8f651f9dcf 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -121,6 +121,7 @@ typedef struct SAggLogicNode { bool hasLast; bool hasTimeLineFunc; bool onlyHasKeepOrderFunc; + bool hasGroupKeyOptimized; } SAggLogicNode; typedef struct SProjectLogicNode { @@ -409,6 +410,7 @@ typedef struct SAggPhysiNode { SNodeList* pGroupKeys; SNodeList* pAggFuncs; bool mergeDataBlock; + bool groupKeyOptimized; } SAggPhysiNode; typedef struct SDownstreamSourceNode { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index f7220be753d110b94d555b06c920a8a94968a75c..2d143912474cb9ef769d2db915f24f110d37b1a2 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -361,33 +361,33 @@ typedef struct SVgDataBlocks { void* pData; // SSubmitReq + SSubmitBlk + ... } SVgDataBlocks; -typedef void (*FFreeDataBlockHash)(SHashObj*); -typedef void (*FFreeDataBlockArray)(SArray*); +typedef void (*FFreeTableBlockHash)(SHashObj*); +typedef void (*FFreeVgourpBlockArray)(SArray*); typedef struct SVnodeModifyOpStmt { - ENodeType nodeType; - ENodeType sqlNodeType; - SArray* pDataBlocks; // data block for each vgroup, SArray. - uint32_t insertType; // insert data from [file|sql statement| bound statement] - const char* pSql; // current sql statement position - int32_t totalRowsNum; - int32_t totalTbNum; - SName targetTableName; - SName usingTableName; - const char* pBoundCols; - struct STableMeta* pTableMeta; - SHashObj* pVgroupsHashObj; - SHashObj* pTableBlockHashObj; - SHashObj* pSubTableHashObj; - SHashObj* pTableNameHashObj; - SHashObj* pDbFNameHashObj; - SArray* pVgDataBlocks; - SVCreateTbReq createTblReq; - TdFilePtr fp; - FFreeDataBlockHash freeHashFunc; - FFreeDataBlockArray freeArrayFunc; - bool usingTableProcessing; - bool fileProcessing; + ENodeType nodeType; + ENodeType sqlNodeType; + SArray* pDataBlocks; // data block for each vgroup, SArray. + uint32_t insertType; // insert data from [file|sql statement| bound statement] + const char* pSql; // current sql statement position + int32_t totalRowsNum; + int32_t totalTbNum; + SName targetTableName; + SName usingTableName; + const char* pBoundCols; + struct STableMeta* pTableMeta; + SHashObj* pVgroupsHashObj; + SHashObj* pTableBlockHashObj; // SHashObj + SHashObj* pSubTableHashObj; + SHashObj* pTableNameHashObj; + SHashObj* pDbFNameHashObj; + SArray* pVgDataBlocks; // SArray + SVCreateTbReq* pCreateTblReq; + TdFilePtr fp; + FFreeTableBlockHash freeHashFunc; + FFreeVgourpBlockArray freeArrayFunc; + bool usingTableProcessing; + bool fileProcessing; } SVnodeModifyOpStmt; typedef struct SExplainOptions { diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 9be79a539f468d20ad28d2278651fbae169ff4a6..8f227459735f8c43f74b760b16b4b133c883c95b 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -58,7 +58,6 @@ typedef struct SParseContext { bool isSuperUser; bool enableSysInfo; bool async; - int8_t schemalessType; const char* svrVer; bool nodeOffline; SArray* pTableMetaPos; // sql table pos => catalog data pos @@ -85,12 +84,12 @@ int32_t qSetSTableIdForRsma(SNode* pStmt, int64_t uid); void qCleanupKeywordsTable(); int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash); -int32_t qResetStmtDataBlock(void* block, bool keepBuf); -int32_t qCloneStmtDataBlock(void** pDst, void* pSrc); -void qFreeStmtDataBlock(void* pDataBlock); -int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgId); -void qDestroyStmtDataBlock(void* pBlock); -STableMeta* qGetTableMetaInDataBlock(void* pDataBlock); +int32_t qResetStmtDataBlock(STableDataCxt* block, bool keepBuf); +int32_t qCloneStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, bool reset); +int32_t qRebuildStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_t uid, uint64_t suid, int32_t vgId, bool rebuildCreateTb); +void qDestroyStmtDataBlock(STableDataCxt* pBlock); +STableMeta* qGetTableMetaInDataBlock(STableDataCxt* pDataBlock); +int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData **pData); int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx); int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery); @@ -105,11 +104,18 @@ void destroyBoundColumnInfo(void* pBoundInfo); int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf, int32_t msgBufLen); -void* smlInitHandle(SQuery* pQuery); -void smlDestroyHandle(void* pHandle); -int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta, +void qDestroyBoundColInfo(void* pInfo); + +SQuery* smlInitHandle(); +int32_t smlBuildRow(STableDataCxt* pTableCxt); +int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void *kv, int32_t index); +STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta); + +int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int16_t msgBufLen); -int32_t smlBuildOutput(void* handle, SHashObj* pVgHash); +int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash); + +int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD *fields, int numFields); int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray); SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap); diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 3f11d2a21832ab5b5bab3c0edc8c0fd5ae085a86..57ddeb657cb6c583e8a97dc089b735d903f2347e 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -163,6 +163,23 @@ typedef struct STargetInfo { int32_t vgId; } STargetInfo; +typedef struct SBoundColInfo { + int16_t* pColIndex; // bound index => schema index + int32_t numOfCols; + int32_t numOfBound; +} SBoundColInfo; + +typedef struct STableDataCxt { + STableMeta* pMeta; + STSchema* pSchema; + SBoundColInfo boundColsInfo; + SArray* pValues; + SSubmitTbData* pData; + TSKEY lastTs; + bool ordered; + bool duplicateTs; +} STableDataCxt; + typedef int32_t (*__async_send_cb_fn_t)(void* param, SDataBuf* pMsg, int32_t code); typedef int32_t (*__async_exec_fn_t)(void* param); @@ -190,6 +207,12 @@ typedef struct SQueryNodeStat { int32_t tableNum; // vg table number, unit is TSDB_TABLE_NUM_UNIT } SQueryNodeStat; +typedef struct SColLocation { + int16_t slotId; + col_id_t colId; + int8_t type; +} SColLocation; + int32_t initTaskQueue(); int32_t cleanupTaskQueue(); @@ -238,6 +261,7 @@ int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t char* parseTagDatatoJson(void* p); int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst); +int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst); void freeVgInfo(SDBVgInfo* vgInfo); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen, @@ -268,7 +292,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_RESTORING || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR) #define SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_MNODE_NOT_FOUND) -#define NO_RET_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) +#define NO_RET_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || (_code) == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED) #define NEED_REDIRECT_ERROR(_code) \ (NO_RET_REDIRECT_ERROR(_code) || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index c00625c51c86dc96f414cb0f719970f9c1b99ead..0f57da2b804c7c58411ee7ffa2cbb1fc4d68b3e8 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -103,6 +103,7 @@ typedef struct { int8_t type; } SStreamQueueItem; +#if 0 typedef struct { int8_t type; int64_t ver; @@ -116,6 +117,21 @@ typedef struct { SArray* dataRefs; // SArray SArray* reqs; // SArray } SStreamMergedSubmit; +#endif + +typedef struct { + int8_t type; + int64_t ver; + int32_t* dataRef; + SPackedData submit; +} SStreamDataSubmit2; + +typedef struct { + int8_t type; + int64_t ver; + SArray* dataRefs; // SArray + SArray* submits; // SArray +} SStreamMergedSubmit2; typedef struct { int8_t type; @@ -219,11 +235,11 @@ static FORCE_INLINE void* streamQueueNextItem(SStreamQueue* queue) { } } -SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq); +SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit); -void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit); +void streamDataSubmitRefDec(SStreamDataSubmit2* pDataSubmit); -SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit); +SStreamDataSubmit2* streamSubmitRefClone(SStreamDataSubmit2* pSubmit); typedef struct { char* qmsg; @@ -355,14 +371,15 @@ void tFreeSStreamTask(SStreamTask* pTask); static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem* pItem) { if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit*)pItem); + SStreamDataSubmit2* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit2*)pItem); if (pSubmitClone == NULL) { qDebug("task %d %p submit enqueue failed since out of memory", pTask->taskId, pTask); terrno = TSDB_CODE_OUT_OF_MEMORY; atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); return -1; } - qDebug("task %d %p submit enqueue %p %p %p", pTask->taskId, pTask, pItem, pSubmitClone, pSubmitClone->data); + qDebug("task %d %p submit enqueue %p %p %p %d %" PRId64, pTask->taskId, pTask, pItem, pSubmitClone, + pSubmitClone->submit.msgStr, pSubmitClone->submit.msgLen, pSubmitClone->submit.ver); taosWriteQitem(pTask->inputQueue->queue, pSubmitClone); // qStreamInput(pTask->exec.executor, pSubmitClone); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE || diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 559dc1009dcdc5795df28b24d529ae90038900f3..4c23c1f5575434cc971a49a042df4ce1720b73ef 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -49,10 +49,12 @@ extern "C" { #define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500 #define SYNC_SNAP_RESEND_MS 1000 * 60 +#define SYNC_VND_COMMIT_MIN_MS 1000 + #define SYNC_MAX_BATCH_SIZE 1 #define SYNC_INDEX_BEGIN 0 #define SYNC_INDEX_INVALID -1 -#define SYNC_TERM_INVALID -1 // 0xFFFFFFFFFFFFFFFF +#define SYNC_TERM_INVALID -1 typedef enum { SYNC_STRATEGY_NO_SNAPSHOT = 0, @@ -78,6 +80,8 @@ typedef enum { } ESyncState; typedef struct SNodeInfo { + int64_t clusterId; + int32_t nodeId; uint16_t nodePort; char nodeFqdn[TSDB_FQDN_LEN]; } SNodeInfo; @@ -230,6 +234,7 @@ int64_t syncOpen(SSyncInfo* pSyncInfo); int32_t syncStart(int64_t rid); void syncStop(int64_t rid); void syncPreStop(int64_t rid); +void syncPostStop(int64_t rid); int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq); int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg); int32_t syncReconfig(int64_t rid, SSyncCfg* pCfg); diff --git a/include/libs/transport/thttp.h b/include/libs/transport/thttp.h index 7d8c588bfc13787377a0512a73c5afdc5ad68700..9a6aee418794b61abd88e132b42964c56c69451c 100644 --- a/include/libs/transport/thttp.h +++ b/include/libs/transport/thttp.h @@ -24,7 +24,7 @@ extern "C" { typedef enum { HTTP_GZIP, HTTP_FLAT } EHttpCompFlag; -int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag); +int32_t taosSendHttpReport(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag); #ifdef __cplusplus } diff --git a/include/os/os.h b/include/os/os.h index b27fa84406b843f15ba99d52f7d9fde8580a0899..809b814491e99724561af2356fe1128cfc86dc16 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -116,6 +116,7 @@ extern "C" { #include "osTimer.h" #include "osTimezone.h" #include "taoserror.h" +#include "tlog.h" #ifdef __cplusplus } diff --git a/include/os/osMemory.h b/include/os/osMemory.h index 4681ff66741d4781c88dba5ccc950094c96f6769..e6a4488287816f21ca9eeee76a3208ff6e6ff12d 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -22,12 +22,16 @@ extern "C" { // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following sectio +#if !defined(WINDOWS) + #ifndef ALLOW_FORBID_FUNC #define malloc MALLOC_FUNC_TAOS_FORBID #define calloc CALLOC_FUNC_TAOS_FORBID #define realloc REALLOC_FUNC_TAOS_FORBID #define free FREE_FUNC_TAOS_FORBID -#endif +#endif // ifndef ALLOW_FORBID_FUNC + +#endif // if !defined(WINDOWS) void *taosMemoryMalloc(int64_t size); void *taosMemoryCalloc(int64_t num, int64_t size); diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 7765a60f8833206a5b86cd38100e1d7fd884d8dd..dbe4d6801e42fa04a9b9da0f2d3f864d0b75f94d 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -70,6 +70,7 @@ typedef struct { SysNameInfo taosGetSysNameInfo(); bool taosCheckCurrentInDll(); +int taosGetlocalhostname(char *hostname, size_t maxLen); #ifdef __cplusplus } diff --git a/include/os/osSystem.h b/include/os/osSystem.h index 8f1f5c58d5200d22583dcc7cc8c13663ce070941..5154c56e4b5b48fb1a85e2dd620b3b3338902478 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -46,27 +46,100 @@ void taosSetTerminalMode(); int32_t taosGetOldTerminalMode(); void taosResetTerminalMode(); +#define STACKSIZE 100 + #if !defined(WINDOWS) -#define taosPrintTrace(flags, level, dflag) \ - { \ - void* array[100]; \ - int32_t size = backtrace(array, 100); \ - char** strings = backtrace_symbols(array, size); \ - if (strings != NULL) { \ - taosPrintLog(flags, level, dflag, "obtained %d stack frames", size); \ - for (int32_t i = 0; i < size; i++) { \ - taosPrintLog(flags, level, dflag, "frame:%d, %s", i, strings[i]); \ - } \ - } \ - \ - taosMemoryFree(strings); \ +#define taosLogTraceToBuf(buf, bufSize, ignoreNum) { \ + void* array[STACKSIZE]; \ + int32_t size = backtrace(array, STACKSIZE); \ + char** strings = backtrace_symbols(array, size); \ + int32_t offset = 0; \ + if (strings != NULL) { \ + offset = snprintf(buf, bufSize - 1, "obtained %d stack frames\n", (ignoreNum > 0) ? size - ignoreNum : size); \ + for (int32_t i = (ignoreNum > 0) ? ignoreNum : 0; i < size; i++) { \ + offset += snprintf(buf + offset, bufSize - 1 - offset, "frame:%d, %s\n", (ignoreNum > 0) ? i - ignoreNum : i, strings[i]); \ + } \ + } \ + \ + taosMemoryFree(strings); \ +} + +#define taosPrintTrace(flags, level, dflag, ignoreNum) \ + { \ + void* array[STACKSIZE]; \ + int32_t size = backtrace(array, STACKSIZE); \ + char** strings = backtrace_symbols(array, size); \ + if (strings != NULL) { \ + taosPrintLog(flags, level, dflag, "obtained %d stack frames", (ignoreNum > 0) ? size - ignoreNum : size); \ + for (int32_t i = (ignoreNum > 0) ? ignoreNum : 0; i < size; i++) { \ + taosPrintLog(flags, level, dflag, "frame:%d, %s", (ignoreNum > 0) ? i - ignoreNum : i, strings[i]); \ + } \ + } \ + \ + taosMemoryFree(strings); \ } #else -#define taosPrintTrace(flags, level, dflag) \ + +#include +#include + +#define taosLogTraceToBuf(buf, bufSize, ignoreNum) { \ + unsigned int i; \ + void* stack[STACKSIZE]; \ + unsigned short frames; \ + SYMBOL_INFO* symbol; \ + HANDLE process; \ + int32_t offset = 0; \ + \ + process = GetCurrentProcess(); \ + \ + SymInitialize(process, NULL, TRUE); \ + \ + frames = CaptureStackBackTrace(0, STACKSIZE, stack, NULL); \ + symbol = (SYMBOL_INFO*)calloc(sizeof(SYMBOL_INFO) + 256 * sizeof(char), 1); \ + if (symbol != NULL) { \ + symbol->MaxNameLen = 255; \ + symbol->SizeOfStruct = sizeof(SYMBOL_INFO); \ + \ + if (frames > 0) { \ + offset = snprintf(buf, bufSize - 1, "obtained %d stack frames\n", (ignoreNum > 0) ? frames - ignoreNum : frames); \ + for (i = (ignoreNum > 0) ? ignoreNum : 0; i < frames; i++) { \ + SymFromAddr(process, (DWORD64)(stack[i]), 0, symbol); \ + offset += snprintf(buf + offset, bufSize - 1 - offset, "frame:%i, %s - 0x%0X\n", (ignoreNum > 0) ? i - ignoreNum : i, symbol->Name, symbol->Address); \ + } \ + } \ + free(symbol); \ + } \ + } + +#define taosPrintTrace(flags, level, dflag, ignoreNum) \ { \ - taosPrintLog(flags, level, dflag, \ - "backtrace not implemented on windows, so detailed stack information cannot be printed"); \ - } + unsigned int i; \ + void* stack[STACKSIZE]; \ + unsigned short frames; \ + SYMBOL_INFO* symbol; \ + HANDLE process; \ + \ + process = GetCurrentProcess(); \ + \ + SymInitialize(process, NULL, TRUE); \ + \ + frames = CaptureStackBackTrace(0, STACKSIZE, stack, NULL); \ + symbol = (SYMBOL_INFO*)calloc(sizeof(SYMBOL_INFO) + 256 * sizeof(char), 1); \ + if (symbol != NULL) { \ + symbol->MaxNameLen = 255; \ + symbol->SizeOfStruct = sizeof(SYMBOL_INFO); \ + \ + if (frames > 0) { \ + taosPrintLog(flags, level, dflag, "obtained %d stack frames\n", (ignoreNum > 0) ? frames - ignoreNum : frames); \ + for (i = (ignoreNum > 0) ? ignoreNum : 0; i < frames; i++) { \ + SymFromAddr(process, (DWORD64)(stack[i]), 0, symbol); \ + taosPrintLog(flags, level, dflag, "frame:%i, %s - 0x%0X\n", (ignoreNum > 0) ? i - ignoreNum : i, symbol->Name, symbol->Address); \ + } \ + } \ + free(symbol); \ + } \ + } #endif #ifdef __cplusplus diff --git a/include/util/tRealloc.h b/include/util/tRealloc.h index f3593d5818cd626c7268ec1b721bd961fba3a845..3229c53039f6dc857dd7e0b2d4774e6875ef8c16 100644 --- a/include/util/tRealloc.h +++ b/include/util/tRealloc.h @@ -52,11 +52,13 @@ _exit: return code; } -static FORCE_INLINE void tFree(uint8_t *pBuf) { - if (pBuf) { - taosMemoryFree(pBuf - sizeof(int64_t)); - } -} +#define tFree(BUF) \ + do { \ + if (BUF) { \ + taosMemoryFree((uint8_t *)(BUF) - sizeof(int64_t)); \ + (BUF) = NULL; \ + } \ + } while (0) #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index e3f5d9e6bf302adbb390d9f74b8307484111705e..52d8a75ee06b10de97ebd488d3a9a2ee439bacea 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -65,6 +65,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RPC_PORT_EADDRINUSE TAOS_DEF_ERROR_CODE(0, 0x0017) // #define TSDB_CODE_RPC_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0018) // #define TSDB_CODE_RPC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x0019) // +#define TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED TAOS_DEF_ERROR_CODE(0, 0x0020) // "Vgroup could not be connected" +#define TSDB_CODE_RPC_SOMENODE_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0021) // //common & util #define TSDB_CODE_OPS_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x0100) // @@ -156,6 +158,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_QUERY_KILLED TAOS_DEF_ERROR_CODE(0, 0X022D) #define TSDB_CODE_TSC_NO_EXEC_NODE TAOS_DEF_ERROR_CODE(0, 0X022E) #define TSDB_CODE_TSC_NOT_STABLE_ERROR TAOS_DEF_ERROR_CODE(0, 0X022F) +#define TSDB_CODE_TSC_STMT_CACHE_ERROR TAOS_DEF_ERROR_CODE(0, 0X0230) +#define TSDB_CODE_TSC_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0X0231) // mnode-common // #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) // 2.x @@ -343,6 +347,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TOPIC_SUBSCRIBED TAOS_DEF_ERROR_CODE(0, 0x03EB) #define TSDB_CODE_MND_CGROUP_USED TAOS_DEF_ERROR_CODE(0, 0x03EC) #define TSDB_CODE_MND_TOPIC_MUST_BE_DELETED TAOS_DEF_ERROR_CODE(0, 0x03ED) +#define TSDB_CODE_MND_INVALID_SUB_OPTION TAOS_DEF_ERROR_CODE(0, 0x03EE) #define TSDB_CODE_MND_IN_REBALANCE TAOS_DEF_ERROR_CODE(0, 0x03EF) // mnode-stream @@ -410,6 +415,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_NO_AVAIL_BUFPOOL TAOS_DEF_ERROR_CODE(0, 0x0528) #define TSDB_CODE_VND_STOPPED TAOS_DEF_ERROR_CODE(0, 0x0529) #define TSDB_CODE_VND_DUP_REQUEST TAOS_DEF_ERROR_CODE(0, 0x0530) +#define TSDB_CODE_VND_QUERY_BUSY TAOS_DEF_ERROR_CODE(0, 0x0531) // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) @@ -517,6 +523,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SYN_STANDBY_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0912) #define TSDB_CODE_SYN_BATCH_ERROR TAOS_DEF_ERROR_CODE(0, 0x0913) #define TSDB_CODE_SYN_RESTORING TAOS_DEF_ERROR_CODE(0, 0x0914) +#define TSDB_CODE_SYN_INVALID_SNAPSHOT_MSG TAOS_DEF_ERROR_CODE(0, 0x0915) // internal +#define TSDB_CODE_SYN_BUFFER_FULL TAOS_DEF_ERROR_CODE(0, 0x0916) // #define TSDB_CODE_SYN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x09FF) // tq @@ -694,6 +702,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SML_INVALID_DATA TAOS_DEF_ERROR_CODE(0, 0x3002) #define TSDB_CODE_SML_INVALID_DB_CONF TAOS_DEF_ERROR_CODE(0, 0x3003) #define TSDB_CODE_SML_NOT_SAME_TYPE TAOS_DEF_ERROR_CODE(0, 0x3004) +#define TSDB_CODE_SML_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x3005) //tsma #define TSDB_CODE_TSMA_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x3100) diff --git a/include/util/tarray.h b/include/util/tarray.h index 0e78397ecb7c35d75211e06170ea5cdb99030e34..f8e872ec66aa2caf38bf984d8b55ef4928526016 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -22,19 +22,6 @@ extern "C" { #endif -#if 0 -#define TARRAY(TYPE) \ - struct { \ - int32_t tarray_size_; \ - int32_t tarray_neles_; \ - struct TYPE* td_array_data_; \ - } - -#define TARRAY_SIZE(ARRAY) (ARRAY)->tarray_size_ -#define TARRAY_NELES(ARRAY) (ARRAY)->tarray_neles_ -#define TARRAY_ELE_AT(ARRAY, IDX) ((ARRAY)->td_array_data_ + idx) -#endif - #define TARRAY_MIN_SIZE 8 #define TARRAY_GET_ELEM(array, index) ((void*)((char*)((array)->pData) + (index) * (array)->elemSize)) #define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize) @@ -46,6 +33,9 @@ typedef struct SArray { void* pData; } SArray; +#define TARRAY_SIZE(array) ((array)->size) +#define TARRAY_DATA(array) ((array)->pData) + /** * * @param size @@ -194,6 +184,13 @@ void taosArrayPopTailBatch(SArray* pArray, size_t cnt); */ void taosArrayRemove(SArray* pArray, size_t index); +/** + * remove batch entry from the given index + * @param pArray + * @param index + */ +void taosArrayRemoveBatch(SArray* pArray, size_t index, size_t num, FDelete fp); + /** * copy the whole array from source to destination * @param pDst diff --git a/include/util/tbuffer.h b/include/util/tbuffer.h deleted file mode 100644 index 2ed1b7326bdb27ffe182b30d67a9369680898d23..0000000000000000000000000000000000000000 --- a/include/util/tbuffer.h +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (c) 2020 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_UTIL_BUFFER_H_ -#define _TD_UTIL_BUFFER_H_ - -#include "os.h" - -#ifdef __cplusplus -extern "C" { -#endif - -//////////////////////////////////////////////////////////////////////////////// -// usage example -/* -#include -#include "texception.h" - -int32_t main( int32_t argc, char** argv ) { - SBufferWriter bw = tbufInitWriter( NULL, false ); - - TRY( 1 ) { - //--------------------- write ------------------------ - // reserve 1024 bytes for the buffer to improve performance - tbufEnsureCapacity( &bw, 1024 ); - - // reserve space for the interger count - size_t pos = tbufReserve( &bw, sizeof(int32_t) ); - // write 5 integers to the buffer - for( int32_t i = 0; i < 5; i++) { - tbufWriteInt32( &bw, i ); - } - // write the integer count to buffer at reserved position - tbufWriteInt32At( &bw, pos, 5 ); - - // write a string to the buffer - tbufWriteString( &bw, "this is a string.\n" ); - // acquire the result and close the write buffer - size_t size = tbufTell( &bw ); - char* data = tbufGetData( &bw, false ); - - //------------------------ read ----------------------- - SBufferReader br = tbufInitReader( data, size, false ); - // read & print out all integers - int32_t count = tbufReadInt32( &br ); - for( int32_t i = 0; i < count; i++ ) { - printf( "%d\n", tbufReadInt32(&br) ); - } - // read & print out a string - puts( tbufReadString(&br, NULL) ); - // try read another integer, this result in an error as there no this integer - tbufReadInt32( &br ); - printf( "you should not see this message.\n" ); - } CATCH( code ) { - printf( "exception code is: %d, you will see this message after print out 5 integers and a string.\n", code ); - } END_TRY - - tbufCloseWriter( &bw ); - return 0; -} -*/ - -typedef struct SBufferReader { - bool endian; - const char* data; - size_t pos; - size_t size; -} SBufferReader; - -typedef struct SBufferWriter { - bool endian; - char* data; - size_t pos; - size_t size; - void* (*allocator)(void*, size_t); -} SBufferWriter; - -// common functions & macros for both reader & writer - -#define tbufTell(buf) ((buf)->pos) - -/* ------------------------ BUFFER WRITER FUNCTIONS AND MACROS ------------------------ */ -// *Allocator*, function to allocate memory, will use 'realloc' if NULL -// *Endian*, if true, writer functions of primitive types will do 'hton' automatically -#define tbufInitWriter(Allocator, Endian) \ - { .endian = (Endian), .data = NULL, .pos = 0, .size = 0, .allocator = ((Allocator) == NULL ? realloc : (Allocator)) } - -void tbufCloseWriter(SBufferWriter* buf); -void tbufEnsureCapacity(SBufferWriter* buf, size_t size); -size_t tbufReserve(SBufferWriter* buf, size_t size); -char* tbufGetData(SBufferWriter* buf, bool takeOver); -void tbufWrite(SBufferWriter* buf, const void* data, size_t size); -void tbufWriteAt(SBufferWriter* buf, size_t pos, const void* data, size_t size); -void tbufWriteStringLen(SBufferWriter* buf, const char* str, size_t len); -void tbufWriteString(SBufferWriter* buf, const char* str); -// the prototype of tbufWriteBinary and tbufWrite are identical -// the difference is: tbufWriteBinary writes the length of the data to the buffer -// first, then the actual data, which means the reader don't need to know data -// size before read. Write only write the data itself, which means the reader -// need to know data size before read. -void tbufWriteBinary(SBufferWriter* buf, const void* data, size_t len); -void tbufWriteBool(SBufferWriter* buf, bool data); -void tbufWriteBoolAt(SBufferWriter* buf, size_t pos, bool data); -void tbufWriteChar(SBufferWriter* buf, char data); -void tbufWriteCharAt(SBufferWriter* buf, size_t pos, char data); -void tbufWriteInt8(SBufferWriter* buf, int8_t data); -void tbufWriteInt8At(SBufferWriter* buf, size_t pos, int8_t data); -void tbufWriteUint8(SBufferWriter* buf, uint8_t data); -void tbufWriteUint8At(SBufferWriter* buf, size_t pos, uint8_t data); -void tbufWriteInt16(SBufferWriter* buf, int16_t data); -void tbufWriteInt16At(SBufferWriter* buf, size_t pos, int16_t data); -void tbufWriteUint16(SBufferWriter* buf, uint16_t data); -void tbufWriteUint16At(SBufferWriter* buf, size_t pos, uint16_t data); -void tbufWriteInt32(SBufferWriter* buf, int32_t data); -void tbufWriteInt32At(SBufferWriter* buf, size_t pos, int32_t data); -void tbufWriteUint32(SBufferWriter* buf, uint32_t data); -void tbufWriteUint32At(SBufferWriter* buf, size_t pos, uint32_t data); -void tbufWriteInt64(SBufferWriter* buf, int64_t data); -void tbufWriteInt64At(SBufferWriter* buf, size_t pos, int64_t data); -void tbufWriteUint64(SBufferWriter* buf, uint64_t data); -void tbufWriteUint64At(SBufferWriter* buf, size_t pos, uint64_t data); -void tbufWriteFloat(SBufferWriter* buf, float data); -void tbufWriteFloatAt(SBufferWriter* buf, size_t pos, float data); -void tbufWriteDouble(SBufferWriter* buf, double data); -void tbufWriteDoubleAt(SBufferWriter* buf, size_t pos, double data); - -/* ------------------------ BUFFER READER FUNCTIONS AND MACROS ------------------------ */ -// *Endian*, if true, reader functions of primitive types will do 'ntoh' automatically -#define tbufInitReader(Data, Size, Endian) \ - { .endian = (Endian), .data = (Data), .pos = 0, .size = ((Data) == NULL ? 0 : (Size)) } - -size_t tbufSkip(SBufferReader* buf, size_t size); -const char* tbufRead(SBufferReader* buf, size_t size); -void tbufReadToBuffer(SBufferReader* buf, void* dst, size_t size); -const char* tbufReadString(SBufferReader* buf, size_t* len); -size_t tbufReadToString(SBufferReader* buf, char* dst, size_t size); -const char* tbufReadBinary(SBufferReader* buf, size_t* len); -size_t tbufReadToBinary(SBufferReader* buf, void* dst, size_t size); -bool tbufReadBool(SBufferReader* buf); -char tbufReadChar(SBufferReader* buf); -int8_t tbufReadInt8(SBufferReader* buf); -uint8_t tbufReadUint8(SBufferReader* buf); -int16_t tbufReadInt16(SBufferReader* buf); -uint16_t tbufReadUint16(SBufferReader* buf); -int32_t tbufReadInt32(SBufferReader* buf); -uint32_t tbufReadUint32(SBufferReader* buf); -int64_t tbufReadInt64(SBufferReader* buf); -uint64_t tbufReadUint64(SBufferReader* buf); -float tbufReadFloat(SBufferReader* buf); -double tbufReadDouble(SBufferReader* buf); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_BUFFER_H_*/ diff --git a/include/util/tcompare.h b/include/util/tcompare.h index c7a3ca20f222c7d919460b31e9f3c55a79325f46..f92e1c3970a828fdfe109ee51a8e1f52f1ae0389 100644 --- a/include/util/tcompare.h +++ b/include/util/tcompare.h @@ -36,17 +36,18 @@ extern "C" { #define FLT_GREATEREQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) > (_y))) #define FLT_LESSEQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) < (_y))) -#define PATTERN_COMPARE_INFO_INITIALIZER \ - { '%', '_' } +#define PATTERN_COMPARE_INFO_INITIALIZER { '%', '_', L'%', L'_' } typedef struct SPatternCompareInfo { - char matchAll; // symbol for match all wildcard, default: '%' - char matchOne; // symbol for match one wildcard, default: '_' + char matchAll; // symbol for match all wildcard, default: '%' + char matchOne; // symbol for match one wildcard, default: '_' + TdUcs4 umatchAll; // unicode version matchAll + TdUcs4 umatchOne; // unicode version matchOne } SPatternCompareInfo; -int32_t patternMatch(const char *pattern, const char *str, size_t size, const SPatternCompareInfo *pInfo); +int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t ssize, const SPatternCompareInfo *pInfo); -int32_t WCSPatternMatch(const TdUcs4 *pattern, const TdUcs4 *str, size_t size, const SPatternCompareInfo *pInfo); +int32_t wcsPatternMatch(const TdUcs4 *pattern, size_t psize, const TdUcs4 *str, size_t ssize, const SPatternCompareInfo *pInfo); int32_t taosArrayCompareString(const void *a, const void *b); @@ -79,9 +80,11 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight); int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight); int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight); -int32_t compareStrRegexComp(const void *pLeft, const void *pRight); -int32_t compareStrRegexCompMatch(const void *pLeft, const void *pRight); -int32_t compareStrRegexCompNMatch(const void *pLeft, const void *pRight); +int32_t comparestrRegexMatch(const void *pLeft, const void *pRight); +int32_t comparestrRegexNMatch(const void *pLeft, const void *pRight); + +int32_t comparewcsRegexMatch(const void *pLeft, const void *pRight); +int32_t comparewcsRegexNMatch(const void *pLeft, const void *pRight); int32_t compareInt8ValDesc(const void *pLeft, const void *pRight); int32_t compareInt16ValDesc(const void *pLeft, const void *pRight); @@ -99,11 +102,11 @@ int32_t compareUint64ValDesc(const void *pLeft, const void *pRight); int32_t compareLenPrefixedStrDesc(const void *pLeft, const void *pRight); int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight); -int32_t compareStrPatternMatch(const void *pLeft, const void *pRight); -int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight); +int32_t comparestrPatternMatch(const void *pLeft, const void *pRight); +int32_t comparestrPatternNMatch(const void *pLeft, const void *pRight); -int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight); -int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight); +int32_t comparewcsPatternMatch(const void *pLeft, const void *pRight); +int32_t comparewcsPatternNMatch(const void *pLeft, const void *pRight); int32_t compareInt8Int16(const void *pLeft, const void *pRight); int32_t compareInt8Int32(const void *pLeft, const void *pRight); diff --git a/include/util/tdef.h b/include/util/tdef.h index e1d421a39930368a5f66744c0f9fbf7a07475831..9036befc028fde0572018d641495ee521aa688ea 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -189,12 +189,14 @@ typedef enum ELogicConditionType { #define TSDB_MAX_COLUMNS 4096 #define TSDB_MIN_COLUMNS 2 // PRIMARY COLUMN(timestamp) + other columns -#define TSDB_NODE_NAME_LEN 64 -#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string -#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string -#define TSDB_CGROUP_LEN 193 // it is a null-terminated string -#define TSDB_DB_NAME_LEN 65 -#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) +#define TSDB_NODE_NAME_LEN 64 +#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string +#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string +#define TSDB_CGROUP_LEN 193 // it is a null-terminated string +#define TSDB_USER_CGROUP_LEN (TSDB_USER_LEN + TSDB_CGROUP_LEN) // it is a null-terminated string +#define TSDB_STREAM_NAME_LEN 193 // it is a null-terminated string +#define TSDB_DB_NAME_LEN 65 +#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_FUNC_NAME_LEN 65 #define TSDB_FUNC_COMMENT_LEN 1024 * 1024 @@ -212,8 +214,8 @@ typedef enum ELogicConditionType { #define TSDB_INDEX_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_INDEX_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_TYPE_STR_MAX_LEN 32 #define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) -#define TSDB_TOPIC_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) -#define TSDB_STREAM_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) +#define TSDB_TOPIC_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_TOPIC_NAME_LEN + TSDB_NAME_DELIMITER_LEN) +#define TSDB_STREAM_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_STREAM_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CGROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) #define TSDB_PARTITION_KEY_LEN (TSDB_SUBSCRIBE_KEY_LEN + 20) #define TSDB_COL_NAME_LEN 65 @@ -254,7 +256,7 @@ typedef enum ELogicConditionType { #define TSDB_EP_LEN (TSDB_FQDN_LEN + 6) #define TSDB_IPv4ADDR_LEN 16 #define TSDB_FILENAME_LEN 128 -#define TSDB_SHOW_SQL_LEN 1024 +#define TSDB_SHOW_SQL_LEN 2048 #define TSDB_SLOW_QUERY_SQL_LEN 512 #define TSDB_SHOW_SUBQUERY_LEN 1000 @@ -310,7 +312,7 @@ typedef enum ELogicConditionType { #define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute #define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved. #define TSDB_MAX_KEEP_NS (365 * 292 * 1440) // data in db to be reserved. -#define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years +#define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years #define TSDB_MIN_MINROWS_FBLOCK 10 #define TSDB_MAX_MINROWS_FBLOCK 1000 #define TSDB_DEFAULT_MINROWS_FBLOCK 100 @@ -498,7 +500,7 @@ enum { #define DEFAULT_PAGESIZE 4096 #define VNODE_TIMEOUT_SEC 60 -#define MNODE_TIMEOUT_SEC 10 +#define MNODE_TIMEOUT_SEC 60 #ifdef __cplusplus } diff --git a/include/util/tencode.h b/include/util/tencode.h index a6dd58297e8c1dba644d86eb5145b273406fbf9e..ff97a205073822be2c98e39469066a0470794a51 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -116,6 +116,7 @@ static int32_t tEncodeI64v(SEncoder* pCoder, int64_t val); static int32_t tEncodeFloat(SEncoder* pCoder, float val); static int32_t tEncodeDouble(SEncoder* pCoder, double val); static int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, uint32_t len); +static int32_t tEncodeBinaryEx(SEncoder* pCoder, const uint8_t* val, uint32_t len); static int32_t tEncodeCStrWithLen(SEncoder* pCoder, const char* val, uint32_t len); static int32_t tEncodeCStr(SEncoder* pCoder, const char* val); diff --git a/include/util/tjson.h b/include/util/tjson.h index df433227ca83ce51e63a1265db9ac28ed0897954..6922930c13625e0a5722bc07af1578feee42c7ff 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -30,6 +30,27 @@ extern "C" { val = _tmp; \ } while (0) +#define tjsonGetInt32ValueFromDouble(pJson, pName, val, code) \ + do { \ + double _tmp = 0; \ + code = tjsonGetDoubleValue(pJson, pName, &_tmp); \ + val = (int32_t)_tmp; \ + } while (0) + +#define tjsonGetInt8ValueFromDouble(pJson, pName, val, code) \ + do { \ + double _tmp = 0; \ + code = tjsonGetDoubleValue(pJson, pName, &_tmp); \ + val = (int8_t)_tmp; \ + } while (0) + +#define tjsonGetUInt16ValueFromDouble(pJson, pName, val, code) \ + do { \ + double _tmp = 0; \ + code = tjsonGetDoubleValue(pJson, pName, &_tmp); \ + val = (uint16_t)_tmp; \ + } while (0) + typedef void SJson; SJson* tjsonCreateObject(); diff --git a/include/util/tlog.h b/include/util/tlog.h index e6ef7f388ff7af4bab6c29a8a62d39d01d31f256..808377fa772893c7e25ce9c03c749b81fef49092 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -83,9 +83,26 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons #endif ; -bool taosAssert(bool condition, const char *file, int32_t line, const char *format, ...); -#define ASSERTS(condition, ...) taosAssert(condition, __FILE__, __LINE__, __VA_ARGS__) -#define ASSERT(condition) ASSERTS(condition, "assert info not provided") +bool taosAssertDebug(bool condition, const char *file, int32_t line, const char *format, ...); +bool taosAssertRelease(bool condition); + +// Disable all asserts that may compromise the performance. +#if defined DISABLE_ASSERT +#define ASSERT(condition) +#define ASSERTS(condition, ...) +#else +#define ASSERTS(condition, ...) taosAssertDebug(condition, __FILE__, __LINE__, __VA_ARGS__) +#ifdef NDEBUG +#define ASSERT(condition) taosAssertRelease(condition) +#else +#define ASSERT(condition) taosAssertDebug(condition, __FILE__, __LINE__, "assert info not provided") +#endif +#endif + +void taosLogCrashInfo(char* nodeType, char* pMsg, int64_t msgLen, int signum, void *sigInfo); +void taosReadCrashInfo(char* filepath, char** pMsg, int64_t* pMsgLen, TdFilePtr* pFd); +void taosReleaseCrashLogFile(TdFilePtr pFile, bool truncateFile); +int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime); // clang-format off #define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} diff --git a/include/util/tutil.h b/include/util/tutil.h index de963001557519b7dd9355187122535c77b3f17a..9fb68aebdc2943b37cb082f758deca40a8feab6b 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -29,11 +29,17 @@ extern "C" { int32_t strdequote(char *src); size_t strtrim(char *src); char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote); +TdUcs4* wcsnchr(const TdUcs4* haystack, TdUcs4 needle, size_t len); + char **strsplit(char *src, const char *delim, int32_t *num); char *strtolower(char *dst, const char *src); char *strntolower(char *dst, const char *src, int32_t n); char *strntolower_s(char *dst, const char *src, int32_t n); int64_t strnatoi(char *num, int32_t len); + +size_t tstrncspn(const char *str, size_t ssize, const char *reject, size_t rsize); +size_t twcsncspn(const TdUcs4 *wcs, size_t size, const TdUcs4 *reject, size_t rsize); + char *strbetween(char *string, char *begin, char *end); char *paGetToken(char *src, char **token, int32_t *tokenLen); diff --git a/include/util/tworker.h b/include/util/tworker.h index 8766f87a0870aef3b3a4d08ed84ba55dc96c50f7..0636f16dbbd50b709de18ce556d8a9dcbdcfba8d 100644 --- a/include/util/tworker.h +++ b/include/util/tworker.h @@ -17,6 +17,7 @@ #define _TD_UTIL_WORKER_H_ #include "tqueue.h" +#include "tarray.h" #ifdef __cplusplus extern "C" { @@ -26,10 +27,10 @@ typedef struct SQWorkerPool SQWorkerPool; typedef struct SWWorkerPool SWWorkerPool; typedef struct SQWorker { - int32_t id; // worker id - int64_t pid; // thread pid - TdThread thread; // thread id - SQWorkerPool *pool; + int32_t id; // worker id + int64_t pid; // thread pid + TdThread thread; // thread id + void *pool; } SQWorker; typedef struct SQWorkerPool { @@ -42,6 +43,14 @@ typedef struct SQWorkerPool { TdThreadMutex mutex; } SQWorkerPool; +typedef struct SAutoQWorkerPool { + float ratio; + STaosQset *qset; + const char *name; + SArray *workers; + TdThreadMutex mutex; +} SAutoQWorkerPool; + typedef struct SWWorker { int32_t id; // worker id int64_t pid; // thread pid @@ -65,6 +74,11 @@ void tQWorkerCleanup(SQWorkerPool *pool); STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp); void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue); +int32_t tAutoQWorkerInit(SAutoQWorkerPool *pool); +void tAutoQWorkerCleanup(SAutoQWorkerPool *pool); +STaosQueue *tAutoQWorkerAllocQueue(SAutoQWorkerPool *pool, void *ahandle, FItem fp); +void tAutoQWorkerFreeQueue(SAutoQWorkerPool *pool, STaosQueue *queue); + int32_t tWWorkerInit(SWWorkerPool *pool); void tWWorkerCleanup(SWWorkerPool *pool); STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp); diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index e22aa85c978c65d24047f85f9cbdaab84e124670..3d3dfc8e7322ac11ca7bc30b95127c2d4590271d 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -43,6 +43,9 @@ # Switch for allowing TDengine to collect and report service usage information # telemetryReporting 1 +# Switch for allowing TDengine to collect and report crash information +# crashReporting 1 + # The maximum number of vnodes supported by this dnode # supportVnodes 0 diff --git a/packaging/release.sh b/packaging/release.sh deleted file mode 100755 index 7a8a08352f7facdd42621b56db9278dd5829b844..0000000000000000000000000000000000000000 --- a/packaging/release.sh +++ /dev/null @@ -1,319 +0,0 @@ -#!/bin/bash -# -# Generate the deb package for ubuntu, or rpm package for centos, or tar.gz package for other linux os - -set -e -# set -x - -# release.sh -v [cluster | edge] -# -c [aarch32 | aarch64 | x64 | x86 | mips64 | loongarch64...] -# -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] -# -V [stable | beta] -# -l [full | lite] -# -s [static | dynamic] -# -d [taos | ...] -# -n [2.0.0.3] -# -m [2.0.0.0] -# -H [ false | true] - -# set parameters by default value -verMode=edge # [cluster, edge, cloud] -verType=stable # [stable, beta] -cpuType=x64 # [aarch32 | aarch64 | x64 | x86 | mips64 loongarch64...] -osType=Linux # [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] -pagMode=full # [full | lite] -soMode=dynamic # [static | dynamic] -dbName=taos # [taos | ...] -allocator=glibc # [glibc | jemalloc] -verNumber="" -verNumberComp="3.0.0.0" -httpdBuild=false - -while getopts "hv:V:c:o:l:s:d:a:n:m:H:" arg; do - case $arg in - v) - #echo "verMode=$OPTARG" - verMode=$(echo $OPTARG) - ;; - V) - #echo "verType=$OPTARG" - verType=$(echo $OPTARG) - ;; - c) - #echo "cpuType=$OPTARG" - cpuType=$(echo $OPTARG) - ;; - l) - #echo "pagMode=$OPTARG" - pagMode=$(echo $OPTARG) - ;; - s) - #echo "soMode=$OPTARG" - soMode=$(echo $OPTARG) - ;; - d) - #echo "dbName=$OPTARG" - dbName=$(echo $OPTARG) - ;; - a) - #echo "allocator=$OPTARG" - allocator=$(echo $OPTARG) - ;; - n) - #echo "verNumber=$OPTARG" - verNumber=$(echo $OPTARG) - ;; - m) - #echo "verNumberComp=$OPTARG" - verNumberComp=$(echo $OPTARG) - ;; - o) - #echo "osType=$OPTARG" - osType=$(echo $OPTARG) - ;; - H) - #echo "httpdBuild=$OPTARG" - httpdBuild=$(echo $OPTARG) - ;; - h) - echo "Usage: $(basename $0) -v [cluster | edge] " - echo " -c [aarch32 | aarch64 | x64 | x86 | mips64 | loongarch64 ...] " - echo " -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] " - echo " -V [stable | beta] " - echo " -l [full | lite] " - echo " -a [glibc | jemalloc] " - echo " -s [static | dynamic] " - echo " -d [taos | ...] " - echo " -n [version number] " - echo " -m [compatible version number] " - echo " -H [false | true] " - exit 0 - ;; - ?) #unknow option - echo "unkonw argument" - exit 1 - ;; - esac -done - -osType=$(uname) - -echo "verMode=${verMode} verType=${verType} cpuType=${cpuType} osType=${osType} pagMode=${pagMode} soMode=${soMode} dbName=${dbName} allocator=${allocator} verNumber=${verNumber} verNumberComp=${verNumberComp} httpdBuild=${httpdBuild}" - -curr_dir=$(pwd) - -if [ "$osType" == "Darwin" ]; then - script_dir=$(dirname $0) - cd ${script_dir} - script_dir="$(pwd)" - top_dir=${script_dir}/.. -else - script_dir="$(dirname $(readlink -f $0))" - top_dir="$(readlink -f ${script_dir}/..)" -fi - -csudo="" -#if command -v sudo > /dev/null; then -# csudo="sudo " -#fi - -function is_valid_version() { - [ -z $1 ] && return 1 || : - - rx='^([0-9]+\.){3}(\*|[0-9]+)$' - if [[ $1 =~ $rx ]]; then - return 0 - fi - return 1 -} - -function vercomp() { - if [[ $1 == $2 ]]; then - echo 0 - exit 0 - fi - - local IFS=. - local i ver1=($1) ver2=($2) - - # fill empty fields in ver1 with zeros - for ((i = ${#ver1[@]}; i < ${#ver2[@]}; i++)); do - ver1[i]=0 - done - - for ((i = 0; i < ${#ver1[@]}; i++)); do - if [[ -z ${ver2[i]} ]]; then - # fill empty fields in ver2 with zeros - ver2[i]=0 - fi - if ((10#${ver1[i]} > 10#${ver2[i]})); then - echo 1 - exit 0 - fi - if ((10#${ver1[i]} < 10#${ver2[i]})); then - echo 2 - exit 0 - fi - done - echo 0 -} - -# 1. check version information -if ( (! is_valid_version $verNumber) || (! is_valid_version $verNumberComp) || [[ "$(vercomp $verNumber $verNumberComp)" == '2' ]]); then - echo "please enter correct version" - exit 0 -fi - -echo "=======================new version number: ${verNumber}, compatible version: ${verNumberComp}======================================" - -build_time=$(date +"%F %R") - -# get commint id from git -gitinfo=$(git rev-parse --verify HEAD) - -if [[ "$verMode" == "cluster" ]] || [[ "$verMode" == "cloud" ]]; then - enterprise_dir="${top_dir}/../enterprise" - cd ${enterprise_dir} - gitinfoOfInternal=$(git rev-parse --verify HEAD) -else - gitinfoOfInternal=NULL -fi - -cd "${curr_dir}" - -# 2. cmake executable file -compile_dir="${top_dir}/debug" -if [ -d ${compile_dir} ]; then - rm -rf ${compile_dir} -fi - -mkdir -p ${compile_dir} -cd ${compile_dir} - -if [[ "$allocator" == "jemalloc" ]]; then - allocator_macro="-DJEMALLOC_ENABLED=true" -else - allocator_macro="" -fi - -if [[ "$dbName" != "taos" ]]; then - source ${enterprise_dir}/packaging/oem/sed_$dbName.sh - replace_community_$dbName -fi - -if [[ "$httpdBuild" == "true" ]]; then - BUILD_HTTP=true -else - BUILD_HTTP=false -fi - -if [[ "$verMode" == "cluster" ]] || [[ "$verMode" == "cloud" ]]; then - BUILD_HTTP=internal -fi - -if [[ "$pagMode" == "full" ]]; then - BUILD_TOOLS=true -else - BUILD_TOOLS=false -fi - -# check support cpu type -if [[ "$cpuType" == "x64" ]] || [[ "$cpuType" == "aarch64" ]] || [[ "$cpuType" == "aarch32" ]] || [[ "$cpuType" == "arm64" ]] || [[ "$cpuType" == "arm32" ]] || [[ "$cpuType" == "mips64" ]] || [[ "$cpuType" == "loongarch64" ]] ; then - if [ "$verMode" == "edge" ]; then - # community-version compile - cmake ../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DPAGMODE=${pagMode} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro} - elif [ "$verMode" == "cloud" ]; then - cmake ../../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DBUILD_TAOSX=true -DBUILD_CLOUD=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro} - elif [ "$verMode" == "cluster" ]; then - if [[ "$dbName" != "taos" ]]; then - replace_enterprise_$dbName - fi - cmake ../../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DBUILD_TAOSX=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro} - fi -else - echo "input cpuType=${cpuType} error!!!" - exit 1 -fi - -ostype=`uname` -if [ "${ostype}" == "Darwin" ]; then - CORES=$(sysctl -n hw.ncpu) -else - CORES=$(grep -c ^processor /proc/cpuinfo) -fi - -if [[ "$allocator" == "jemalloc" ]]; then - # jemalloc need compile first, so disable parallel build - make -j ${CORES} && ${csudo}make install -else - make -j ${CORES} && ${csudo}make install -fi - -cd ${curr_dir} - -# 3. Call the corresponding script for packaging -if [ "$osType" != "Darwin" ]; then - if [[ "$verMode" != "cluster" ]] && [[ "$verMode" != "cloud" ]] && [[ "$pagMode" == "full" ]] && [[ "$cpuType" == "x64" ]] && [[ "$dbName" == "taos" ]]; then - ret='0' - command -v dpkg >/dev/null 2>&1 || { ret='1'; } - if [ "$ret" -eq 0 ]; then - echo "====do deb package for the ubuntu system====" - output_dir="${top_dir}/debs" - if [ -d ${output_dir} ]; then - rm -rf ${output_dir} - fi - mkdir -p ${output_dir} - cd ${script_dir}/deb - ${csudo}./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} - - if [[ "$pagMode" == "full" ]]; then - if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then - cd ${top_dir}/tools/taos-tools/packaging/deb - taos_tools_ver=$(git tag |grep -v taos | sort | tail -1) - [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" - - ${csudo}./make-taos-tools-deb.sh ${top_dir} \ - ${compile_dir} ${output_dir} ${taos_tools_ver} ${cpuType} ${osType} ${verMode} ${verType} - fi - fi - else - echo "==========dpkg command not exist, so not release deb package!!!" - fi - ret='0' - command -v rpmbuild >/dev/null 2>&1 || { ret='1'; } - if [ "$ret" -eq 0 ]; then - echo "====do rpm package for the centos system====" - output_dir="${top_dir}/rpms" - if [ -d ${output_dir} ]; then - rm -rf ${output_dir} - fi - mkdir -p ${output_dir} - cd ${script_dir}/rpm - ${csudo}./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} - - if [[ "$pagMode" == "full" ]]; then - if [ -d ${top_dir}/tools/taos-tools/packaging/rpm ]; then - cd ${top_dir}/tools/taos-tools/packaging/rpm - taos_tools_ver=$(git tag |grep -v taos | sort | tail -1) - [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" - - ${csudo}./make-taos-tools-rpm.sh ${top_dir} \ - ${compile_dir} ${output_dir} ${taos_tools_ver} ${cpuType} ${osType} ${verMode} ${verType} - fi - fi - else - echo "==========rpmbuild command not exist, so not release rpm package!!!" - fi - fi - - echo "====do tar.gz package for all systems====" - cd ${script_dir}/tools - - ${csudo}./makepkg.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${verNumberComp} ${dbName} - ${csudo}./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} - -else - cd ${script_dir}/tools - ./makepkg.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${verNumberComp} ${dbName} - ./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} -fi diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 63009e5421ec9db8e787980846c00b14beaab75a..2a078b5eabd6129ae7bf6bc4302e888341329ea4 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -481,11 +481,11 @@ function install_adapter_config() { ${csudo}mkdir -p ${cfg_install_dir} [ -f ${script_dir}/cfg/${adapterName}.toml ] && ${csudo}cp ${script_dir}/cfg/${adapterName}.toml ${cfg_install_dir} [ -f ${cfg_install_dir}/${adapterName}.toml ] && ${csudo}chmod 644 ${cfg_install_dir}/${adapterName}.toml + else + [ -f ${script_dir}/cfg/${adapterName}.toml ] && + ${csudo}cp -f ${script_dir}/cfg/${adapterName}.toml ${cfg_install_dir}/${adapterName}.toml.new fi - [ -f ${script_dir}/cfg/${adapterName}.toml ] && - ${csudo}cp -f ${script_dir}/cfg/${adapterName}.toml ${cfg_install_dir}/${adapterName}.toml.new - [ -f ${cfg_install_dir}/${adapterName}.toml ] && ${csudo}ln -s ${cfg_install_dir}/${adapterName}.toml ${install_main_dir}/cfg/${adapterName}.toml @@ -499,9 +499,10 @@ function install_config() { ${csudo}mkdir -p ${cfg_install_dir} [ -f ${script_dir}/cfg/${configFile} ] && ${csudo}cp ${script_dir}/cfg/${configFile} ${cfg_install_dir} ${csudo}chmod 644 ${cfg_install_dir}/* + else + ${csudo}cp -f ${script_dir}/cfg/${configFile} ${cfg_install_dir}/${configFile}.new fi - ${csudo}cp -f ${script_dir}/cfg/${configFile} ${cfg_install_dir}/${configFile}.new ${csudo}ln -s ${cfg_install_dir}/${configFile} ${install_main_dir}/cfg [ ! -z $1 ] && return 0 || : # only install client diff --git a/packaging/tools/make_install.bat b/packaging/tools/make_install.bat index c19519f9a1fc4c014c01496ca32925ba1eb19516..bf7418ad7935cb33f4a9d765e6c319d86383db69 100644 --- a/packaging/tools/make_install.bat +++ b/packaging/tools/make_install.bat @@ -14,6 +14,7 @@ set binary_dir=%3 set binary_dir=%binary_dir:/=\\% set osType=%4 set verNumber=%5 +set Enterprise=%6 set target_dir=C:\\TDengine if not exist %target_dir% ( @@ -57,7 +58,33 @@ if exist %binary_dir%\\build\\lib\\taosws.dll ( if exist %binary_dir%\\build\\bin\\taosdump.exe ( copy %binary_dir%\\build\\bin\\taosdump.exe %target_dir% > nul ) - +if %Enterprise% == TRUE ( + if exist %binary_dir%\\build\\bin\\taosx.exe ( + copy %binary_dir%\\build\\bin\\taosx.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\tmq_sim.exe ( + copy %binary_dir%\\build\\bin\\tmq_sim.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\tsim.exe ( + copy %binary_dir%\\build\\bin\\tsim.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\tmq_taosx_ci.exe ( + copy %binary_dir%\\build\\bin\\tmq_taosx_ci.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\tmq_demo.exe ( + copy %binary_dir%\\build\\bin\\tmq_demo.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\dumper.exe ( + copy %binary_dir%\\build\\bin\\dumper.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\runUdf.exe ( + copy %binary_dir%\\build\\bin\\runUdf.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\create_table.exe ( + copy %binary_dir%\\build\\bin\\create_table.exe %target_dir% > nul + ) +) + copy %binary_dir%\\build\\bin\\taosd.exe %target_dir% > nul copy %binary_dir%\\build\\bin\\udfd.exe %target_dir% > nul diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 482345dcd819dce683d3aa68438e6fa9fd5a4a9f..b246e0225f87889a553a6762ba26f83ce9bd2afd 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -109,6 +109,13 @@ function kill_taosadapter() { fi } +function kill_taoskeeper() { + pid=$(ps -ef | grep "taoskeeper" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} + function kill_taosd() { # ${csudo}pkill -f taosd || : pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}') @@ -161,6 +168,7 @@ function install_bin() { ${csudo}rm -f ${bin_link_dir}/udfd || : ${csudo}rm -f ${bin_link_dir}/taosadapter || : ${csudo}rm -f ${bin_link_dir}/taosBenchmark || : + ${csudo}rm -f ${bin_link_dir}/taoskeeper || : ${csudo}rm -f ${bin_link_dir}/taosdemo || : ${csudo}rm -f ${bin_link_dir}/taosdump || : ${csudo}rm -f ${bin_link_dir}/rmtaos || : @@ -179,6 +187,7 @@ function install_bin() { [ -x ${bin_dir}/taosdump ] && ${csudo}ln -s ${bin_dir}/taosdump ${bin_link_dir}/taosdump || : [ -x ${bin_dir}/set_core.sh ] && ${csudo}ln -s ${bin_dir}/set_core.sh ${bin_link_dir}/set_core || : [ -x ${bin_dir}/remove.sh ] && ${csudo}ln -s ${bin_dir}/remove.sh ${bin_link_dir}/rmtaos || : + [ -x ${bin_dir}/taoskeeper ] && ${csudo}ln -sf ${bin_dir}/taoskeeper ${bin_link_dir}/taoskeeper || : } function add_newHostname_to_hosts() { @@ -351,6 +360,22 @@ function install_taosadapter_config() { ${csudo}ln -s ${cfg_install_dir}/taosadapter.toml ${cfg_dir} } +function install_taoskeeper_config() { + if [ ! -f "${cfg_install_dir}/keeper.toml" ]; then + [ ! -d %{cfg_install_dir} ] && + ${csudo}${csudo}mkdir -p ${cfg_install_dir} + [ -f ${cfg_dir}/keeper.toml ] && ${csudo}cp ${cfg_dir}/keeper.toml ${cfg_install_dir} + [ -f ${cfg_install_dir}/keeper.toml ] && + ${csudo}chmod 644 ${cfg_install_dir}/keeper.toml + fi + + [ -f ${cfg_dir}/keeper.toml ] && + ${csudo}mv ${cfg_dir}/keeper.toml ${cfg_dir}/keeper.toml.new + + [ -f ${cfg_install_dir}/keeper.toml ] && + ${csudo}ln -s ${cfg_install_dir}/keeper.toml ${cfg_dir} +} + function install_config() { if [ ! -f "${cfg_install_dir}/taos.cfg" ]; then ${csudo}${csudo}mkdir -p ${cfg_install_dir} @@ -583,6 +608,7 @@ function install_TDengine() { install_bin install_config install_taosadapter_config + install_taoskeeper_config install_taosadapter_service install_service install_app diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index ea76f726ea1b4ee7e89cd28053c11ac1d6dca2e1..3fc5cc9cecb6d164ce6ec009f67dcd7de2a99564 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -149,7 +149,6 @@ typedef struct STscObj { int32_t numOfReqs; // number of sqlObj bound to this connection SAppInstInfo* pAppInfo; SHashObj* pRequests; - int8_t schemalessType; // todo remove it, this attribute should be move to request } STscObj; typedef struct SResultColumn { @@ -313,6 +312,8 @@ extern SAppInfo appInfo; extern int32_t clientReqRefPool; extern int32_t clientConnRefPool; extern int32_t timestampDeltaLimit; +extern int64_t lastClusterId; + __async_send_cb_fn_t getMsgRspHandle(int32_t msgType); @@ -340,6 +341,7 @@ void resetConnectDB(STscObj* pTscObj); int taos_options_imp(TSDB_OPTION option, const char* str); void* openTransporter(const char* user, const char* auth, int32_t numOfThreads); +void tscStopCrashReport(); typedef struct AsyncArg { SRpcMsg msg; diff --git a/source/client/inc/clientLog.h b/source/client/inc/clientLog.h index 0cb36ff61db570f2c6fb488d2964fc38c2cfd7f4..c29f495201d7b9ec0360abf6a5414798739b3da8 100644 --- a/source/client/inc/clientLog.h +++ b/source/client/inc/clientLog.h @@ -30,7 +30,7 @@ extern "C" { #define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0) #define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", DEBUG_TRACE, cDebugFlag, __VA_ARGS__); }} while(0) #define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0) -//#define tscPerf(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", DEBUG_INFO, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscPerf(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", 0, cDebugFlag, __VA_ARGS__); }} while(0) // clang-format on #ifdef __cplusplus diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h new file mode 100644 index 0000000000000000000000000000000000000000..15d91641a470aceb65e5cecf65a28b0cac635b64 --- /dev/null +++ b/source/client/inc/clientSml.h @@ -0,0 +1,246 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TDENGINE_CLIENTSML_H +#define TDENGINE_CLIENTSML_H +#ifdef __cplusplus +extern "C" { +#endif + +#include "catalog.h" +#include "clientInt.h" +#include "osThread.h" +#include "query.h" +#include "taos.h" +#include "taoserror.h" +#include "tcommon.h" +#include "tdef.h" +#include "tglobal.h" +#include "tlog.h" +#include "tmsg.h" +#include "tname.h" +#include "ttime.h" +#include "ttypes.h" +#include "cJSON.h" + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#ifndef likely +#define likely(expr) expect((expr) != 0, 1) +#endif +#ifndef unlikely +#define unlikely(expr) expect((expr) != 0, 0) +#endif + +#define SPACE ' ' +#define COMMA ',' +#define EQUAL '=' +#define QUOTE '"' +#define SLASH '\\' + +#define JUMP_SPACE(sql, sqlEnd) \ + while (sql < sqlEnd) { \ + if (unlikely(*sql == SPACE)) \ + sql++; \ + else \ + break; \ + } + +#define IS_INVALID_COL_LEN(len) ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN) +#define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN) + +#define TS "_ts" +#define TS_LEN 3 +#define VALUE "_value" +#define VALUE_LEN 6 + +#define OTD_JSON_FIELDS_NUM 4 +#define MAX_RETRY_TIMES 5 +typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; + +typedef enum { + SCHEMA_ACTION_NULL, + SCHEMA_ACTION_CREATE_STABLE, + SCHEMA_ACTION_ADD_COLUMN, + SCHEMA_ACTION_ADD_TAG, + SCHEMA_ACTION_CHANGE_COLUMN_SIZE, + SCHEMA_ACTION_CHANGE_TAG_SIZE, +} ESchemaAction; + +typedef struct { + const void *key; + int32_t keyLen; + void *value; + bool used; +}Node; + +typedef struct NodeList{ + Node data; + struct NodeList* next; +}NodeList; + +typedef struct { + char *measure; + char *tags; + char *cols; + char *timestamp; + + int32_t measureLen; + int32_t measureTagsLen; + int32_t tagsLen; + int32_t colsLen; + int32_t timestampLen; + + SArray *colArray; +} SSmlLineInfo; + +typedef struct { + const char *sTableName; // super table name + int32_t sTableNameLen; + char childTableName[TSDB_TABLE_NAME_LEN]; + uint64_t uid; + void *key; // for openTsdb + + SArray *tags; + + // elements are SHashObj for find by key quickly + SArray *cols; + STableDataCxt *tableDataCtx; +} SSmlTableInfo; + +typedef struct { + SArray *tags; // save the origin order to create table + SHashObj *tagHash; // elements are + + SArray *cols; + SHashObj *colHash; + + STableMeta *tableMeta; +} SSmlSTableMeta; + +typedef struct { + int32_t len; + char *buf; +} SSmlMsgBuf; + +typedef struct { + int32_t code; + int32_t lineNum; + + int32_t numOfSTables; + int32_t numOfCTables; + int32_t numOfCreateSTables; + int32_t numOfAlterColSTables; + int32_t numOfAlterTagSTables; + + int64_t parseTime; + int64_t schemaTime; + int64_t insertBindTime; + int64_t insertRpcTime; + int64_t endTime; +} SSmlCostInfo; + +typedef struct { + int64_t id; + + SMLProtocolType protocol; + int8_t precision; + bool reRun; + bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) + bool isRawLine; + int32_t ttl; + int32_t uid; // used for automatic create child table + + NodeList *childTables; + NodeList *superTables; + SHashObj *pVgHash; + + STscObj *taos; + SCatalog *pCatalog; + SRequestObj *pRequest; + SQuery *pQuery; + + SSmlCostInfo cost; + int32_t lineNum; + SSmlMsgBuf msgBuf; + + cJSON *root; // for parse json + int8_t offset[OTD_JSON_FIELDS_NUM]; + SSmlLineInfo *lines; // element is SSmlLineInfo + bool parseJsonByLib; + SArray *tagJsonArray; + + // + SArray *preLineTagKV; + SArray *maxTagKVs; + SArray *preLineColKV; + + SSmlLineInfo preLine; + STableMeta *currSTableMeta; + STableDataCxt *currTableDataCtx; + bool needModifySchema; +} SSmlHandle; + +#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \ +&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0) + +#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \ +&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0) + +#define IS_SAME_KEY (maxKV->keyLen == kv.keyLen && memcmp(maxKV->key, kv.key, kv.keyLen) == 0) + +extern int64_t smlFactorNS[3]; +extern int64_t smlFactorS[3]; + +typedef int32_t (*_equal_fn_sml)(const void *, const void *); + +SSmlHandle *smlBuildSmlInfo(TAOS *taos); +void smlDestroyInfo(SSmlHandle *info); +int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset); +int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset); +//SArray *smlJsonParseTags(char *start, char *end); +bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); +void* nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn); +int nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn); +int nodeListSize(NodeList* list); +bool smlDoubleToInt64OverFlow(double num); +int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); +bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); +int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision); +int8_t smlGetTsTypeByLen(int32_t len); +SSmlTableInfo* smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen); +SSmlSTableMeta* smlBuildSTableMeta(bool isDataFormat); +int32_t smlSetCTableName(SSmlTableInfo *oneTable); +STableMeta* smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen); +int32_t is_same_child_table_telnet(const void *a, const void *b); +int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len); +int32_t smlClearForRerun(SSmlHandle *info); +int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg); +uint8_t smlGetTimestampLen(int64_t num); +void clearColValArray(SArray* pCols); +void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag); + +int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); +int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); +int32_t smlParseJSON(SSmlHandle *info, char *payload); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_CLIENTSML_H diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index e5507ccf816b9fd591e6e27ee2974badcb5734ed..2b42de93e3f45ccb95c085c31825d4e0a117538c 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -21,8 +21,6 @@ extern "C" { #endif #include "catalog.h" -typedef void STableDataBlocks; - typedef enum { STMT_TYPE_INSERT = 1, STMT_TYPE_MULTI_INSERT, @@ -43,8 +41,8 @@ typedef enum { } STMT_STATUS; typedef struct SStmtTableCache { - STableDataBlocks *pDataBlock; - void *boundTags; + STableDataCxt *pDataCtx; + void *boundTags; } SStmtTableCache; typedef struct SStmtQueryResInfo { @@ -71,10 +69,11 @@ typedef struct SStmtBindInfo { } SStmtBindInfo; typedef struct SStmtExecInfo { - int32_t affectedRows; - SRequestObj *pRequest; - SHashObj *pBlockHash; - bool autoCreateTbl; + int32_t affectedRows; + SRequestObj *pRequest; + SHashObj *pBlockHash; + STableDataCxt *pCurrBlock; + SSubmitTbData *pCurrTbData; } SStmtExecInfo; typedef struct SStmtSQLInfo { diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 75a72647bd4b62ec8b328828f1a06691bfc8ac15..28a55156aa1dba2ebe65353c0b8b48e593a7917c 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -28,13 +28,16 @@ #include "trpc.h" #include "tsched.h" #include "ttime.h" +#include "thttp.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 SAppInfo appInfo; +int64_t lastClusterId = 0; int32_t clientReqRefPool = -1; int32_t clientConnRefPool = -1; +int32_t clientStop = 0; int32_t timestampDeltaLimit = 900; // s @@ -62,7 +65,10 @@ static int32_t registerRequest(SRequestObj *pRequest, STscObj *pTscObj) { static void deregisterRequest(SRequestObj *pRequest) { const static int64_t SLOW_QUERY_INTERVAL = 3000000L; // todo configurable - assert(pRequest != NULL); + if(pRequest == NULL){ + tscError("pRequest == NULL"); + return; + } STscObj *pTscObj = pRequest->pTscObj; SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; @@ -76,13 +82,19 @@ static void deregisterRequest(SRequestObj *pRequest) { "current:%d, app current:%d", pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000.0, num, currentInst); + tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 + "us, exec:%" PRId64 "us, stmtType:%d", + duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, + pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - pRequest->metric.ctgEnd, + pRequest->metric.execEnd - pRequest->metric.semanticEnd, pRequest->stmtType); + if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType) { - // tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 - // "us, exec:%" PRId64 "us", - // duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, - // pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - - // pRequest->metric.ctgEnd, pRequest->metric.execEnd - pRequest->metric.semanticEnd); - atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); + // tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 + // "us, exec:%" PRId64 "us", + // duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, + // pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - + // pRequest->metric.ctgEnd, pRequest->metric.execEnd - pRequest->metric.semanticEnd); + // atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { // tscPerf("select duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 // "us, planner:%" PRId64 "us, exec:%" PRId64 "us, reqId:0x%" PRIx64, @@ -264,7 +276,6 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c taosThreadMutexInit(&pObj->mutex, NULL); pObj->id = taosAddRef(clientConnRefPool, pObj); - pObj->schemalessType = 1; atomic_add_fetch_64(&pObj->pAppInfo->numOfConns, 1); @@ -385,6 +396,150 @@ void destroyRequest(SRequestObj *pRequest) { removeRequest(pRequest->self); } +void taosClientCrash(int signum, void *sigInfo, void *context) { + taosIgnSignal(SIGTERM); + taosIgnSignal(SIGHUP); + taosIgnSignal(SIGINT); + taosIgnSignal(SIGBREAK); + +#if !defined(WINDOWS) + taosIgnSignal(SIGBUS); +#endif + taosIgnSignal(SIGABRT); + taosIgnSignal(SIGFPE); + taosIgnSignal(SIGSEGV); + + char *pMsg = NULL; + const char *flags = "UTL FATAL "; + ELogLevel level = DEBUG_FATAL; + int32_t dflag = 255; + int64_t msgLen= -1; + + if (tsEnableCrashReport) { + if (taosGenCrashJsonMsg(signum, &pMsg, lastClusterId, appInfo.startTime)) { + taosPrintLog(flags, level, dflag, "failed to generate crash json msg"); + goto _return; + } else { + msgLen = strlen(pMsg); + } + } + +_return: + + taosLogCrashInfo("taos", pMsg, msgLen, signum, sigInfo); + +#ifdef _TD_DARWIN_64 + exit(signum); +#elif defined(WINDOWS) + exit(signum); +#endif +} + +void crashReportThreadFuncUnexpectedStopped(void) { atomic_store_32(&clientStop, -1); } + +static void *tscCrashReportThreadFp(void *param) { + setThreadName("client-crashReport"); + char filepath[PATH_MAX] = {0}; + snprintf(filepath, sizeof(filepath), "%s%s.taosCrashLog", tsLogDir, TD_DIRSEP); + char *pMsg = NULL; + int64_t msgLen = 0; + TdFilePtr pFile = NULL; + bool truncateFile = false; + int32_t sleepTime = 200; + int32_t reportPeriodNum = 3600 * 1000 / sleepTime; + int32_t loopTimes = reportPeriodNum; + +#ifdef WINDOWS + if (taosCheckCurrentInDll()) { + atexit(crashReportThreadFuncUnexpectedStopped); + } +#endif + + while (1) { + if (clientStop) break; + if (loopTimes++ < reportPeriodNum) { + taosMsleep(sleepTime); + continue; + } + + taosReadCrashInfo(filepath, &pMsg, &msgLen, &pFile); + if (pMsg && msgLen > 0) { + if (taosSendHttpReport(tsTelemServer, tsClientCrashReportUri, tsTelemPort, pMsg, msgLen, HTTP_FLAT) != 0) { + tscError("failed to send crash report"); + if (pFile) { + taosReleaseCrashLogFile(pFile, false); + continue; + } + } else { + tscInfo("succeed to send crash report"); + truncateFile = true; + } + } else { + tscDebug("no crash info"); + } + + taosMemoryFree(pMsg); + + if (pMsg && msgLen > 0) { + pMsg = NULL; + continue; + } + + if (pFile) { + taosReleaseCrashLogFile(pFile, truncateFile); + truncateFile = false; + } + + taosMsleep(sleepTime); + loopTimes = 0; + } + + clientStop = -1; + return NULL; +} + +int32_t tscCrashReportInit() { + if (!tsEnableCrashReport) { + return 0; + } + + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + TdThread crashReportThread; + if (taosThreadCreate(&crashReportThread, &thAttr, tscCrashReportThreadFp, NULL) != 0) { + tscError("failed to create crashReport thread since %s", strerror(errno)); + return -1; + } + + taosThreadAttrDestroy(&thAttr); + return 0; +} + +void tscStopCrashReport() { + if (!tsEnableCrashReport) { + return; + } + + if (atomic_val_compare_exchange_32(&clientStop, 0, 1)) { + tscDebug("hb thread already stopped"); + return; + } + + while (atomic_load_32(&clientStop) > 0) { + taosMsleep(100); + } +} + +static void tscSetSignalHandle() { +#if !defined(WINDOWS) + taosSetSignal(SIGBUS, taosClientCrash); +#endif + taosSetSignal(SIGABRT, taosClientCrash); + taosSetSignal(SIGFPE, taosClientCrash); + taosSetSignal(SIGSEGV, taosClientCrash); +} + void taos_init_imp(void) { // In the APIs of other program language, taos_cleanup is not available yet. // So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning. @@ -392,6 +547,10 @@ void taos_init_imp(void) { errno = TSDB_CODE_SUCCESS; taosSeedRand(taosGetTimestampSec()); + appInfo.pid = taosGetPId(); + appInfo.startTime = taosGetTimestampMs(); + appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + deltaToUtcInitOnce(); if (taosCreateLog("taoslog", 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { @@ -404,10 +563,13 @@ void taos_init_imp(void) { return; } + tscSetSignalHandle(); + initQueryModuleMsgHandle(); if (taosConvInit() != 0) { - ASSERTS(0, "failed to init conv"); + tscError("failed to init conv"); + return; } rpcInit(); @@ -433,9 +595,8 @@ void taos_init_imp(void) { taosGetAppName(appInfo.appName, NULL); taosThreadMutexInit(&appInfo.mutex, NULL); - appInfo.pid = taosGetPId(); - appInfo.startTime = taosGetTimestampMs(); - appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + tscCrashReportInit(); + tscDebug("client is initialized successfully"); } diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 47ed2cf035a13314b81f2208718d2c466efb5c68..3cb8a2e1bd9e8c6c2c5a68f2e266611d209326a9 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -376,7 +376,6 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { desc.subPlanNum = 0; } desc.subPlanNum = taosArrayGetSize(desc.subDesc); - ASSERT(desc.subPlanNum == taosArrayGetSize(desc.subDesc)); } else { desc.subDesc = NULL; } @@ -813,7 +812,10 @@ static void hbStopThread() { } SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { - hbMgrInit(); + if(hbMgrInit() != 0){ + terrno = TSDB_CODE_TSC_INTERNAL_ERROR; + return NULL; + } SAppHbMgr *pAppHbMgr = taosMemoryMalloc(sizeof(SAppHbMgr)); if (pAppHbMgr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -899,16 +901,28 @@ int hbMgrInit() { TdThreadMutexAttr attr = {0}; int ret = taosThreadMutexAttrInit(&attr); - assert(ret == 0); + if(ret != 0){ + uError("hbMgrInit:taosThreadMutexAttrInit error") + return ret; + } ret = taosThreadMutexAttrSetType(&attr, PTHREAD_MUTEX_RECURSIVE); - assert(ret == 0); + if(ret != 0){ + uError("hbMgrInit:taosThreadMutexAttrSetType error") + return ret; + } ret = taosThreadMutexInit(&clientHbMgr.lock, &attr); - assert(ret == 0); + if(ret != 0){ + uError("hbMgrInit:taosThreadMutexInit error") + return ret; + } ret = taosThreadMutexAttrDestroy(&attr); - assert(ret == 0); + if(ret != 0){ + uError("hbMgrInit:taosThreadMutexAttrDestroy error") + return ret; + } // init handle funcs hbMgrInitHandle(); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index aaf27a77161a9f415cf63782ce266000977e5297..aeb73784f20c1d84a9d2d894026c9e4d3c863a5b 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -239,7 +239,6 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC .pTransporter = pTscObj->pAppInfo->pTransporter, .pStmtCb = pStmtCb, .pUser = pTscObj->user, - .schemalessType = pTscObj->schemalessType, .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)), .enableSysInfo = pTscObj->sysInfo, .svrVer = pTscObj->sVer, @@ -273,7 +272,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC int32_t execLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { SRetrieveTableRsp* pRsp = NULL; - int32_t code = qExecCommand(pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp); + int32_t code = qExecCommand(&pRequest->pTscObj->id, pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp); if (TSDB_CODE_SUCCESS == code && NULL != pRsp) { code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, true); } @@ -311,7 +310,7 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { return; } - int32_t code = qExecCommand(pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp); + int32_t code = qExecCommand(&pRequest->pTscObj->id ,pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp); if (TSDB_CODE_SUCCESS == code && NULL != pRsp) { code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, true); } @@ -453,7 +452,10 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra } void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols) { - ASSERT(pSchema != NULL && numOfCols > 0); + if(pResInfo == NULL || pSchema == NULL || numOfCols <= 0){ + tscError("invalid paras, pResInfo == NULL || pSchema == NULL || numOfCols <= 0"); + return; + } pResInfo->numOfCols = numOfCols; if (pResInfo->fields != NULL) { @@ -464,7 +466,10 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t } pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); - ASSERT(numOfCols == pResInfo->numOfCols); + if(numOfCols != pResInfo->numOfCols){ + tscError("numOfCols:%d != pResInfo->numOfCols:%d", numOfCols, pResInfo->numOfCols); + return; + } for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { pResInfo->fields[i].bytes = pSchema[i].bytes; @@ -741,47 +746,21 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList } int32_t handleSubmitExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet* epset) { - int32_t code = 0; - SArray* pArray = NULL; - SSubmitRsp* pRsp = (SSubmitRsp*)res; - if (pRsp->nBlocks <= 0) { - taosMemoryFreeClear(pRsp->pBlocks); + SArray* pArray = NULL; + SSubmitRsp2* pRsp = (SSubmitRsp2*)res; + if (NULL == pRsp->aCreateTbRsp) { return TSDB_CODE_SUCCESS; } - pArray = taosArrayInit(pRsp->nBlocks, sizeof(STbSVersion)); - if (NULL == pArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < pRsp->nBlocks; ++i) { - SSubmitBlkRsp* blk = pRsp->pBlocks + i; - if (blk->pMeta) { - handleCreateTbExecRes(blk->pMeta, pCatalog); - tFreeSTableMetaRsp(blk->pMeta); - taosMemoryFreeClear(blk->pMeta); - } - - if (NULL == blk->tblFName || 0 == blk->tblFName[0]) { - continue; + int32_t tbNum = taosArrayGetSize(pRsp->aCreateTbRsp); + for (int32_t i = 0; i < tbNum; ++i) { + SVCreateTbRsp* pTbRsp = (SVCreateTbRsp*)taosArrayGet(pRsp->aCreateTbRsp, i); + if (pTbRsp->pMeta) { + handleCreateTbExecRes(pTbRsp->pMeta, pCatalog); } - - STbSVersion tbSver = {.tbFName = blk->tblFName, .sver = blk->sver}; - taosArrayPush(pArray, &tbSver); } - SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = *epset}; - - code = catalogChkTbMetaVersion(pCatalog, &conn, pArray); - -_return: - - taosArrayDestroy(pArray); - return code; + return TSDB_CODE_SUCCESS; } int32_t handleQueryExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet* epset) { @@ -1253,7 +1232,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t int64_t transporterId = 0; asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body); - + tsem_wait(&pRequest->body.rspSem); if (pRequest->code != TSDB_CODE_SUCCESS) { const char* errorMsg = @@ -1366,7 +1345,10 @@ int32_t doProcessMsgFromServer(void* param) { SEpSet* pEpSet = arg->pEpset; SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle; - assert(pMsg->info.ahandle != NULL); + if(pMsg->info.ahandle == NULL){ + tscError("doProcessMsgFromServer pMsg->info.ahandle == NULL"); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } STscObj* pTscObj = NULL; STraceId* trace = &pMsg->info.traceId; @@ -1379,8 +1361,10 @@ int32_t doProcessMsgFromServer(void* param) { if (pSendInfo->requestObjRefId != 0) { SRequestObj* pRequest = (SRequestObj*)taosAcquireRef(clientReqRefPool, pSendInfo->requestObjRefId); if (pRequest) { - assert(pRequest->self == pSendInfo->requestObjRefId); - + if(pRequest->self != pSendInfo->requestObjRefId){ + tscError("doProcessMsgFromServer pRequest->self:%"PRId64" != pSendInfo->requestObjRefId:%"PRId64, pRequest->self, pSendInfo->requestObjRefId); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } pRequest->metric.rsp = taosGetTimestampUs(); pTscObj = pRequest->pTscObj; /* @@ -1430,6 +1414,21 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { memcpy((void*)tEpSet, (void*)pEpSet, sizeof(SEpSet)); } + // pMsg is response msg + if (pMsg->msgType == TDMT_MND_CONNECT + 1) { + // restore origin code + if (pMsg->code == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED) { + pMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + } else if (pMsg->code == TSDB_CODE_RPC_SOMENODE_BROKEN_LINK) { + pMsg->code = TSDB_CODE_RPC_BROKEN_LINK; + } + } else { + // uniform to one error code: TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED + if (pMsg->code == TSDB_CODE_RPC_SOMENODE_BROKEN_LINK) { + pMsg->code = TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED; + } + } + AsyncArg* arg = taosMemoryCalloc(1, sizeof(AsyncArg)); arg->msg = *pMsg; arg->pEpset = tEpSet; @@ -1507,7 +1506,9 @@ void doSetOneRowPtr(SReqResultInfo* pResultInfo) { } void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) { - assert(pRequest != NULL); + if(pRequest == NULL){ + return NULL; + } SReqResultInfo* pResultInfo = &pRequest->body.resInfo; if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { @@ -1561,7 +1562,9 @@ static void syncFetchFn(void* param, TAOS_RES* res, int32_t numOfRows) { } void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) { - assert(pRequest != NULL); + if(pRequest == NULL){ + return NULL; + } SReqResultInfo* pResultInfo = &pRequest->body.resInfo; if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { @@ -1625,8 +1628,10 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int char* pStart = pCol->offset[j] + pCol->pData; int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(p)); - ASSERT(len <= bytes); - ASSERT((p + len) < (pResultInfo->convertBuf[i] + colLength[i])); + if(len > bytes || (p + len) >= (pResultInfo->convertBuf[i] + colLength[i])){ + tscError("doConvertUCS4 error, invalid data. len:%d, bytes:%d, (p + len):%p, (pResultInfo->convertBuf[i] + colLength[i]):%p", len, bytes, (p + len), (pResultInfo->convertBuf[i] + colLength[i])); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } varDataSetLen(p, len); pCol->offset[j] = (p - pResultInfo->convertBuf[i]); @@ -1643,9 +1648,6 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int } int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) { - int32_t cols = *(int32_t*)(p + sizeof(int32_t) * 3); - ASSERT(numOfCols == cols); - return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) * 3 + sizeof(uint64_t) + numOfCols * (sizeof(int8_t) + sizeof(int32_t)); } @@ -1655,6 +1657,12 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column // length | + int32_t cols = *(int32_t*)(p + sizeof(int32_t) * 3); + if(ASSERT(numOfCols == cols)){ + tscError("estimateJsonLen error: numOfCols:%d != cols:%d", numOfCols, cols); + return -1; + } + int32_t len = getVersion1BlockMetaSize(p, numOfCols); int32_t* colLength = (int32_t*)(p + len); len += sizeof(int32_t) * numOfCols; @@ -1688,7 +1696,8 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) { len += (VARSTR_HEADER_SIZE + 5); } else { - ASSERT(0); + tscError("estimateJsonLen error: invalid type:%d", jsonInnerType); + return -1; } } } else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { @@ -1722,12 +1731,21 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int char* p = (char*)pResultInfo->pData; int32_t dataLen = estimateJsonLen(pResultInfo, numOfCols, numOfRows); + if(dataLen <= 0){ + return TSDB_CODE_TSC_INTERNAL_ERROR; + } pResultInfo->convertJson = taosMemoryCalloc(1, dataLen); if (pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; char* p1 = pResultInfo->convertJson; int32_t totalLen = 0; + int32_t cols = *(int32_t*)(p + sizeof(int32_t) * 3); + if(ASSERT(numOfCols == cols)){ + tscError("doConvertJson error: numOfCols:%d != cols:%d", numOfCols, cols); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } + int32_t len = getVersion1BlockMetaSize(p, numOfCols); memcpy(p1, p, len); @@ -1748,8 +1766,10 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int for (int32_t i = 0; i < numOfCols; ++i) { int32_t colLen = htonl(colLength[i]); int32_t colLen1 = htonl(colLength1[i]); - ASSERT(colLen < dataLen); - + if(ASSERT(colLen < dataLen)){ + tscError("doConvertJson error: colLen:%d >= dataLen:%d", colLen, dataLen); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) { int32_t* offset = (int32_t*)pStart; int32_t* offset1 = (int32_t*)pStart1; @@ -1794,7 +1814,8 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false"); varDataSetLen(dst, strlen(varDataVal(dst))); } else { - ASSERT(0); + tscError("doConvertJson error: invalid type:%d", jsonInnerType); + return TSDB_CODE_TSC_INTERNAL_ERROR; } offset1[j] = len; @@ -1832,7 +1853,10 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows, bool convertUcs4) { - assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL); + if(ASSERT(numOfCols > 0 && pFields != NULL && pResultInfo != NULL)){ + tscError("setResultDataPtr paras error"); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } if (numOfRows == 0) { return TSDB_CODE_SUCCESS; } @@ -1861,7 +1885,10 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 int32_t cols = *(int32_t*)p; p += sizeof(int32_t); - ASSERT(rows == numOfRows && cols == numOfCols); + if(ASSERT(rows == numOfRows && cols == numOfCols)){ + tscError("setResultDataPtr paras error:rows;%d numOfRows:%d cols:%d numOfCols:%d", rows, numOfRows, cols, numOfCols); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } int32_t hasColumnSeg = *(int32_t*)p; p += sizeof(int32_t); @@ -1888,7 +1915,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 colLength[i] = htonl(colLength[i]); if (colLength[i] >= dataLen) { tscError("invalid colLength %d, dataLen %d", colLength[i], dataLen); - ASSERT(0); + return TSDB_CODE_TSC_INTERNAL_ERROR; } if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { @@ -1926,7 +1953,11 @@ char* getDbOfConnection(STscObj* pObj) { } void setConnectionDB(STscObj* pTscObj, const char* db) { - assert(db != NULL && pTscObj != NULL); + if(db == NULL || pTscObj == NULL){ + tscError("setConnectionDB para is NULL"); + return; + } + taosThreadMutexLock(&pTscObj->mutex); tstrncpy(pTscObj->db, db, tListLen(pTscObj->db)); taosThreadMutexUnlock(&pTscObj->mutex); @@ -1944,7 +1975,10 @@ void resetConnectDB(STscObj* pTscObj) { int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4, bool freeAfterUse) { - assert(pResultInfo != NULL && pRsp != NULL); + if(pResultInfo == NULL || pRsp == NULL){ + tscError("setQueryResultFromRsp paras is null"); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } if (freeAfterUse) taosMemoryFreeClear(pResultInfo->pRspMsg); diff --git a/source/client/src/clientJniConnector.c b/source/client/src/clientJniConnector.c index 859d4ec80ff6e954259415dabaf1761eb3e7129e..750ba684f42c2f35cd6f5180e6d1d722ad3e0e16 100644 --- a/source/client/src/clientJniConnector.c +++ b/source/client/src/clientJniConnector.c @@ -574,7 +574,11 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp(JNI TAOS_RES *tres = (TAOS_RES *)res; int32_t numOfFields = taos_num_fields(tres); - assert(numOfFields > 0); + if(numOfFields <= 0){ + jniError("jobj:%p, conn:%p, query interrupted. taos_num_fields error code:%d, msg:%s", jobj, tscon, numOfFields, + taos_errstr(tres)); + return JNI_RESULT_SET_NULL; + } void *data; int32_t numOfRows; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 87f5e5fa40ff06efb0983689767f3b4f582be55b..1179f4d23a024c5657e6cc082143a919bcb7683c 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -55,6 +55,8 @@ void taos_cleanup(void) { return; } + tscStopCrashReport(); + int32_t id = clientReqRefPool; clientReqRefPool = -1; taosCloseRef(id); @@ -106,7 +108,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha if (pass == NULL) { pass = TSDB_DEFAULT_PASS; } - + STscObj *pObj = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY); if (pObj) { int64_t *rid = taosMemoryCalloc(1, sizeof(int64_t)); @@ -291,7 +293,6 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { tscError("invalid result passed to taos_fetch_row"); return NULL; } - return NULL; } int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { @@ -355,9 +356,13 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) case TSDB_DATA_TYPE_NCHAR: { int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); if (fields[i].type == TSDB_DATA_TYPE_BINARY) { - assert(charLen <= fields[i].bytes && charLen >= 0); + if(ASSERT(charLen <= fields[i].bytes && charLen >= 0)){ + tscError("taos_print_row error binary. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes); + } } else { - assert(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0); + if(ASSERT(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0)){ + tscError("taos_print_row error. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes); + } } memcpy(str + len, row[i], charLen); @@ -430,6 +435,22 @@ const char *taos_data_type(int type) { return "TSDB_DATA_TYPE_NCHAR"; case TSDB_DATA_TYPE_JSON: return "TSDB_DATA_TYPE_JSON"; + case TSDB_DATA_TYPE_UTINYINT: + return "TSDB_DATA_TYPE_UTINYINT"; + case TSDB_DATA_TYPE_USMALLINT: + return "TSDB_DATA_TYPE_USMALLINT"; + case TSDB_DATA_TYPE_UINT: + return "TSDB_DATA_TYPE_UINT"; + case TSDB_DATA_TYPE_UBIGINT: + return "TSDB_DATA_TYPE_UBIGINT"; + case TSDB_DATA_TYPE_VARBINARY: + return "TSDB_DATA_TYPE_VARBINARY"; + case TSDB_DATA_TYPE_DECIMAL: + return "TSDB_DATA_TYPE_DECIMAL"; + case TSDB_DATA_TYPE_BLOB: + return "TSDB_DATA_TYPE_BLOB"; + case TSDB_DATA_TYPE_MEDIUMBLOB: + return "TSDB_DATA_TYPE_MEDIUMBLOB"; default: return "UNKNOWN"; } @@ -577,7 +598,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) { (*numOfRows) = pResultInfo->numOfRows; return 0; } else { - ASSERT(0); + tscError("taos_fetch_block_s invalid res type"); return -1; } } @@ -670,6 +691,32 @@ const char *taos_get_server_info(TAOS *taos) { return pTscObj->sDetailVer; } +int taos_get_current_db(TAOS *taos, char *database, int len, int *required) { + STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); + if (pTscObj == NULL) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return -1; + } + + int code = TSDB_CODE_SUCCESS; + taosThreadMutexLock(&pTscObj->mutex); + if(database == NULL || len <= 0){ + if(required != NULL) *required = strlen(pTscObj->db) + 1; + terrno = TSDB_CODE_INVALID_PARA; + code = -1; + }else if(len < strlen(pTscObj->db) + 1){ + tstrncpy(database, pTscObj->db, len); + if(required) *required = strlen(pTscObj->db) + 1; + terrno = TSDB_CODE_INVALID_PARA; + code = -1; + }else{ + strcpy(database, pTscObj->db); + code = 0; + } + taosThreadMutexUnlock(&pTscObj->mutex); + return code; +} + static void destoryTablesReq(void *p) { STablesReq *pRes = (STablesReq *)p; taosArrayDestroy(pRes->pTables); @@ -802,7 +849,7 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c tstrerror(code)); if (code == TSDB_CODE_SUCCESS) { - //pWrapper->pCatalogReq->forceUpdate = false; + // pWrapper->pCatalogReq->forceUpdate = false; code = qContinueParseSql(pWrapper->pParseCtx, pWrapper->pCatalogReq, pResultMeta, pQuery); } @@ -831,8 +878,8 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest) tstrerror(code), pWrapper->pRequest->requestId); destorySqlCallbackWrapper(pWrapper); terrno = code; - pWrapper->pRequest->code = code; - pWrapper->pRequest->body.queryFp(pWrapper->pRequest->body.param, pWrapper->pRequest, code); + pRequest->code = code; + pRequest->body.queryFp(pRequest->body.param, pRequest, code); } } @@ -866,7 +913,6 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) { .pTransporter = pTscObj->pAppInfo->pTransporter, .pStmtCb = NULL, .pUser = pTscObj->user, - .schemalessType = pTscObj->schemalessType, .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)), .enableSysInfo = pTscObj->sysInfo, .async = true, @@ -1001,8 +1047,14 @@ static void fetchCallback(void *pResult, void *param, int32_t code) { } void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { - ASSERT(res != NULL && fp != NULL); - ASSERT(TD_RES_QUERY(res)); + if(ASSERT(res != NULL && fp != NULL)){ + tscError("taos_fetch_rows_a invalid paras"); + return; + } + if(ASSERT(TD_RES_QUERY(res))){ + tscError("taos_fetch_rows_a res is NULL"); + return; + } SRequestObj *pRequest = res; pRequest->body.fetchFp = fp; @@ -1045,9 +1097,14 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { } void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { - ASSERT(res != NULL && fp != NULL); - ASSERT(TD_RES_QUERY(res)); - + if(ASSERT(res != NULL && fp != NULL)){ + tscError("taos_fetch_rows_a invalid paras"); + return; + } + if(ASSERT(TD_RES_QUERY(res))){ + tscError("taos_fetch_rows_a res is NULL"); + return; + } SRequestObj *pRequest = res; SReqResultInfo *pResultInfo = &pRequest->body.resInfo; @@ -1059,8 +1116,14 @@ void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { } const void *taos_get_raw_block(TAOS_RES *res) { - ASSERT(res != NULL); - ASSERT(TD_RES_QUERY(res)); + if(ASSERT(res != NULL)){ + tscError("taos_fetch_rows_a invalid paras"); + return NULL; + } + if(ASSERT(TD_RES_QUERY(res))){ + tscError("taos_fetch_rows_a res is NULL"); + return NULL; + } SRequestObj *pRequest = res; return pRequest->body.resInfo.pData; @@ -1168,6 +1231,54 @@ _return: return code; } +int taos_get_tables_vgId(TAOS *taos, const char *db, const char *table[], int tableNum, int *vgId) { + if (NULL == taos) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return terrno; + } + + if (NULL == db || NULL == table || NULL == vgId || tableNum <= 0) { + tscError("invalid input param, db:%p, table:%p, vgId:%p, tbNum:%d", db, table, vgId, tableNum); + terrno = TSDB_CODE_TSC_INVALID_INPUT; + return terrno; + } + + int64_t connId = *(int64_t *)taos; + SRequestObj *pRequest = NULL; + char *sql = "taos_get_table_vgId"; + int32_t code = buildRequest(connId, sql, strlen(sql), NULL, false, &pRequest, 0); + if (code != TSDB_CODE_SUCCESS) { + return terrno; + } + + pRequest->syncQuery = true; + + STscObj *pTscObj = pRequest->pTscObj; + SCatalog *pCtg = NULL; + code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCtg); + if (code != TSDB_CODE_SUCCESS) { + goto _return; + } + + SRequestConnInfo conn = { + .pTrans = pTscObj->pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self}; + + conn.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + + code = catalogGetTablesHashVgId(pCtg, &conn, pTscObj->acctId, db, table, tableNum, vgId); + if (code) { + goto _return; + } + +_return: + + terrno = code; + + destroyRequest(pRequest); + return code; +} + + int taos_load_table_info(TAOS *taos, const char *tableNameList) { if (NULL == taos) { terrno = TSDB_CODE_TSC_DISCONNECTED; @@ -1331,6 +1442,13 @@ int taos_stmt_get_col_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fiel return stmtGetColFields(stmt, fieldNum, fields); } +// let stmt to reclaim TAOS_FIELD_E that was allocated by `taos_stmt_get_tag_fields`/`taos_stmt_get_col_fields` +void taos_stmt_reclaim_fields(TAOS_STMT *stmt, TAOS_FIELD_E *fields) { + (void)stmt; + if (!fields) return; + taosMemoryFree(fields); +} + int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { if (stmt == NULL || bind == NULL) { tscError("NULL parameter for %s", __FUNCTION__); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 85027ff371c921cc6bf9137d4f7eaece092c2e7d..2191c54315d6324e819337fa41db3875e1506594 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -119,6 +119,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { // update the appInstInfo pTscObj->pAppInfo->clusterId = connectRsp.clusterId; + lastClusterId = connectRsp.clusterId; pTscObj->connType = connectRsp.connType; @@ -149,7 +150,6 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pRequest) { pMsgSendInfo->msgType = pRequest->type; pMsgSendInfo->target.type = TARGET_TYPE_MNODE; - assert(pRequest != NULL); pMsgSendInfo->msgInfo = pRequest->body.requestMsg; pMsgSendInfo->fp = getMsgRspHandle(pRequest->type); return pMsgSendInfo; @@ -273,7 +273,9 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { } int32_t processCreateSTableRsp(void* param, SDataBuf* pMsg, int32_t code) { - assert(pMsg != NULL && param != NULL); + if(pMsg == NULL || param == NULL){ + return TSDB_CODE_TSC_INVALID_INPUT; + } SRequestObj* pRequest = param; if (code != TSDB_CODE_SUCCESS) { @@ -454,7 +456,10 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS); - ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); + if(len != rspSize - sizeof(SRetrieveTableRsp)){ + uError("buildShowVariablesRsp error, len:%d != rspSize - sizeof(SRetrieveTableRsp):%" PRIu64, len, (uint64_t) (rspSize - sizeof(SRetrieveTableRsp))); + return TSDB_CODE_TSC_INVALID_INPUT; + } blockDataDestroy(pBlock); return TSDB_CODE_SUCCESS; diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 1c0bef9e928ba98bec1f54bf3f6ed27dacb7a580..3fd6fed4fceb19f52026ab19ca1733b1b0a86a2e 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -373,7 +373,10 @@ static char* processCreateTable(SMqMetaRsp* metaRsp) { } static char* processAutoCreateTable(STaosxRsp* rsp) { - ASSERT(rsp->createTableNum != 0); + if(rsp->createTableNum <= 0){ + uError("WriteRaw:processAutoCreateTable rsp->createTableNum <= 0"); + goto _exit; + } SDecoder* decoder = taosMemoryCalloc(rsp->createTableNum, sizeof(SDecoder)); SVCreateTbReq* pCreateReq = taosMemoryCalloc(rsp->createTableNum, sizeof(SVCreateTbReq)); @@ -389,7 +392,10 @@ static char* processAutoCreateTable(STaosxRsp* rsp) { goto _exit; } - ASSERT(pCreateReq[iReq].type == TSDB_CHILD_TABLE); + if(pCreateReq[iReq].type != TSDB_CHILD_TABLE){ + uError("WriteRaw:processAutoCreateTable pCreateReq[iReq].type != TSDB_CHILD_TABLE"); + goto _exit; + } } string = buildCreateCTableJson(pCreateReq, rsp->createTableNum); @@ -494,7 +500,10 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { char* buf = NULL; if (vAlterTbReq.tagType == TSDB_DATA_TYPE_JSON) { - ASSERT(tTagIsJson(vAlterTbReq.pTagVal) == true); + if(!tTagIsJson(vAlterTbReq.pTagVal)){ + uError("processAlterTable isJson false"); + goto _exit; + } buf = parseTagDatatoJson(vAlterTbReq.pTagVal); } else { buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 1, 1); @@ -1240,22 +1249,12 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { return code; } -typedef struct { - SVgroupInfo vg; - void* data; -} VgData; - -static void destroyVgHash(void* data) { - VgData* vgData = (VgData*)data; - taosMemoryFreeClear(vgData->data); -} - int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD* fields, int numFields) { int32_t code = TSDB_CODE_SUCCESS; STableMeta* pTableMeta = NULL; SQuery* pQuery = NULL; - SSubmitReq* subReq = NULL; + SHashObj* pVgHash = NULL; SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); if (!pRequest) { @@ -1300,149 +1299,25 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbname); goto end; } - uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); - uint64_t uid = pTableMeta->uid; - int32_t numOfCols = pTableMeta->tableInfo.numOfColumns; - - uint16_t fLen = 0; - int32_t rowSize = 0; - int16_t nVar = 0; - for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - SSchema* schema = pTableMeta->schema + i; - fLen += TYPE_BYTES[schema->type]; - rowSize += schema->bytes; - if (IS_VAR_DATA_TYPE(schema->type)) { - nVar++; - } - } - - fLen -= sizeof(TSKEY); - - int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(numOfCols - 1); - int32_t schemaLen = 0; - int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; - - int32_t totalLen = sizeof(SSubmitReq) + submitLen; - subReq = taosMemoryCalloc(1, totalLen); - SSubmitBlk* blk = POINTER_SHIFT(subReq, sizeof(SSubmitReq)); - void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); - STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTableMeta->sversion); - tdSRowSetTpInfo(&rb, numOfCols, fLen); - int32_t dataLen = 0; - - // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column - // length | - char* pStart = pData + getVersion1BlockMetaSize(pData, numFields); - int32_t* colLength = (int32_t*)pStart; - pStart += sizeof(int32_t) * numFields; - - SResultColumn* pCol = taosMemoryCalloc(numFields, sizeof(SResultColumn)); - - for (int32_t i = 0; i < numFields; ++i) { - if (IS_VAR_DATA_TYPE(fields[i].type)) { - pCol[i].offset = (int32_t*)pStart; - pStart += rows * sizeof(int32_t); - } else { - pCol[i].nullbitmap = pStart; - pStart += BitmapLen(rows); - } - - pCol[i].pData = pStart; - pStart += colLength[i]; - } - - SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - for (int i = 0; i < numFields; i++) { - TAOS_FIELD* schema = &fields[i]; - taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); - } - - for (int32_t j = 0; j < rows; j++) { - tdSRowResetBuf(&rb, rowData); - int32_t offset = 0; - for (int32_t k = 0; k < numOfCols; k++) { - const SSchema* pColumn = &pTableMeta->schema[k]; - int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); - if (!index) { // add none - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k); - } else { - if (IS_VAR_DATA_TYPE(pColumn->type)) { - if (pCol[*index].offset[j] != -1) { - char* data = pCol[*index].pData + pCol[*index].offset[j]; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); - } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } - } else { - if (!colDataIsNull_f(pCol[*index].nullbitmap, j)) { - char* data = pCol[*index].pData + pColumn->bytes * j; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); - } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } - } - } - - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } - } - tdSRowEnd(&rb); - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - dataLen += rowLen; - } - - taosHashCleanup(schemaHash); - taosMemoryFree(pCol); - - blk->uid = htobe64(uid); - blk->suid = htobe64(suid); - blk->sversion = htonl(pTableMeta->sversion); - blk->schemaLen = htonl(schemaLen); - blk->numOfRows = htonl(rows); - blk->dataLen = htonl(dataLen); - subReq->length = sizeof(SSubmitReq) + sizeof(SSubmitBlk) + schemaLen + dataLen; - subReq->numOfBlocks = 1; - - pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == pQuery) { - uError("create SQuery error"); + pQuery = smlInitHandle(); + if (pQuery == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto end; } - pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->haveResultSet = false; - pQuery->msgType = TDMT_VND_SUBMIT; - pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT); - if (NULL == pQuery->pRoot) { - uError("create pQuery->pRoot error"); - code = TSDB_CODE_OUT_OF_MEMORY; + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)); + + code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:rawBlockBindData failed"); goto end; } - SVnodeModifyOpStmt* nodeStmt = (SVnodeModifyOpStmt*)(pQuery->pRoot); - nodeStmt->pDataBlocks = taosArrayInit(1, POINTER_BYTES); - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; + code = smlBuildOutput(pQuery, pVgHash); + if (code != TSDB_CODE_SUCCESS) { + uError("smlBuildOutput failed"); + return code; } - dst->vg = vgData; - dst->numOfTables = subReq->numOfBlocks; - dst->size = subReq->length; - dst->pData = (char*)subReq; - subReq->header.vgId = htonl(dst->vg.vgId); - subReq->version = htonl(1); - subReq->header.contLen = htonl(subReq->length); - subReq->length = htonl(subReq->length); - subReq->numOfBlocks = htonl(subReq->numOfBlocks); - subReq = NULL; // no need free - taosArrayPush(nodeStmt->pDataBlocks, &dst); launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; @@ -1450,7 +1325,8 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch end: taosMemoryFreeClear(pTableMeta); qDestroyQuery(pQuery); - taosMemoryFree(subReq); + destroyRequest(pRequest); + taosHashCleanup(pVgHash); return code; } @@ -1458,7 +1334,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) int32_t code = TSDB_CODE_SUCCESS; STableMeta* pTableMeta = NULL; SQuery* pQuery = NULL; - SSubmitReq* subReq = NULL; + SHashObj* pVgHash = NULL; SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); if (!pRequest) { @@ -1503,138 +1379,25 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbname); goto end; } - uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); - uint64_t uid = pTableMeta->uid; - int32_t numOfCols = pTableMeta->tableInfo.numOfColumns; - - uint16_t fLen = 0; - int32_t rowSize = 0; - int16_t nVar = 0; - for (int i = 0; i < numOfCols; i++) { - SSchema* schema = pTableMeta->schema + i; - fLen += TYPE_BYTES[schema->type]; - rowSize += schema->bytes; - if (IS_VAR_DATA_TYPE(schema->type)) { - nVar++; - } - } - fLen -= sizeof(TSKEY); - - int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(numOfCols - 1); - int32_t schemaLen = 0; - int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; - - int32_t totalLen = sizeof(SSubmitReq) + submitLen; - subReq = taosMemoryCalloc(1, totalLen); - SSubmitBlk* blk = POINTER_SHIFT(subReq, sizeof(SSubmitReq)); - void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); - STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTableMeta->sversion); - tdSRowSetTpInfo(&rb, numOfCols, fLen); - int32_t dataLen = 0; - - // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column - // length | - char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols); - int32_t* colLength = (int32_t*)pStart; - pStart += sizeof(int32_t) * numOfCols; - - SResultColumn* pCol = taosMemoryCalloc(numOfCols, sizeof(SResultColumn)); - - for (int32_t i = 0; i < numOfCols; ++i) { - if (IS_VAR_DATA_TYPE(pTableMeta->schema[i].type)) { - pCol[i].offset = (int32_t*)pStart; - pStart += rows * sizeof(int32_t); - } else { - pCol[i].nullbitmap = pStart; - pStart += BitmapLen(rows); - } - - pCol[i].pData = pStart; - pStart += colLength[i]; - } - - for (int32_t j = 0; j < rows; j++) { - tdSRowResetBuf(&rb, rowData); - int32_t offset = 0; - for (int32_t k = 0; k < numOfCols; k++) { - const SSchema* pColumn = &pTableMeta->schema[k]; - - if (IS_VAR_DATA_TYPE(pColumn->type)) { - if (pCol[k].offset[j] != -1) { - char* data = pCol[k].pData + pCol[k].offset[j]; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); - } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } - } else { - if (!colDataIsNull_f(pCol[k].nullbitmap, j)) { - char* data = pCol[k].pData + pColumn->bytes * j; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); - } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } - } - - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } - } - tdSRowEnd(&rb); - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - dataLen += rowLen; - } - - taosMemoryFree(pCol); - - blk->uid = htobe64(uid); - blk->suid = htobe64(suid); - blk->sversion = htonl(pTableMeta->sversion); - blk->schemaLen = htonl(schemaLen); - blk->numOfRows = htonl(rows); - blk->dataLen = htonl(dataLen); - subReq->length = sizeof(SSubmitReq) + sizeof(SSubmitBlk) + schemaLen + dataLen; - subReq->numOfBlocks = 1; - - pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == pQuery) { - uError("create SQuery error"); + pQuery = smlInitHandle(); + if (pQuery == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(subReq); goto end; } - pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->haveResultSet = false; - pQuery->msgType = TDMT_VND_SUBMIT; - pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT); - if (NULL == pQuery->pRoot) { - uError("create pQuery->pRoot error"); - code = TSDB_CODE_OUT_OF_MEMORY; + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)); + + code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:rawBlockBindData failed"); goto end; } - SVnodeModifyOpStmt* nodeStmt = (SVnodeModifyOpStmt*)(pQuery->pRoot); - nodeStmt->pDataBlocks = taosArrayInit(1, POINTER_BYTES); - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; + code = smlBuildOutput(pQuery, pVgHash); + if (code != TSDB_CODE_SUCCESS) { + uError("smlBuildOutput failed"); + return code; } - dst->vg = vgData; - dst->numOfTables = subReq->numOfBlocks; - dst->size = subReq->length; - dst->pData = (char*)subReq; - subReq->header.vgId = htonl(dst->vg.vgId); - subReq->version = htonl(1); - subReq->header.contLen = htonl(subReq->length); - subReq->length = htonl(subReq->length); - subReq->numOfBlocks = htonl(subReq->numOfBlocks); - subReq = NULL; // no need free - taosArrayPush(nodeStmt->pDataBlocks, &dst); launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; @@ -1642,7 +1405,8 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) end: taosMemoryFreeClear(pTableMeta); qDestroyQuery(pQuery); - taosMemoryFree(subReq); + destroyRequest(pRequest); + taosHashCleanup(pVgHash); return code; } @@ -1679,8 +1443,6 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { goto end; } - pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - taosHashSetFreeFp(pVgHash, destroyVgHash); struct SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { @@ -1694,6 +1456,12 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { conn.requestObjRefId = pRequest->self; conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + pQuery = smlInitHandle(); + if (pQuery == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); uDebug("raw data block num:%d\n", rspObj.rsp.blockNum); while (++rspObj.resIter < rspObj.rsp.blockNum) { SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); @@ -1701,14 +1469,6 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { uError("WriteRaw:no schema, iter:%d", rspObj.resIter); goto end; } - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter); - setResSchemaInfo(&rspObj.resInfo, pSW->pSchema, pSW->nCols); - - code = setQueryResultFromRsp(&rspObj.resInfo, pRetrieve, false, false); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw: setQueryResultFromRsp error"); - goto end; - } const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.blockTbName, rspObj.resIter); if (!tbName) { @@ -1722,13 +1482,6 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { strcpy(pName.dbname, pRequest->pDb); strcpy(pName.tname, tbName); - VgData vgData = {0}; - code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &(vgData.vg)); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName); - goto end; - } - code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { uError("WriteRaw:catalogGetTableMeta table not exist. table name: %s", tbName); @@ -1740,164 +1493,29 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { goto end; } - uint16_t fLen = 0; - int32_t rowSize = 0; - int16_t nVar = 0; - for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - SSchema* schema = &pTableMeta->schema[i]; - fLen += TYPE_BYTES[schema->type]; - rowSize += schema->bytes; - if (IS_VAR_DATA_TYPE(schema->type)) { - nVar++; - } - } - fLen -= sizeof(TSKEY); - - int32_t rows = rspObj.resInfo.numOfRows; - int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(pTableMeta->tableInfo.numOfColumns - 1); - int32_t schemaLen = 0; - int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; - - SSubmitReq* subReq = NULL; - SSubmitBlk* blk = NULL; - void* hData = taosHashGet(pVgHash, &vgData.vg.vgId, sizeof(vgData.vg.vgId)); - if (hData) { - vgData = *(VgData*)hData; - - int32_t totalLen = ((SSubmitReq*)(vgData.data))->length + submitLen; - void* tmp = taosMemoryRealloc(vgData.data, totalLen); - if (tmp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - vgData.data = tmp; - ((VgData*)hData)->data = tmp; - subReq = (SSubmitReq*)(vgData.data); - blk = POINTER_SHIFT(vgData.data, subReq->length); - } else { - int32_t totalLen = sizeof(SSubmitReq) + submitLen; - void* tmp = taosMemoryCalloc(1, totalLen); - if (tmp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - vgData.data = tmp; - taosHashPut(pVgHash, (const char*)&vgData.vg.vgId, sizeof(vgData.vg.vgId), (char*)&vgData, sizeof(vgData)); - subReq = (SSubmitReq*)(vgData.data); - subReq->length = sizeof(SSubmitReq); - subReq->numOfBlocks = 0; - - blk = POINTER_SHIFT(vgData.data, sizeof(SSubmitReq)); + SVgroupInfo vg; + code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName); + goto end; } - // pSW->pSchema should be same as pTableMeta->schema - // ASSERT(pSW->nCols == pTableMeta->tableInfo.numOfColumns); - uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); - uint64_t uid = pTableMeta->uid; - int16_t sver = pTableMeta->sversion; - - void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); - STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - - SRowBuilder rb = {0}; - tdSRowInit(&rb, sver); - tdSRowSetTpInfo(&rb, pTableMeta->tableInfo.numOfColumns, fLen); - int32_t totalLen = 0; - - SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - for (int i = 0; i < pSW->nCols; i++) { - SSchema* schema = &pSW->pSchema[i]; - taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); + void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId)); + if (hData == NULL) { + taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); } - for (int32_t j = 0; j < rows; j++) { - tdSRowResetBuf(&rb, rowData); - - doSetOneRowPtr(&rspObj.resInfo); - rspObj.resInfo.current += 1; - - int32_t offset = 0; - for (int32_t k = 0; k < pTableMeta->tableInfo.numOfColumns; k++) { - const SSchema* pColumn = &pTableMeta->schema[k]; - int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); - if (!index) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k); - } else { - char* colData = rspObj.resInfo.row[*index]; - if (!colData) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } else { - if (IS_VAR_DATA_TYPE(pColumn->type)) { - colData -= VARSTR_HEADER_SIZE; - } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); - } - } - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } - } - tdSRowEnd(&rb); - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - totalLen += rowLen; + code = rawBlockBindData(pQuery, pTableMeta, pRetrieve->data, NULL, NULL, 0); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:rawBlockBindData failed"); + goto end; } - - taosHashCleanup(schemaHash); - blk->uid = htobe64(uid); - blk->suid = htobe64(suid); - blk->sversion = htonl(sver); - blk->schemaLen = htonl(schemaLen); - blk->numOfRows = htonl(rows); - blk->dataLen = htonl(totalLen); - subReq->length += sizeof(SSubmitBlk) + schemaLen + totalLen; - subReq->numOfBlocks++; - taosMemoryFreeClear(pTableMeta); - rspObj.resInfo.pRspMsg = NULL; - doFreeReqResultInfo(&rspObj.resInfo); - } - - pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == pQuery) { - uError("create SQuery error"); - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->haveResultSet = false; - pQuery->msgType = TDMT_VND_SUBMIT; - pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT); - if (NULL == pQuery->pRoot) { - uError("create pQuery->pRoot error"); - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; } - SVnodeModifyOpStmt* nodeStmt = (SVnodeModifyOpStmt*)(pQuery->pRoot); - - int32_t numOfVg = taosHashGetSize(pVgHash); - nodeStmt->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); - VgData* vData = (VgData*)taosHashIterate(pVgHash, NULL); - while (vData) { - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - dst->vg = vData->vg; - SSubmitReq* subReq = (SSubmitReq*)(vData->data); - dst->numOfTables = subReq->numOfBlocks; - dst->size = subReq->length; - dst->pData = (char*)subReq; - vData->data = NULL; // no need free - subReq->header.vgId = htonl(dst->vg.vgId); - subReq->version = htonl(1); - subReq->header.contLen = htonl(subReq->length); - subReq->length = htonl(subReq->length); - subReq->numOfBlocks = htonl(subReq->numOfBlocks); - taosArrayPush(nodeStmt->pDataBlocks, &dst); - vData = (VgData*)taosHashIterate(pVgHash, vData); + code = smlBuildOutput(pQuery, pVgHash); + if (code != TSDB_CODE_SUCCESS) { + uError("smlBuildOutput failed"); + return code; } launchQueryImpl(pRequest, pQuery, true, NULL); @@ -1905,8 +1523,6 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { end: tDeleteSMqDataRsp(&rspObj.rsp); - rspObj.resInfo.pRspMsg = NULL; - doFreeReqResultInfo(&rspObj.resInfo); tDecoderClear(&decoder); qDestroyQuery(pQuery); destroyRequest(pRequest); @@ -1948,8 +1564,6 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) goto end; } - pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - taosHashSetFreeFp(pVgHash, destroyVgHash); struct SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { @@ -1963,6 +1577,13 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) conn.requestObjRefId = pRequest->self; conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + pQuery = smlInitHandle(); + if (pQuery == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + uDebug("raw data block num:%d\n", rspObj.rsp.blockNum); while (++rspObj.resIter < rspObj.rsp.blockNum) { SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); @@ -1970,14 +1591,6 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) uError("WriteRaw:no schema, iter:%d", rspObj.resIter); goto end; } - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter); - setResSchemaInfo(&rspObj.resInfo, pSW->pSchema, pSW->nCols); - - code = setQueryResultFromRsp(&rspObj.resInfo, pRetrieve, false, false); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw: setQueryResultFromRsp error"); - goto end; - } const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.blockTbName, rspObj.resIter); if (!tbName) { @@ -1991,44 +1604,32 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) strcpy(pName.dbname, pRequest->pDb); strcpy(pName.tname, tbName); - VgData vgData = {0}; - code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &(vgData.vg)); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName); - goto end; - } - // find schema data info - int32_t schemaLen = 0; - void* schemaData = NULL; + SVCreateTbReq pCreateReq = {0}; + for (int j = 0; j < rspObj.rsp.createTableNum; j++) { void** dataTmp = taosArrayGet(rspObj.rsp.createTableReq, j); int32_t* lenTmp = taosArrayGet(rspObj.rsp.createTableLen, j); - SDecoder decoderTmp = {0}; - SVCreateTbReq pCreateReq = {0}; - + SDecoder decoderTmp = {0}; tDecoderInit(&decoderTmp, *dataTmp, *lenTmp); + memset(&pCreateReq, 0, sizeof(SVCreateTbReq)); if (tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq) < 0) { tDecoderClear(&decoderTmp); - taosMemoryFreeClear(pCreateReq.comment); - taosArrayDestroy(pCreateReq.ctb.tagName); goto end; } - ASSERT(pCreateReq.type == TSDB_CHILD_TABLE); + if(pCreateReq.type != TSDB_CHILD_TABLE){ + uError("WriteRaw:pCreateReq.type != TSDB_CHILD_TABLE. table name: %s", tbName); + code = TSDB_CODE_TSC_INVALID_VALUE; + goto end; + } if (strcmp(tbName, pCreateReq.name) == 0) { - schemaLen = *lenTmp; - schemaData = *dataTmp; strcpy(pName.tname, pCreateReq.ctb.stbName); tDecoderClear(&decoderTmp); - taosMemoryFreeClear(pCreateReq.comment); - taosArrayDestroy(pCreateReq.ctb.tagName); break; } tDecoderClear(&decoderTmp); - taosMemoryFreeClear(pCreateReq.comment); - taosArrayDestroy(pCreateReq.ctb.tagName); } code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); @@ -2042,167 +1643,23 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) goto end; } - uint16_t fLen = 0; - int32_t rowSize = 0; - int16_t nVar = 0; - for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - SSchema* schema = &pTableMeta->schema[i]; - fLen += TYPE_BYTES[schema->type]; - rowSize += schema->bytes; - if (IS_VAR_DATA_TYPE(schema->type)) { - nVar++; - } - } - fLen -= sizeof(TSKEY); - - int32_t rows = rspObj.resInfo.numOfRows; - int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(pTableMeta->tableInfo.numOfColumns - 1); - - int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; - - SSubmitReq* subReq = NULL; - SSubmitBlk* blk = NULL; - void* hData = taosHashGet(pVgHash, &vgData.vg.vgId, sizeof(vgData.vg.vgId)); - if (hData) { - vgData = *(VgData*)hData; - - int32_t totalLen = ((SSubmitReq*)(vgData.data))->length + submitLen; - void* tmp = taosMemoryRealloc(vgData.data, totalLen); - if (tmp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - vgData.data = tmp; - ((VgData*)hData)->data = tmp; - subReq = (SSubmitReq*)(vgData.data); - blk = POINTER_SHIFT(vgData.data, subReq->length); - } else { - int32_t totalLen = sizeof(SSubmitReq) + submitLen; - void* tmp = taosMemoryCalloc(1, totalLen); - if (tmp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - vgData.data = tmp; - taosHashPut(pVgHash, (const char*)&vgData.vg.vgId, sizeof(vgData.vg.vgId), (char*)&vgData, sizeof(vgData)); - subReq = (SSubmitReq*)(vgData.data); - subReq->length = sizeof(SSubmitReq); - subReq->numOfBlocks = 0; - - blk = POINTER_SHIFT(vgData.data, sizeof(SSubmitReq)); - } - - // pSW->pSchema should be same as pTableMeta->schema - // ASSERT(pSW->nCols == pTableMeta->tableInfo.numOfColumns); - uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); - uint64_t uid = pTableMeta->uid; - int16_t sver = pTableMeta->sversion; - - void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); - if (schemaData) { - memcpy(blkSchema, schemaData, schemaLen); - } - STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - - SRowBuilder rb = {0}; - tdSRowInit(&rb, sver); - tdSRowSetTpInfo(&rb, pTableMeta->tableInfo.numOfColumns, fLen); - int32_t totalLen = 0; - - SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - for (int i = 0; i < pSW->nCols; i++) { - SSchema* schema = &pSW->pSchema[i]; - taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); + SVgroupInfo vg; + code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName); + goto end; } - for (int32_t j = 0; j < rows; j++) { - tdSRowResetBuf(&rb, rowData); - - doSetOneRowPtr(&rspObj.resInfo); - rspObj.resInfo.current += 1; - - int32_t offset = 0; - for (int32_t k = 0; k < pTableMeta->tableInfo.numOfColumns; k++) { - const SSchema* pColumn = &pTableMeta->schema[k]; - int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); - if (!index) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k); - } else { - char* colData = rspObj.resInfo.row[*index]; - if (!colData) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } else { - if (IS_VAR_DATA_TYPE(pColumn->type)) { - colData -= VARSTR_HEADER_SIZE; - } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); - } - } - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } - } - tdSRowEnd(&rb); - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - totalLen += rowLen; + void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId)); + if (hData == NULL) { + taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); } - taosHashCleanup(schemaHash); - blk->uid = htobe64(uid); - blk->suid = htobe64(suid); - blk->sversion = htonl(sver); - blk->schemaLen = htonl(schemaLen); - blk->numOfRows = htonl(rows); - blk->dataLen = htonl(totalLen); - subReq->length += sizeof(SSubmitBlk) + schemaLen + totalLen; - subReq->numOfBlocks++; - taosMemoryFreeClear(pTableMeta); - rspObj.resInfo.pRspMsg = NULL; - doFreeReqResultInfo(&rspObj.resInfo); - } - - pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == pQuery) { - uError("create SQuery error"); - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->haveResultSet = false; - pQuery->msgType = TDMT_VND_SUBMIT; - pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT); - if (NULL == pQuery->pRoot) { - uError("create pQuery->pRoot error"); - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - SVnodeModifyOpStmt* nodeStmt = (SVnodeModifyOpStmt*)(pQuery->pRoot); - - int32_t numOfVg = taosHashGetSize(pVgHash); - nodeStmt->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); - - VgData* vData = (VgData*)taosHashIterate(pVgHash, NULL); - while (vData) { - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = rawBlockBindData(pQuery, pTableMeta, pRetrieve->data, &pCreateReq, NULL, 0); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:rawBlockBindData failed"); goto end; } - dst->vg = vData->vg; - SSubmitReq* subReq = (SSubmitReq*)(vData->data); - dst->numOfTables = subReq->numOfBlocks; - dst->size = subReq->length; - dst->pData = (char*)subReq; - vData->data = NULL; // no need free - subReq->header.vgId = htonl(dst->vg.vgId); - subReq->version = htonl(1); - subReq->header.contLen = htonl(subReq->length); - subReq->length = htonl(subReq->length); - subReq->numOfBlocks = htonl(subReq->numOfBlocks); - taosArrayPush(nodeStmt->pDataBlocks, &dst); - vData = (VgData*)taosHashIterate(pVgHash, vData); } launchQueryImpl(pRequest, pQuery, true, NULL); @@ -2210,8 +1667,6 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) end: tDeleteSTaosxRsp(&rspObj.rsp); - rspObj.resInfo.pRspMsg = NULL; - doFreeReqResultInfo(&rspObj.resInfo); tDecoderClear(&decoder); qDestroyQuery(pQuery); destroyRequest(pRequest); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 8c77ef664a1b54f3e702b20fbfeb7e05a120b9e8..f25456999b44f1d44db55c9071563b3bc52baecd 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1,219 +1,101 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #include #include #include #include -#include "cJSON.h" -#include "catalog.h" -#include "clientInt.h" -#include "osSemaphore.h" -#include "osThread.h" -#include "query.h" -#include "taos.h" -#include "taoserror.h" -#include "tcommon.h" -#include "tdef.h" -#include "tglobal.h" -#include "tlog.h" -#include "tmsg.h" -#include "tname.h" -#include "ttime.h" -#include "ttypes.h" - -//================================================================================================= - -#define SPACE ' ' -#define COMMA ',' -#define EQUAL '=' -#define QUOTE '"' -#define SLASH '\\' - -#define JUMP_SPACE(sql, sqlEnd) \ - while (sql < sqlEnd) { \ - if (*sql == SPACE) \ - sql++; \ - else \ - break; \ - } -// comma , -#define IS_SLASH_COMMA(sql) (*(sql) == COMMA && *((sql)-1) == SLASH) -#define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH) -// space -#define IS_SLASH_SPACE(sql) (*(sql) == SPACE && *((sql)-1) == SLASH) -#define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH) -// equal = -#define IS_SLASH_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) == SLASH) -#define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH) -// quote " -#define IS_SLASH_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) == SLASH) -#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH) -// SLASH -#define IS_SLASH_SLASH(sql) (*(sql) == SLASH && *((sql)-1) == SLASH) - -#define IS_SLASH_LETTER(sql) \ - (IS_SLASH_COMMA(sql) || IS_SLASH_SPACE(sql) || IS_SLASH_EQUAL(sql) || IS_SLASH_QUOTE(sql) || IS_SLASH_SLASH(sql)) - -#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) - -#define PROCESS_SLASH(key, keyLen) \ - for (int i = 1; i < keyLen; ++i) { \ - if (IS_SLASH_LETTER(key + i)) { \ - MOVE_FORWARD_ONE(key + i, keyLen - i); \ - i--; \ - keyLen--; \ - } \ - } - -#define IS_INVALID_COL_LEN(len) ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN) -#define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN) - -#define OTD_JSON_SUB_FIELDS_NUM 2 -#define OTD_JSON_FIELDS_NUM 4 - -#define TS "_ts" -#define TS_LEN 3 -#define VALUE "_value" -#define VALUE_LEN 6 - -#define BINARY_ADD_LEN 2 // "binary" 2 means " " -#define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" " - -#define MAX_RETRY_TIMES 5 -//================================================================================================= -typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; - -typedef enum { - SCHEMA_ACTION_NULL, - SCHEMA_ACTION_CREATE_STABLE, - SCHEMA_ACTION_ADD_COLUMN, - SCHEMA_ACTION_ADD_TAG, - SCHEMA_ACTION_CHANGE_COLUMN_SIZE, - SCHEMA_ACTION_CHANGE_TAG_SIZE, -} ESchemaAction; - -typedef struct { - const char *measure; - const char *tags; - const char *cols; - const char *timestamp; - - int32_t measureLen; - int32_t measureTagsLen; - int32_t tagsLen; - int32_t colsLen; - int32_t timestampLen; -} SSmlLineInfo; - -typedef struct { - const char *sTableName; // super table name - int32_t sTableNameLen; - char childTableName[TSDB_TABLE_NAME_LEN]; - uint64_t uid; - - SArray *tags; - - // if info->formatData is true, elements are SArray. - // if info->formatData is false, elements are SHashObj for find by key quickly - SArray *cols; -} SSmlTableInfo; - -typedef struct { - SArray *tags; // save the origin order to create table - SHashObj *tagHash; // elements are - - SArray *cols; - SHashObj *colHash; - - STableMeta *tableMeta; -} SSmlSTableMeta; - -typedef struct { - int32_t len; - char *buf; -} SSmlMsgBuf; - -typedef struct { - int32_t code; - int32_t lineNum; - - int32_t numOfSTables; - int32_t numOfCTables; - int32_t numOfCreateSTables; - int32_t numOfAlterColSTables; - int32_t numOfAlterTagSTables; - - int64_t parseTime; - int64_t schemaTime; - int64_t insertBindTime; - int64_t insertRpcTime; - int64_t endTime; -} SSmlCostInfo; - -typedef struct { - SRequestObj *request; - tsem_t sem; - int32_t cnt; - int32_t total; - TdThreadSpinlock lock; -} Params; - -typedef struct { - int64_t id; - Params *params; - - SMLProtocolType protocol; - int8_t precision; - bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) - bool isRawLine; - int32_t ttl; - - SHashObj *childTables; - SHashObj *superTables; - SHashObj *pVgHash; - void *exec; - - STscObj *taos; - SCatalog *pCatalog; - SRequestObj *pRequest; - SQuery *pQuery; - - SSmlCostInfo cost; - int32_t affectedRows; - SSmlMsgBuf msgBuf; - SHashObj *dumplicateKey; // for dumplicate key - SArray *colsContainer; // for cols parse, if dataFormat == false - - cJSON *root; // for parse json -} SSmlHandle; -//================================================================================================= - -//================================================================================================= -static volatile int64_t linesSmlHandleId = 0; -static int64_t smlGenId() { - int64_t id; +#include "clientSml.h" - do { - id = atomic_add_fetch_64(&linesSmlHandleId, 1); - } while (id == 0); +int64_t smlToMilli[3] = {3600000LL, 60000LL, 1000LL}; +int64_t smlFactorNS[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; +int64_t smlFactorS[3] = {1000LL, 1000000LL, 1000000000LL}; - return id; +void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) { + NodeList *tmp = list; + while (tmp) { + if (fn == NULL) { + if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { + return tmp->data.value; + } + } else { + if (tmp->data.used && fn(tmp->data.key, key) == 0) { + return tmp->data.value; + } + } + + tmp = tmp->next; + } + return NULL; } -static inline bool smlDoubleToInt64OverFlow(double num) { - if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; - return false; +int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) { + NodeList *tmp = *list; + while (tmp) { + if (!tmp->data.used) break; + if (fn == NULL) { + if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { + return -1; + } + } else { + if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) { + return -1; + } + } + + tmp = tmp->next; + } + if (tmp) { + tmp->data.key = key; + tmp->data.keyLen = len; + tmp->data.value = value; + tmp->data.used = true; + } else { + NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList)); + if (newNode == NULL) { + return -1; + } + newNode->data.key = key; + newNode->data.keyLen = len; + newNode->data.value = value; + newNode->data.used = true; + newNode->next = *list; + *list = newNode; + } + return 0; } -static inline bool smlCheckDuplicateKey(const char *key, int32_t keyLen, SHashObj *pHash) { - void *val = taosHashGet(pHash, key, keyLen); - if (val) { - return true; +int nodeListSize(NodeList *list) { + int cnt = 0; + while (list) { + if (list->data.used) + cnt++; + else + break; + list = list->next; } - taosHashPut(pHash, key, keyLen, key, 1); + return cnt; +} + +inline bool smlDoubleToInt64OverFlow(double num) { + if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; return false; } -static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { +int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { if (pBuf->buf) { memset(pBuf->buf, 0, pBuf->len); if (msg1) strncat(pBuf->buf, msg1, pBuf->len); @@ -226,391 +108,307 @@ static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const return TSDB_CODE_SML_INVALID_DATA; } -static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSmlKv *kv, bool isTag, - ESchemaAction *action, SSmlHandle *info) { - uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL; - if (index) { - if (colField[*index].type != kv->type) { - uError("SML:0x%" PRIx64 " point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, - kv->key, colField[*index].type, kv->type); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if ((colField[*index].type == TSDB_DATA_TYPE_VARCHAR && - (colField[*index].bytes - VARSTR_HEADER_SIZE) < kv->length) || - (colField[*index].type == TSDB_DATA_TYPE_NCHAR && - ((colField[*index].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE < kv->length))) { - if (isTag) { - *action = SCHEMA_ACTION_CHANGE_TAG_SIZE; - } else { - *action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE; - } - } - } else { - if (isTag) { - *action = SCHEMA_ACTION_ADD_TAG; - } else { - *action = SCHEMA_ACTION_ADD_COLUMN; - } +int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision) { + char *endPtr = NULL; + int64_t tsInt64 = taosStr2Int64(value, &endPtr, 10); + if (unlikely(value + len != endPtr)) { + return -1; } - return 0; -} -static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { - int32_t result = 1; - while (result <= length) { - result *= 2; - } - if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { - result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE; - } else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - result = (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + if (unlikely(fromPrecision >= TSDB_TIME_PRECISION_HOURS)) { + int64_t unit = smlToMilli[fromPrecision - TSDB_TIME_PRECISION_HOURS]; + if (unit > INT64_MAX / tsInt64) { + return -1; + } + tsInt64 *= unit; + fromPrecision = TSDB_TIME_PRECISION_MILLI; } - if (type == TSDB_DATA_TYPE_NCHAR) { - result = result * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; - } else if (type == TSDB_DATA_TYPE_BINARY) { - result = result + VARSTR_HEADER_SIZE; - } - return result; + return convertTimePrecision(tsInt64, fromPrecision, toPrecision); } -static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, - ESchemaAction *action, bool isTag) { - int32_t code = TSDB_CODE_SUCCESS; - for (int j = 0; j < taosArrayGetSize(cols); ++j) { - if (j == 0 && !isTag) continue; - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j); - code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info); - if (code != TSDB_CODE_SUCCESS) { - return code; - } +int8_t smlGetTsTypeByLen(int32_t len) { + if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { + return TSDB_TIME_PRECISION_SECONDS; + } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { + return TSDB_TIME_PRECISION_MILLI; + } else { + return -1; } - return TSDB_CODE_SUCCESS; } -static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { - SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - int32_t i = 0; - for (; i < length; i++) { - taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); +SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen) { + SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); + if (!tag) { + return NULL; } - if (isTag) { - i = 0; - } else { - i = 1; - } - for (; i < taosArrayGetSize(cols); i++) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); - if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { - taosHashCleanup(hashTmp); - return -1; - } - } - taosHashCleanup(hashTmp); - return 0; -} + tag->sTableName = measure; + tag->sTableNameLen = measureLen; -static int32_t getBytes(uint8_t type, int32_t length) { - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - return smlFindNearestPowerOf2(length, type); - } else { - return tDataTypes[type].bytes; + tag->cols = taosArrayInit(numRows, POINTER_BYTES); + if (tag->cols == NULL) { + uError("SML:smlBuildTableInfo failed to allocate memory"); + goto cleanup; } + + // tag->tags = taosArrayInit(16, sizeof(SSmlKv)); + // if (tag->tags == NULL) { + // uError("SML:smlBuildTableInfo failed to allocate memory"); + // goto cleanup; + // } + return tag; + +cleanup: + taosMemoryFree(tag); + return NULL; } -static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, - SArray *results, int32_t numOfCols, bool isTag) { - for (int j = 0; j < taosArrayGetSize(cols); ++j) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j); - ESchemaAction action = SCHEMA_ACTION_NULL; - smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); - if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) { - SField field = {0}; - field.type = kv->type; - field.bytes = getBytes(kv->type, kv->length); - memcpy(field.name, kv->key, kv->keyLen); - taosArrayPush(results, &field); - } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { - uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); - uint16_t newIndex = *index; - if (isTag) newIndex -= numOfCols; - SField *field = (SField *)taosArrayGet(results, newIndex); - field->bytes = getBytes(kv->type, kv->length); +static int32_t smlParseTableName(SArray *tags, char *childTableName) { + size_t childTableNameLen = strlen(tsSmlChildTableName); + if (childTableNameLen <= 0) return TSDB_CODE_SUCCESS; + + for (int i = 0; i < taosArrayGetSize(tags); i++) { + SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); + // handle child table name + if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) { + memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); + break; } } + return TSDB_CODE_SUCCESS; } -// static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData, -// int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){ -static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta, - ESchemaAction action) { - SRequestObj *pRequest = NULL; - SMCreateStbReq pReq = {0}; - int32_t code = TSDB_CODE_SUCCESS; - SCmdMsgInfo pCmdMsg = {0}; +int32_t smlSetCTableName(SSmlTableInfo *oneTable) { + smlParseTableName(oneTable->tags, oneTable->childTableName); - // put front for free - pReq.numOfColumns = taosArrayGetSize(pColumns); - pReq.pColumns = pColumns; - pReq.numOfTags = taosArrayGetSize(pTags); - pReq.pTags = pTags; + if (strlen(oneTable->childTableName) == 0) { + SArray *dst = taosArrayDup(oneTable->tags, NULL); + RandTableName rName = {dst, oneTable->sTableName, (uint8_t)oneTable->sTableNameLen, oneTable->childTableName}; - code = buildRequest(info->taos->id, "", 0, NULL, false, &pRequest, 0); - if (code != TSDB_CODE_SUCCESS) { - goto end; + buildChildTableName(&rName); + taosArrayDestroy(dst); } + return TSDB_CODE_SUCCESS; +} - pRequest->syncQuery = true; - if (!pRequest->pDb) { - code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; - goto end; +SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { + SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); + if (!meta) { + return NULL; } - if (action == SCHEMA_ACTION_CREATE_STABLE) { - pReq.colVer = 1; - pReq.tagVer = 1; - pReq.suid = 0; - pReq.source = TD_REQ_FROM_APP; - } else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { - pReq.colVer = pTableMeta->sversion; - pReq.tagVer = pTableMeta->tversion + 1; - pReq.suid = pTableMeta->uid; - pReq.source = TD_REQ_FROM_TAOX; - } else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE) { - pReq.colVer = pTableMeta->sversion + 1; - pReq.tagVer = pTableMeta->tversion; - pReq.suid = pTableMeta->uid; - pReq.source = TD_REQ_FROM_TAOX; - } + if (unlikely(!isDataFormat)) { + meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (meta->tagHash == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } - if (pReq.numOfTags == 0) { - pReq.numOfTags = 1; - SField field = {0}; - field.type = TSDB_DATA_TYPE_NCHAR; - field.bytes = 1; - strcpy(field.name, tsSmlTagName); - taosArrayPush(pReq.pTags, &field); + meta->colHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (meta->colHash == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } } - pReq.commentLen = -1; - pReq.igExists = true; - tNameExtractFullName(pName, pReq.name); - - pCmdMsg.epSet = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - pCmdMsg.msgType = TDMT_MND_CREATE_STB; - pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq); - pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); - if (NULL == pCmdMsg.pMsg) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; + meta->tags = taosArrayInit(32, sizeof(SSmlKv)); + if (meta->tags == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; } - tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); - SQuery pQuery; - memset(&pQuery, 0, sizeof(pQuery)); - pQuery.execMode = QUERY_EXEC_MODE_RPC; - pQuery.pCmdMsg = &pCmdMsg; - pQuery.msgType = pQuery.pCmdMsg->msgType; - pQuery.stableQuery = true; - - launchQueryImpl(pRequest, &pQuery, true, NULL); - - if (pRequest->code == TSDB_CODE_SUCCESS) { - catalogRemoveTableMeta(info->pCatalog, pName); + meta->cols = taosArrayInit(32, sizeof(SSmlKv)); + if (meta->cols == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; } - code = pRequest->code; - taosMemoryFree(pCmdMsg.pMsg); + return meta; -end: - destroyRequest(pRequest); - tFreeSMCreateStbReq(&pReq); - return code; +cleanup: + taosMemoryFree(meta); + return NULL; } -static int32_t smlModifyDBSchemas(SSmlHandle *info) { - int32_t code = 0; - SHashObj *hashTmp = NULL; - STableMeta *pTableMeta = NULL; +// uint16_t smlCalTypeSum(char* endptr, int32_t left){ +// uint16_t sum = 0; +// for(int i = 0; i < left; i++){ +// sum += endptr[i]; +// } +// return sum; +// } - SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; - tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); +#define RETURN_FALSE \ + smlBuildInvalidDataMsg(msg, "invalid data", pVal); \ + return false; - SRequestConnInfo conn = {0}; - conn.pTrans = info->taos->pAppInfo->pTransporter; - conn.requestId = info->pRequest->requestId; - conn.requestObjRefId = info->pRequest->self; - conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - - SSmlSTableMeta **tableMetaSml = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); - while (tableMetaSml) { - SSmlSTableMeta *sTableData = *tableMetaSml; - bool needCheckMeta = false; // for multi thread - - size_t superTableLen = 0; - void *superTable = taosHashGetKey(tableMetaSml, &superTableLen); - memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - memcpy(pName.tname, superTable, superTableLen); - - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - - if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) { - SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); - SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); - smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true); - smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false); - - code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); - goto end; - } - info->cost.numOfCreateSTables++; - taosMemoryFreeClear(pTableMeta); - - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); - goto end; - } - } else if (code == TSDB_CODE_SUCCESS) { - hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, - HASH_NO_LOCK); - for (uint16_t i = pTableMeta->tableInfo.numOfColumns; - i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); - } - - ESchemaAction action = SCHEMA_ACTION_NULL; - code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->tags, &action, true); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - if (action != SCHEMA_ACTION_NULL) { - SArray *pColumns = - taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); - SArray *pTags = - taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - SField field = {0}; - field.type = pTableMeta->schema[i].type; - field.bytes = pTableMeta->schema[i].bytes; - strcpy(field.name, pTableMeta->schema[i].name); - if (i < pTableMeta->tableInfo.numOfColumns) { - taosArrayPush(pColumns, &field); - } else { - taosArrayPush(pTags, &field); - } - } - smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, - pTableMeta->tableInfo.numOfColumns, true); - - code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); - goto end; - } - - info->cost.numOfAlterTagSTables++; - taosMemoryFreeClear(pTableMeta); - code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - } +#define SET_DOUBLE \ + kvVal->type = TSDB_DATA_TYPE_DOUBLE; \ + kvVal->d = result; + +#define SET_FLOAT \ + if (!IS_VALID_FLOAT(result)) { \ + smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_FLOAT; \ + kvVal->f = (float)result; + +#define SET_BIGINT \ + if (smlDoubleToInt64OverFlow(result)) { \ + errno = 0; \ + int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \ + if (errno == ERANGE) { \ + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_BIGINT; \ + kvVal->i = tmp; \ + return true; \ + } \ + kvVal->type = TSDB_DATA_TYPE_BIGINT; \ + kvVal->i = (int64_t)result; + +#define SET_INT \ + if (!IS_VALID_INT(result)) { \ + smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_INT; \ + kvVal->i = result; + +#define SET_SMALL_INT \ + if (!IS_VALID_SMALLINT(result)) { \ + smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_SMALLINT; \ + kvVal->i = result; + +#define SET_UBIGINT \ + if (result >= (double)UINT64_MAX || result < 0) { \ + errno = 0; \ + uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \ + if (errno == ERANGE || result < 0) { \ + smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ + kvVal->u = tmp; \ + return true; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ + kvVal->u = result; + +#define SET_UINT \ + if (!IS_VALID_UINT(result)) { \ + smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UINT; \ + kvVal->u = result; + +#define SET_USMALL_INT \ + if (!IS_VALID_USMALLINT(result)) { \ + smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_USMALLINT; \ + kvVal->u = result; + +#define SET_TINYINT \ + if (!IS_VALID_TINYINT(result)) { \ + smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_TINYINT; \ + kvVal->i = result; + +#define SET_UTINYINT \ + if (!IS_VALID_UTINYINT(result)) { \ + smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UTINYINT; \ + kvVal->u = result; + +bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->length; + char *endptr = NULL; + double result = taosStr2Double(pVal, &endptr); + if (pVal == endptr) { + RETURN_FALSE + } - taosHashClear(hashTmp); - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + int32_t left = len - (endptr - pVal); + if (left == 0) { + SET_DOUBLE + } else if (left == 3) { + if (endptr[0] == 'f' || endptr[0] == 'F') { + if (endptr[1] == '6' && endptr[2] == '4') { + SET_DOUBLE + } else if (endptr[1] == '3' && endptr[2] == '2') { + SET_FLOAT + } else { + RETURN_FALSE } - action = SCHEMA_ACTION_NULL; - code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, &action, false); - if (code != TSDB_CODE_SUCCESS) { - goto end; + } else if (endptr[0] == 'i' || endptr[0] == 'I') { + if (endptr[1] == '6' && endptr[2] == '4') { + SET_BIGINT + } else if (endptr[1] == '3' && endptr[2] == '2') { + SET_INT + } else if (endptr[1] == '1' && endptr[2] == '6') { + SET_SMALL_INT + } else { + RETURN_FALSE } - if (action != SCHEMA_ACTION_NULL) { - SArray *pColumns = - taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); - SArray *pTags = - taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - SField field = {0}; - field.type = pTableMeta->schema[i].type; - field.bytes = pTableMeta->schema[i].bytes; - strcpy(field.name, pTableMeta->schema[i].name); - if (i < pTableMeta->tableInfo.numOfColumns) { - taosArrayPush(pColumns, &field); - } else { - taosArrayPush(pTags, &field); - } - } - - smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns, - pTableMeta->tableInfo.numOfColumns, false); - - code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); - goto end; - } - - info->cost.numOfAlterColSTables++; - taosMemoryFreeClear(pTableMeta); - code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); - goto end; - } + } else if (endptr[0] == 'u' || endptr[0] == 'U') { + if (endptr[1] == '6' && endptr[2] == '4') { + SET_UBIGINT + } else if (endptr[1] == '3' && endptr[2] == '2') { + SET_UINT + } else if (endptr[1] == '1' && endptr[2] == '6') { + SET_USMALL_INT + } else { + RETURN_FALSE } - - needCheckMeta = true; - taosHashCleanup(hashTmp); - hashTmp = NULL; } else { - uError("SML:0x%" PRIx64 " load table meta error: %s", info->id, tstrerror(code)); - goto end; + RETURN_FALSE } - - if (needCheckMeta) { - code = smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags, - sTableData->tags, true); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " check tag failed. super table name %s", info->id, pName.tname); - goto end; + } else if (left == 2) { + if (endptr[0] == 'i' || endptr[0] == 'I') { + if (endptr[1] == '8') { + SET_TINYINT + } else { + RETURN_FALSE } - code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols, false); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " check cols failed. super table name %s", info->id, pName.tname); - goto end; + } else if (endptr[0] == 'u' || endptr[0] == 'U') { + if (endptr[1] == '8') { + SET_UTINYINT + } else { + RETURN_FALSE } + } else { + RETURN_FALSE } - - sTableData->tableMeta = pTableMeta; - - tableMetaSml = (SSmlSTableMeta **)taosHashIterate(info->superTables, tableMetaSml); + } else if (left == 1) { + if (endptr[0] == 'i' || endptr[0] == 'I') { + SET_BIGINT + } else if (endptr[0] == 'u' || endptr[0] == 'U') { + SET_UBIGINT + } else { + RETURN_FALSE + } + } else { + RETURN_FALSE; } - return 0; - -end: - taosHashCleanup(hashTmp); - taosMemoryFreeClear(pTableMeta); - // catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); - return code; + return true; } -static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { +bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->length; char *endptr = NULL; @@ -708,764 +506,462 @@ static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { return true; } -static bool smlParseBool(SSmlKv *kvVal) { - const char *pVal = kvVal->value; - int32_t len = kvVal->length; - if ((len == 1) && (pVal[0] == 't' || pVal[0] == 'T')) { - kvVal->i = true; - return true; - } +STableMeta *smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen) { + STableMeta *pTableMeta = NULL; - if ((len == 1) && (pVal[0] == 'f' || pVal[0] == 'F')) { - kvVal->i = false; - return true; - } + SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; + tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); - if ((len == 4) && !strncasecmp(pVal, "true", len)) { - kvVal->i = true; - return true; - } - if ((len == 5) && !strncasecmp(pVal, "false", len)) { - kvVal->i = false; - return true; - } - return false; -} + SRequestConnInfo conn = {0}; + conn.pTrans = info->taos->pAppInfo->pTransporter; + conn.requestId = info->pRequest->requestId; + conn.requestObjRefId = info->pRequest->self; + conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + memcpy(pName.tname, measure, measureLen); -static bool smlIsBinary(const char *pVal, uint16_t len) { - // binary: "abc" - if (len < 2) { - return false; - } - if (pVal[0] == '"' && pVal[len - 1] == '"') { - return true; + int32_t code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + return NULL; } - return false; + return pTableMeta; } -static bool smlIsNchar(const char *pVal, uint16_t len) { - // nchar: L"abc" - if (len < 3) { - return false; - } - if ((pVal[0] == 'l' || pVal[0] == 'L') && pVal[1] == '"' && pVal[len - 1] == '"') { - return true; - } - return false; -} +static int64_t smlGenId() { + static volatile int64_t linesSmlHandleId = 0; -static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { - char *endPtr = NULL; - int64_t tsInt64 = taosStr2Int64(value, &endPtr, 10); - if (value + len != endPtr) { - return -1; - } - double ts = tsInt64; - switch (type) { - case TSDB_TIME_PRECISION_HOURS: - ts *= NANOSECOND_PER_HOUR; - tsInt64 *= NANOSECOND_PER_HOUR; - break; - case TSDB_TIME_PRECISION_MINUTES: - ts *= NANOSECOND_PER_MINUTE; - tsInt64 *= NANOSECOND_PER_MINUTE; - break; - case TSDB_TIME_PRECISION_SECONDS: - ts *= NANOSECOND_PER_SEC; - tsInt64 *= NANOSECOND_PER_SEC; - break; - case TSDB_TIME_PRECISION_MILLI: - ts *= NANOSECOND_PER_MSEC; - tsInt64 *= NANOSECOND_PER_MSEC; - break; - case TSDB_TIME_PRECISION_MICRO: - ts *= NANOSECOND_PER_USEC; - tsInt64 *= NANOSECOND_PER_USEC; - break; - case TSDB_TIME_PRECISION_NANO: - break; - default: - ASSERT(0); - } - if (ts >= (double)INT64_MAX || ts < 0) { - return -1; - } + int64_t id = 0; + do { + id = atomic_add_fetch_64(&linesSmlHandleId, 1); + } while (id == 0); - return tsInt64; + return id; } -static int8_t smlGetTsTypeByLen(int32_t len) { - if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { - return TSDB_TIME_PRECISION_SECONDS; - } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { - return TSDB_TIME_PRECISION_MILLI; +static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSmlKv *kv, bool isTag, + ESchemaAction *action, SSmlHandle *info) { + uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL; + if (index) { + if (colField[*index].type != kv->type) { + uError("SML:0x%" PRIx64 " point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, + kv->key, colField[*index].type, kv->type); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if ((colField[*index].type == TSDB_DATA_TYPE_VARCHAR && + (colField[*index].bytes - VARSTR_HEADER_SIZE) < kv->length) || + (colField[*index].type == TSDB_DATA_TYPE_NCHAR && + ((colField[*index].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE < kv->length))) { + if (isTag) { + *action = SCHEMA_ACTION_CHANGE_TAG_SIZE; + } else { + *action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE; + } + } } else { - return -1; + if (isTag) { + *action = SCHEMA_ACTION_ADD_TAG; + } else { + *action = SCHEMA_ACTION_ADD_COLUMN; + } } + return 0; } -static int8_t smlGetTsTypeByPrecision(int8_t precision) { - switch (precision) { - case TSDB_SML_TIMESTAMP_HOURS: - return TSDB_TIME_PRECISION_HOURS; - case TSDB_SML_TIMESTAMP_MILLI_SECONDS: - return TSDB_TIME_PRECISION_MILLI; - case TSDB_SML_TIMESTAMP_NANO_SECONDS: - case TSDB_SML_TIMESTAMP_NOT_CONFIGURED: - return TSDB_TIME_PRECISION_NANO; - case TSDB_SML_TIMESTAMP_MICRO_SECONDS: - return TSDB_TIME_PRECISION_MICRO; - case TSDB_SML_TIMESTAMP_SECONDS: - return TSDB_TIME_PRECISION_SECONDS; - case TSDB_SML_TIMESTAMP_MINUTES: - return TSDB_TIME_PRECISION_MINUTES; - default: - return -1; +static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { + int32_t result = 1; + while (result <= length) { + result *= 2; } -} - -static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t len) { - if (len == 0 || (len == 1 && data[0] == '0')) { - return taosGetTimestampNs(); + if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { + result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE; + } else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + result = (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; } - int8_t tsType = smlGetTsTypeByPrecision(info->precision); - if (tsType == -1) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp precision", NULL); - return -1; - } - - int64_t ts = smlGetTimeValue(data, len, tsType); - if (ts == -1) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); - return -1; - } - return ts; -} - -static int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len) { - if (!data) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL); - return -1; - } - if (len == 1 && data[0] == '0') { - return taosGetTimestampNs(); - } - int8_t tsType = smlGetTsTypeByLen(len); - if (tsType == -1) { - smlBuildInvalidDataMsg(&info->msgBuf, - "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data); - return -1; - } - int64_t ts = smlGetTimeValue(data, len, tsType); - if (ts == -1) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); - return -1; - } - return ts; -} - -static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArray *cols) { - int64_t ts = 0; - if (info->protocol == TSDB_SML_LINE_PROTOCOL) { - // uError("SML:data:%s,len:%d", data, len); - ts = smlParseInfluxTime(info, data, len); - } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - ts = smlParseOpenTsdbTime(info, data, len); - } else { - ASSERT(0); - } - uDebug("SML:0x%" PRIx64 " smlParseTS:%" PRId64, info->id, ts); - - if (ts <= 0) { - uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); - return TSDB_CODE_INVALID_TIMESTAMP; - } - - // add ts to - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) { - return TSDB_CODE_OUT_OF_MEMORY; + if (type == TSDB_DATA_TYPE_NCHAR) { + result = result * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; + } else if (type == TSDB_DATA_TYPE_BINARY) { + result = result + VARSTR_HEADER_SIZE; } - - kv->key = TS; - kv->keyLen = TS_LEN; - kv->i = ts; - kv->type = TSDB_DATA_TYPE_TIMESTAMP; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - taosArrayPush(cols, &kv); - - return TSDB_CODE_SUCCESS; + return result; } -static int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { - // binary - if (smlIsBinary(pVal->value, pVal->length)) { - pVal->type = TSDB_DATA_TYPE_BINARY; - pVal->length -= BINARY_ADD_LEN; - if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - pVal->value += (BINARY_ADD_LEN - 1); - return TSDB_CODE_SUCCESS; - } - // nchar - if (smlIsNchar(pVal->value, pVal->length)) { - pVal->type = TSDB_DATA_TYPE_NCHAR; - pVal->length -= NCHAR_ADD_LEN; - if (pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; +static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, + ESchemaAction *action, bool isTag) { + int32_t code = TSDB_CODE_SUCCESS; + for (int j = 0; j < taosArrayGetSize(cols); ++j) { + if (j == 0 && !isTag) continue; + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); + code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info); + if (code != TSDB_CODE_SUCCESS) { + return code; } - pVal->value += (NCHAR_ADD_LEN - 1); - return TSDB_CODE_SUCCESS; - } - - // bool - if (smlParseBool(pVal)) { - pVal->type = TSDB_DATA_TYPE_BOOL; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - return TSDB_CODE_SUCCESS; } - // number - if (smlParseNumber(pVal, msg)) { - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - return TSDB_CODE_SUCCESS; - } - - return TSDB_CODE_TSC_INVALID_VALUE; + return TSDB_CODE_SUCCESS; } -static int32_t smlParseInfluxString(const char *sql, const char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) { - if (!sql) return TSDB_CODE_SML_INVALID_DATA; - JUMP_SPACE(sql, sqlEnd) - if (*sql == COMMA) return TSDB_CODE_SML_INVALID_DATA; - elements->measure = sql; - - // parse measure - while (sql < sqlEnd) { - if ((sql != elements->measure) && IS_SLASH_LETTER(sql)) { - MOVE_FORWARD_ONE(sql, sqlEnd - sql); - sqlEnd--; - continue; - } - if (IS_COMMA(sql)) { - break; - } - - if (IS_SPACE(sql)) { - break; - } - sql++; - } - elements->measureLen = sql - elements->measure; - if (IS_INVALID_TABLE_LEN(elements->measureLen)) { - smlBuildInvalidDataMsg(msg, "measure is empty or too large than 192", NULL); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; +static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { + SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + int32_t i = 0; + for (; i < length; i++) { + taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); } - // parse tag - if (*sql == SPACE) { - elements->tagsLen = 0; + if (isTag) { + i = 0; } else { - if (*sql == COMMA) sql++; - elements->tags = sql; - while (sql < sqlEnd) { - if (IS_SPACE(sql)) { - break; - } - sql++; - } - elements->tagsLen = sql - elements->tags; - } - elements->measureTagsLen = sql - elements->measure; - - // parse cols - JUMP_SPACE(sql, sqlEnd) - elements->cols = sql; - bool isInQuote = false; - while (sql < sqlEnd) { - if (IS_QUOTE(sql)) { - isInQuote = !isInQuote; - } - if (!isInQuote && IS_SPACE(sql)) { - break; - } - sql++; - } - if (isInQuote) { - smlBuildInvalidDataMsg(msg, "only one quote", elements->cols); - return TSDB_CODE_SML_INVALID_DATA; - } - elements->colsLen = sql - elements->cols; - if (elements->colsLen == 0) { - smlBuildInvalidDataMsg(msg, "cols is empty", NULL); - return TSDB_CODE_SML_INVALID_DATA; + i = 1; } - - // parse timestamp - JUMP_SPACE(sql, sqlEnd) - elements->timestamp = sql; - while (sql < sqlEnd) { - if (isspace(*sql)) { - break; + for (; i < taosArrayGetSize(cols); i++) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { + taosHashCleanup(hashTmp); + return -1; } - sql++; } - elements->timestampLen = sql - elements->timestamp; - - return TSDB_CODE_SUCCESS; + taosHashCleanup(hashTmp); + return 0; } -static void smlParseTelnetElement(const char **sql, const char *sqlEnd, const char **data, int32_t *len) { - while (*sql < sqlEnd) { - if (**sql != SPACE && !(*data)) { - *data = *sql; - } else if (**sql == SPACE && *data) { - *len = *sql - *data; - break; - } - (*sql)++; +static int32_t getBytes(uint8_t type, int32_t length) { + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + return smlFindNearestPowerOf2(length, type); + } else { + return tDataTypes[type].bytes; } } -static int32_t smlParseTelnetTags(const char *data, const char *sqlEnd, SArray *cols, char *childTableName, - SHashObj *dumplicateKey, SSmlMsgBuf *msg) { - if (!cols) return TSDB_CODE_OUT_OF_MEMORY; - const char *sql = data; - size_t childTableNameLen = strlen(tsSmlChildTableName); - while (sql < sqlEnd) { - JUMP_SPACE(sql, sqlEnd) - if (*sql == '\0') break; - - const char *key = sql; - int32_t keyLen = 0; - - // parse key - while (sql < sqlEnd) { - if (*sql == SPACE) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; - } - if (*sql == EQUAL) { - keyLen = sql - key; - sql++; - break; - } - sql++; - } - - if (IS_INVALID_COL_LEN(keyLen)) { - smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { - smlBuildInvalidDataMsg(msg, "dumplicate key", key); - return TSDB_CODE_TSC_DUP_NAMES; - } - - // parse value - const char *value = sql; - int32_t valueLen = 0; - while (sql < sqlEnd) { - // parse value - if (*sql == SPACE) { - break; - } - if (*sql == EQUAL) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; - } - sql++; - } - valueLen = sql - value; - - if (valueLen == 0) { - smlBuildInvalidDataMsg(msg, "invalid value", value); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - // handle child table name - if (childTableNameLen != 0 && strncmp(key, tsSmlChildTableName, keyLen) == 0) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, value, (valueLen < TSDB_TABLE_NAME_LEN ? valueLen : TSDB_TABLE_NAME_LEN)); - continue; - } - - if (valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; +static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, + SArray *results, int32_t numOfCols, bool isTag) { + for (int j = 0; j < taosArrayGetSize(cols); ++j) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); + ESchemaAction action = SCHEMA_ACTION_NULL; + smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); + if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) { + SField field = {0}; + field.type = kv->type; + field.bytes = getBytes(kv->type, kv->length); + memcpy(field.name, kv->key, kv->keyLen); + taosArrayPush(results, &field); + } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { + uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); + uint16_t newIndex = *index; + if (isTag) newIndex -= numOfCols; + SField *field = (SField *)taosArrayGet(results, newIndex); + field->bytes = getBytes(kv->type, kv->length); } - - // add kv to SSmlKv - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - kv->key = key; - kv->keyLen = keyLen; - kv->value = value; - kv->length = valueLen; - kv->type = TSDB_DATA_TYPE_NCHAR; - - taosArrayPush(cols, &kv); } - return TSDB_CODE_SUCCESS; } -// format: =[ =] -static int32_t smlParseTelnetString(SSmlHandle *info, const char *sql, const char *sqlEnd, SSmlTableInfo *tinfo, - SArray *cols) { - if (!sql) return TSDB_CODE_SML_INVALID_DATA; +// static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData, +// int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){ +static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta, + ESchemaAction action) { + SRequestObj *pRequest = NULL; + SMCreateStbReq pReq = {0}; + int32_t code = TSDB_CODE_SUCCESS; + SCmdMsgInfo pCmdMsg = {0}; + + // put front for free + pReq.numOfColumns = taosArrayGetSize(pColumns); + pReq.pColumns = pColumns; + pReq.numOfTags = taosArrayGetSize(pTags); + pReq.pTags = pTags; - // parse metric - smlParseTelnetElement(&sql, sqlEnd, &tinfo->sTableName, &tinfo->sTableNameLen); - if (!(tinfo->sTableName) || IS_INVALID_TABLE_LEN(tinfo->sTableNameLen)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + code = buildRequest(info->taos->id, "", 0, NULL, false, &pRequest, 0); + if (code != TSDB_CODE_SUCCESS) { + goto end; } - // parse timestamp - const char *timestamp = NULL; - int32_t tLen = 0; - smlParseTelnetElement(&sql, sqlEnd, ×tamp, &tLen); - if (!timestamp || tLen == 0) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); - return TSDB_CODE_SML_INVALID_DATA; + pRequest->syncQuery = true; + if (!pRequest->pDb) { + code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + goto end; } - int32_t ret = smlParseTS(info, timestamp, tLen, cols); - if (ret != TSDB_CODE_SUCCESS) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); - return ret; + if (action == SCHEMA_ACTION_CREATE_STABLE) { + pReq.colVer = 1; + pReq.tagVer = 1; + pReq.suid = 0; + pReq.source = TD_REQ_FROM_APP; + } else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { + pReq.colVer = pTableMeta->sversion; + pReq.tagVer = pTableMeta->tversion + 1; + pReq.suid = pTableMeta->uid; + pReq.source = TD_REQ_FROM_TAOX; + } else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE) { + pReq.colVer = pTableMeta->sversion + 1; + pReq.tagVer = pTableMeta->tversion; + pReq.suid = pTableMeta->uid; + pReq.source = TD_REQ_FROM_TAOX; } - // parse value - const char *value = NULL; - int32_t valueLen = 0; - smlParseTelnetElement(&sql, sqlEnd, &value, &valueLen); - if (!value || valueLen == 0) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); - return TSDB_CODE_TSC_INVALID_VALUE; + if (pReq.numOfTags == 0) { + pReq.numOfTags = 1; + SField field = {0}; + field.type = TSDB_DATA_TYPE_NCHAR; + field.bytes = 1; + strcpy(field.name, tsSmlTagName); + taosArrayPush(pReq.pTags, &field); } - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - taosArrayPush(cols, &kv); - kv->key = VALUE; - kv->keyLen = VALUE_LEN; - kv->value = value; - kv->length = valueLen; - if ((ret = smlParseValue(kv, &info->msgBuf)) != TSDB_CODE_SUCCESS) { - return ret; + pReq.commentLen = -1; + pReq.igExists = true; + tNameExtractFullName(pName, pReq.name); + + pCmdMsg.epSet = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + pCmdMsg.msgType = TDMT_MND_CREATE_STB; + pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq); + pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); + if (NULL == pCmdMsg.pMsg) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; } + tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); + + SQuery pQuery; + memset(&pQuery, 0, sizeof(pQuery)); + pQuery.execMode = QUERY_EXEC_MODE_RPC; + pQuery.pCmdMsg = &pCmdMsg; + pQuery.msgType = pQuery.pCmdMsg->msgType; + pQuery.stableQuery = true; - // parse tags - ret = smlParseTelnetTags(sql, sqlEnd, tinfo->tags, tinfo->childTableName, info->dumplicateKey, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); - return ret; + launchQueryImpl(pRequest, &pQuery, true, NULL); + + if (pRequest->code == TSDB_CODE_SUCCESS) { + catalogRemoveTableMeta(info->pCatalog, pName); } + code = pRequest->code; + taosMemoryFree(pCmdMsg.pMsg); - return TSDB_CODE_SUCCESS; +end: + destroyRequest(pRequest); + tFreeSMCreateStbReq(&pReq); + return code; } -static int32_t smlParseCols(const char *data, int32_t len, SArray *cols, char *childTableName, bool isTag, - SHashObj *dumplicateKey, SSmlMsgBuf *msg) { - if (len == 0) { +static int32_t smlModifyDBSchemas(SSmlHandle *info) { + if (info->dataFormat && !info->needModifySchema) { return TSDB_CODE_SUCCESS; } + int32_t code = 0; + SHashObj *hashTmp = NULL; + STableMeta *pTableMeta = NULL; - size_t childTableNameLen = strlen(tsSmlChildTableName); - const char *sql = data; - while (sql < data + len) { - const char *key = sql; - int32_t keyLen = 0; - - while (sql < data + len) { - // parse key - if (IS_COMMA(sql)) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; - } - if (IS_EQUAL(sql)) { - keyLen = sql - key; - sql++; - break; - } - sql++; - } + SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; + tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); - if (IS_INVALID_COL_LEN(keyLen)) { - smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { - smlBuildInvalidDataMsg(msg, "dumplicate key", key); - return TSDB_CODE_TSC_DUP_NAMES; - } + SRequestConnInfo conn = {0}; + conn.pTrans = info->taos->pAppInfo->pTransporter; + conn.requestId = info->pRequest->requestId; + conn.requestObjRefId = info->pRequest->self; + conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - // parse value - const char *value = sql; - int32_t valueLen = 0; - bool isInQuote = false; - while (sql < data + len) { - // parse value - if (!isTag && IS_QUOTE(sql)) { - isInQuote = !isInQuote; - sql++; - continue; - } - if (!isInQuote && IS_COMMA(sql)) { - break; - } - if (!isInQuote && IS_EQUAL(sql)) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; - } - sql++; - } - valueLen = sql - value; - sql++; + NodeList *tmp = info->superTables; + while (tmp) { + SSmlSTableMeta *sTableData = (SSmlSTableMeta *)tmp->data.value; + bool needCheckMeta = false; // for multi thread - if (isInQuote) { - smlBuildInvalidDataMsg(msg, "only one quote", value); - return TSDB_CODE_SML_INVALID_DATA; - } - if (valueLen == 0) { - smlBuildInvalidDataMsg(msg, "invalid value", value); - return TSDB_CODE_SML_INVALID_DATA; - } - PROCESS_SLASH(key, keyLen) - PROCESS_SLASH(value, valueLen) + size_t superTableLen = (size_t)tmp->data.keyLen; + const void *superTable = tmp->data.key; + memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + memcpy(pName.tname, superTable, superTableLen); - // handle child table name - if (childTableName && childTableNameLen != 0 && strncmp(key, tsSmlChildTableName, keyLen) == 0) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, value, (valueLen < TSDB_TABLE_NAME_LEN ? valueLen : TSDB_TABLE_NAME_LEN)); - continue; - } + code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - // add kv to SSmlKv - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - if (cols) taosArrayPush(cols, &kv); + if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) { + SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); + SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); + smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true); + smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false); - kv->key = key; - kv->keyLen = keyLen; - kv->value = value; - kv->length = valueLen; - if (isTag) { - if (valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - kv->type = TSDB_DATA_TYPE_NCHAR; - } else { - int32_t ret = smlParseValue(kv, msg); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); + goto end; } - } - } + info->cost.numOfCreateSTables++; + taosMemoryFreeClear(pTableMeta); - return TSDB_CODE_SUCCESS; -} + code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); + goto end; + } + } else if (code == TSDB_CODE_SUCCESS) { + hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, + HASH_NO_LOCK); + for (uint16_t i = pTableMeta->tableInfo.numOfColumns; + i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { + taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + } -static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, SSmlMsgBuf *msg) { - for (int i = 0; i < taosArrayGetSize(cols); ++i) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); + ESchemaAction action = SCHEMA_ACTION_NULL; + code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->tags, &action, true); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + if (action != SCHEMA_ACTION_NULL) { + SArray *pColumns = + taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); + SArray *pTags = + taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); - if (index) { - SSmlKv **value = (SSmlKv **)taosArrayGet(metaArray, *index); - if (kv->type != (*value)->type) { - smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); - return TSDB_CODE_SML_NOT_SAME_TYPE; - } else { - if (IS_VAR_DATA_TYPE(kv->type)) { // update string len, if bigger - if (kv->length > (*value)->length) { - *value = kv; + for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { + SField field = {0}; + field.type = pTableMeta->schema[i].type; + field.bytes = pTableMeta->schema[i].bytes; + strcpy(field.name, pTableMeta->schema[i].name); + if (i < pTableMeta->tableInfo.numOfColumns) { + taosArrayPush(pColumns, &field); + } else { + taosArrayPush(pTags, &field); } } - } - } else { - size_t tmp = taosArrayGetSize(metaArray); - ASSERT(tmp <= INT16_MAX); - int16_t size = tmp; - taosArrayPush(metaArray, &kv); - taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); - } - } + smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, + pTableMeta->tableInfo.numOfColumns, true); - return TSDB_CODE_SUCCESS; -} + code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); + goto end; + } -static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) { - for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); - taosArrayPush(metaArray, &kv); - taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); - } -} + info->cost.numOfAlterTagSTables++; + taosMemoryFreeClear(pTableMeta); + code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + } -static SSmlTableInfo *smlBuildTableInfo() { - SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); - if (!tag) { - return NULL; - } + taosHashClear(hashTmp); + for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { + taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + } + action = SCHEMA_ACTION_NULL; + code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, &action, false); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + if (action != SCHEMA_ACTION_NULL) { + SArray *pColumns = + taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); + SArray *pTags = + taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - tag->cols = taosArrayInit(16, POINTER_BYTES); - if (tag->cols == NULL) { - uError("SML:smlBuildTableInfo failed to allocate memory"); - goto cleanup; - } + for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { + SField field = {0}; + field.type = pTableMeta->schema[i].type; + field.bytes = pTableMeta->schema[i].bytes; + strcpy(field.name, pTableMeta->schema[i].name); + if (i < pTableMeta->tableInfo.numOfColumns) { + taosArrayPush(pColumns, &field); + } else { + taosArrayPush(pTags, &field); + } + } - tag->tags = taosArrayInit(16, POINTER_BYTES); - if (tag->tags == NULL) { - uError("SML:smlBuildTableInfo failed to allocate memory"); - goto cleanup; - } - return tag; + smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns, + pTableMeta->tableInfo.numOfColumns, false); -cleanup: - taosMemoryFree(tag); - return NULL; -} + code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); + goto end; + } -static void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) { - if (info->dataFormat) { - for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { - SArray *kvArray = (SArray *)taosArrayGetP(tag->cols, i); - for (int j = 0; j < taosArrayGetSize(kvArray); ++j) { - SSmlKv *p = (SSmlKv *)taosArrayGetP(kvArray, j); - taosMemoryFree(p); + info->cost.numOfAlterColSTables++; + taosMemoryFreeClear(pTableMeta); + code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); + goto end; + } } - taosArrayDestroy(kvArray); + + needCheckMeta = true; + taosHashCleanup(hashTmp); + hashTmp = NULL; + } else { + uError("SML:0x%" PRIx64 " load table meta error: %s", info->id, tstrerror(code)); + goto end; } - } else { - for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { - SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i); - void **p1 = (void **)taosHashIterate(kvHash, NULL); - while (p1) { - taosMemoryFree(*p1); - p1 = (void **)taosHashIterate(kvHash, p1); + + if (needCheckMeta) { + code = smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags, + sTableData->tags, true); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " check tag failed. super table name %s", info->id, pName.tname); + goto end; + } + code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols, false); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " check cols failed. super table name %s", info->id, pName.tname); + goto end; } - taosHashCleanup(kvHash); } - } - for (size_t i = 0; i < taosArrayGetSize(tag->tags); i++) { - SSmlKv *p = (SSmlKv *)taosArrayGetP(tag->tags, i); - taosMemoryFree(p); - } - taosArrayDestroy(tag->cols); - taosArrayDestroy(tag->tags); - taosMemoryFree(tag); -} -static int32_t smlKvTimeArrayCompare(const void *key1, const void *key2) { - SArray *s1 = *(SArray **)key1; - SArray *s2 = *(SArray **)key2; - SSmlKv *kv1 = (SSmlKv *)taosArrayGetP(s1, 0); - SSmlKv *kv2 = (SSmlKv *)taosArrayGetP(s2, 0); - ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP); - if (kv1->i < kv2->i) { - return -1; - } else if (kv1->i > kv2->i) { - return 1; - } else { - return 0; - } -} + sTableData->tableMeta = pTableMeta; -static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) { - SHashObj *s1 = *(SHashObj **)key1; - SHashObj *s2 = *(SHashObj **)key2; - SSmlKv **kv1pp = (SSmlKv **)taosHashGet(s1, TS, TS_LEN); - SSmlKv **kv2pp = (SSmlKv **)taosHashGet(s2, TS, TS_LEN); - if (!kv1pp || !kv2pp) { - uError("smlKvTimeHashCompare kv is null"); - return -1; - } - SSmlKv *kv1 = *kv1pp; - SSmlKv *kv2 = *kv2pp; - if (!kv1 || kv1->type != TSDB_DATA_TYPE_TIMESTAMP) { - uError("smlKvTimeHashCompare kv1"); - return -1; - } - if (!kv2 || kv2->type != TSDB_DATA_TYPE_TIMESTAMP) { - uError("smlKvTimeHashCompare kv2"); - return -1; - } - if (kv1->i < kv2->i) { - return -1; - } else if (kv1->i > kv2->i) { - return 1; - } else { - return 0; + tmp = tmp->next; } + return 0; + +end: + taosHashCleanup(hashTmp); + taosMemoryFreeClear(pTableMeta); + // catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); + return code; } -static int32_t smlDealCols(SSmlTableInfo *oneTable, bool dataFormat, SArray *cols) { - if (dataFormat) { - void *p = taosArraySearch(oneTable->cols, &cols, smlKvTimeArrayCompare, TD_GT); - if (p == NULL) { - taosArrayPush(oneTable->cols, &cols); - } else { - taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols); +/* +static int32_t smlCheckDupUnit(SHashObj *dumplicateKey, SArray *tags, SSmlMsgBuf *msg){ + for(int i = 0; i < taosArrayGetSize(tags); i++) { + SSmlKv *tag = taosArrayGet(tags, i); + if (smlCheckDuplicateKey(tag->key, tag->keyLen, dumplicateKey)) { + smlBuildInvalidDataMsg(msg, "dumplicate key", tag->key); + return TSDB_CODE_TSC_DUP_NAMES; } - return TSDB_CODE_SUCCESS; - } - - SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (!kvHash) { - uError("SML:smlDealCols failed to allocate memory"); - return TSDB_CODE_OUT_OF_MEMORY; - } - for (size_t i = 0; i < taosArrayGetSize(cols); i++) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); - taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); - } - - void *p = taosArraySearch(oneTable->cols, &kvHash, smlKvTimeHashCompare, TD_GT); - if (p == NULL) { - taosArrayPush(oneTable->cols, &kvHash); - } else { - taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &kvHash); } return TSDB_CODE_SUCCESS; } -static SSmlSTableMeta *smlBuildSTableMeta() { - SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); - if (!meta) { - return NULL; - } - meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (meta->tagHash == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; +static int32_t smlJudgeDupColName(SArray *cols, SArray *tags, SSmlMsgBuf *msg) { + SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + int ret = smlCheckDupUnit(dumplicateKey, cols, msg); + if(ret != TSDB_CODE_SUCCESS){ + goto end; } - - meta->colHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (meta->colHash == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; + ret = smlCheckDupUnit(dumplicateKey, tags, msg); + if(ret != TSDB_CODE_SUCCESS){ + goto end; } - meta->tags = taosArrayInit(32, POINTER_BYTES); - if (meta->tags == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; - } + end: + taosHashCleanup(dumplicateKey); + return ret; +} +*/ - meta->cols = taosArrayInit(32, POINTER_BYTES); - if (meta->cols == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; +static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) { + for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); + if (ret == 0) { + taosArrayPush(metaArray, kv); + } } - return meta; - -cleanup: - taosMemoryFree(meta); - return NULL; } static void smlDestroySTableMeta(SSmlSTableMeta *meta) { @@ -1477,840 +973,244 @@ static void smlDestroySTableMeta(SSmlSTableMeta *meta) { taosMemoryFree(meta); } -static void smlDestroyCols(SArray *cols) { - if (!cols) return; +static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg) { for (int i = 0; i < taosArrayGetSize(cols); ++i) { - void *kv = taosArrayGetP(cols, i); - taosMemoryFree(kv); - } -} - -static void smlDestroyInfo(SSmlHandle *info) { - if (!info) return; - qDestroyQuery(info->pQuery); - smlDestroyHandle(info->exec); - - // destroy info->childTables - void **p1 = (void **)taosHashIterate(info->childTables, NULL); - while (p1) { - smlDestroyTableInfo(info, (SSmlTableInfo *)(*p1)); - p1 = (void **)taosHashIterate(info->childTables, p1); - } - taosHashCleanup(info->childTables); - - // destroy info->superTables - p1 = (void **)taosHashIterate(info->superTables, NULL); - while (p1) { - smlDestroySTableMeta((SSmlSTableMeta *)(*p1)); - p1 = (void **)taosHashIterate(info->superTables, p1); - } - taosHashCleanup(info->superTables); + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - // destroy info->pVgHash - taosHashCleanup(info->pVgHash); - taosHashCleanup(info->dumplicateKey); - if (!info->dataFormat) { - taosArrayDestroy(info->colsContainer); - } - destroyRequest(info->pRequest); - - cJSON_Delete(info->root); - taosMemoryFreeClear(info); -} - -static SSmlHandle *smlBuildSmlInfo(STscObj *pTscObj, SRequestObj *request, SMLProtocolType protocol, int8_t precision) { - int32_t code = TSDB_CODE_SUCCESS; - SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle)); - if (NULL == info) { - return NULL; - } - info->id = smlGenId(); - - info->pQuery = (SQuery *)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == info->pQuery) { - uError("SML:0x%" PRIx64 " create info->pQuery error", info->id); - goto cleanup; - } - info->pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - info->pQuery->haveResultSet = false; - info->pQuery->msgType = TDMT_VND_SUBMIT; - info->pQuery->pRoot = (SNode *)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT); - if (NULL == info->pQuery->pRoot) { - uError("SML:0x%" PRIx64 " create info->pQuery->pRoot error", info->id); - goto cleanup; - } - - if (pTscObj) { - info->taos = pTscObj; - code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); - goto cleanup; - } - } - - info->precision = precision; - info->protocol = protocol; - if (protocol == TSDB_SML_LINE_PROTOCOL) { - info->dataFormat = tsSmlDataFormat; - } else { - info->dataFormat = true; - } - - if (request) { - info->pRequest = request; - info->msgBuf.buf = info->pRequest->msgBuf; - info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; - info->pRequest->stmtType = info->pQuery->pRoot->type; - } - - info->exec = smlInitHandle(info->pQuery); - info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - - info->dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (!info->dataFormat) { - info->colsContainer = taosArrayInit(32, POINTER_BYTES); - if (NULL == info->colsContainer) { - uError("SML:0x%" PRIx64 " create info failed", info->id); - goto cleanup; - } - } - if (NULL == info->exec || NULL == info->childTables || NULL == info->superTables || NULL == info->pVgHash || - NULL == info->dumplicateKey) { - uError("SML:0x%" PRIx64 " create info failed", info->id); - goto cleanup; - } - - return info; -cleanup: - smlDestroyInfo(info); - return NULL; -} - -/************* TSDB_SML_JSON_PROTOCOL function start **************/ -static int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *root, SSmlTableInfo *tinfo) { - cJSON *metric = cJSON_GetObjectItem(root, "metric"); - if (!cJSON_IsString(metric)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - tinfo->sTableNameLen = strlen(metric->valuestring); - if (IS_INVALID_TABLE_LEN(tinfo->sTableNameLen)) { - uError("OTD:0x%" PRIx64 " Metric lenght is 0 or large than 192", info->id); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - - tinfo->sTableName = metric->valuestring; - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsVal) { - int32_t size = cJSON_GetArraySize(root); - if (size != OTD_JSON_SUB_FIELDS_NUM) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *value = cJSON_GetObjectItem(root, "value"); - if (!cJSON_IsNumber(value)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *type = cJSON_GetObjectItem(root, "type"); - if (!cJSON_IsString(type)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - double timeDouble = value->valuedouble; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - - if (timeDouble == 0) { - *tsVal = taosGetTimestampNs(); - return TSDB_CODE_SUCCESS; - } - - if (timeDouble < 0) { - return TSDB_CODE_INVALID_TIMESTAMP; - } - - *tsVal = timeDouble; - size_t typeLen = strlen(type->valuestring); - if (typeLen == 1 && (type->valuestring[0] == 's' || type->valuestring[0] == 'S')) { - // seconds - *tsVal = *tsVal * NANOSECOND_PER_SEC; - timeDouble = timeDouble * NANOSECOND_PER_SEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - } else if (typeLen == 2 && (type->valuestring[1] == 's' || type->valuestring[1] == 'S')) { - switch (type->valuestring[0]) { - case 'm': - case 'M': - // milliseconds - *tsVal = *tsVal * NANOSECOND_PER_MSEC; - timeDouble = timeDouble * NANOSECOND_PER_MSEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - break; - case 'u': - case 'U': - // microseconds - *tsVal = *tsVal * NANOSECOND_PER_USEC; - timeDouble = timeDouble * NANOSECOND_PER_USEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; + int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); + if (index) { + SSmlKv *value = (SSmlKv *)taosArrayGet(metaArray, *index); + if (isTag) { + if (kv->length > value->length) { + value->length = kv->length; } - break; - case 'n': - case 'N': - break; - default: - return TSDB_CODE_TSC_INVALID_JSON; - } - } else { - return TSDB_CODE_TSC_INVALID_JSON; - } - - return TSDB_CODE_SUCCESS; -} - -static uint8_t smlGetTimestampLen(int64_t num) { - uint8_t len = 0; - while ((num /= 10) != 0) { - len++; - } - len++; - return len; -} - -static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) { - // Timestamp must be the first KV to parse - int64_t tsVal = 0; - - cJSON *timestamp = cJSON_GetObjectItem(root, "timestamp"); - if (cJSON_IsNumber(timestamp)) { - // timestamp value 0 indicates current system time - double timeDouble = timestamp->valuedouble; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - - if (timeDouble < 0) { - return TSDB_CODE_INVALID_TIMESTAMP; - } - - uint8_t tsLen = smlGetTimestampLen((int64_t)timeDouble); - tsVal = (int64_t)timeDouble; - if (tsLen == TSDB_TIME_PRECISION_SEC_DIGITS) { - tsVal = tsVal * NANOSECOND_PER_SEC; - timeDouble = timeDouble * NANOSECOND_PER_SEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - } else if (tsLen == TSDB_TIME_PRECISION_MILLI_DIGITS) { - tsVal = tsVal * NANOSECOND_PER_MSEC; - timeDouble = timeDouble * NANOSECOND_PER_MSEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - } else if (timeDouble == 0) { - tsVal = taosGetTimestampNs(); - } else { - return TSDB_CODE_INVALID_TIMESTAMP; - } - } else if (cJSON_IsObject(timestamp)) { - int32_t ret = smlParseTSFromJSONObj(info, timestamp, &tsVal); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " Failed to parse timestamp from JSON Obj", info->id); - return ret; - } - } else { - return TSDB_CODE_TSC_INVALID_JSON; - } - - // add ts to - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) { - return TSDB_CODE_OUT_OF_MEMORY; - } - kv->key = TS; - kv->keyLen = TS_LEN; - kv->i = tsVal; - kv->type = TSDB_DATA_TYPE_TIMESTAMP; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - taosArrayPush(cols, &kv); - return TSDB_CODE_SUCCESS; -} - -static int32_t smlConvertJSONBool(SSmlKv *pVal, char *typeStr, cJSON *value) { - if (strcasecmp(typeStr, "bool") != 0) { - uError("OTD:invalid type(%s) for JSON Bool", typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - pVal->type = TSDB_DATA_TYPE_BOOL; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valueint; - - return TSDB_CODE_SUCCESS; -} - -static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { - // tinyint - if (strcasecmp(typeStr, "i8") == 0 || strcasecmp(typeStr, "tinyint") == 0) { - if (!IS_VALID_TINYINT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(tinyint)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_TINYINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // smallint - if (strcasecmp(typeStr, "i16") == 0 || strcasecmp(typeStr, "smallint") == 0) { - if (!IS_VALID_SMALLINT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(smallint)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_SMALLINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // int - if (strcasecmp(typeStr, "i32") == 0 || strcasecmp(typeStr, "int") == 0) { - if (!IS_VALID_INT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(int)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_INT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // bigint - if (strcasecmp(typeStr, "i64") == 0 || strcasecmp(typeStr, "bigint") == 0) { - pVal->type = TSDB_DATA_TYPE_BIGINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - if (value->valuedouble >= (double)INT64_MAX) { - pVal->i = INT64_MAX; - } else if (value->valuedouble <= (double)INT64_MIN) { - pVal->i = INT64_MIN; - } else { - pVal->i = value->valuedouble; - } - return TSDB_CODE_SUCCESS; - } - // float - if (strcasecmp(typeStr, "f32") == 0 || strcasecmp(typeStr, "float") == 0) { - if (!IS_VALID_FLOAT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(float)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_FLOAT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->f = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // double - if (strcasecmp(typeStr, "f64") == 0 || strcasecmp(typeStr, "double") == 0) { - pVal->type = TSDB_DATA_TYPE_DOUBLE; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->d = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - - // if reach here means type is unsupported - uError("OTD:invalid type(%s) for JSON Number", typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; -} - -static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) { - if (strcasecmp(typeStr, "binary") == 0) { - pVal->type = TSDB_DATA_TYPE_BINARY; - } else if (strcasecmp(typeStr, "nchar") == 0) { - pVal->type = TSDB_DATA_TYPE_NCHAR; - } else { - uError("OTD:invalid type(%s) for JSON String", typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - pVal->length = (int16_t)strlen(value->valuestring); - - if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - if (pVal->type == TSDB_DATA_TYPE_NCHAR && - pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - - pVal->value = value->valuestring; - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseValueFromJSONObj(cJSON *root, SSmlKv *kv) { - int32_t ret = TSDB_CODE_SUCCESS; - int32_t size = cJSON_GetArraySize(root); - - if (size != OTD_JSON_SUB_FIELDS_NUM) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *value = cJSON_GetObjectItem(root, "value"); - if (value == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *type = cJSON_GetObjectItem(root, "type"); - if (!cJSON_IsString(type)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - switch (value->type) { - case cJSON_True: - case cJSON_False: { - ret = smlConvertJSONBool(kv, type->valuestring, value); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - break; - } - case cJSON_Number: { - ret = smlConvertJSONNumber(kv, type->valuestring, value); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - break; - } - case cJSON_String: { - ret = smlConvertJSONString(kv, type->valuestring, value); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + continue; } - break; - } - default: - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { - switch (root->type) { - case cJSON_True: - case cJSON_False: { - kv->type = TSDB_DATA_TYPE_BOOL; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - kv->i = root->valueint; - break; - } - case cJSON_Number: { - kv->type = TSDB_DATA_TYPE_DOUBLE; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - kv->d = root->valuedouble; - break; - } - case cJSON_String: { - /* set default JSON type to binary/nchar according to - * user configured parameter tsDefaultJSONStrType - */ - - char *tsDefaultJSONStrType = "nchar"; // todo - smlConvertJSONString(kv, tsDefaultJSONStrType, root); - break; - } - case cJSON_Object: { - int32_t ret = smlParseValueFromJSONObj(root, kv); - if (ret != TSDB_CODE_SUCCESS) { - uError("OTD:Failed to parse value from JSON Obj"); - return ret; + if (kv->type != value->type) { + smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); + return TSDB_CODE_SML_NOT_SAME_TYPE; + } + + if (IS_VAR_DATA_TYPE(kv->type) && (kv->length > value->length)) { // update string len, if bigger + value->length = kv->length; + } + } else { + size_t tmp = taosArrayGetSize(metaArray); + if (tmp > INT16_MAX) { + uError("too many cols or tags"); + return -1; + } + int16_t size = tmp; + int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); + if (ret == 0) { + taosArrayPush(metaArray, kv); } - break; } - default: - return TSDB_CODE_TSC_INVALID_JSON; } return TSDB_CODE_SUCCESS; } -static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { - if (!cols) return TSDB_CODE_OUT_OF_MEMORY; - cJSON *metricVal = cJSON_GetObjectItem(root, "value"); - if (metricVal == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; +void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) { + for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { + SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i); + taosHashCleanup(kvHash); } - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) { - return TSDB_CODE_OUT_OF_MEMORY; + if (info->parseJsonByLib) { + SSmlLineInfo *key = (SSmlLineInfo *)(tag->key); + if (key != NULL) taosMemoryFree(key->tags); } - taosArrayPush(cols, &kv); + taosMemoryFree(tag->key); + taosArrayDestroy(tag->cols); + taosArrayDestroy(tag->tags); + taosMemoryFree(tag); +} - kv->key = VALUE; - kv->keyLen = VALUE_LEN; - int32_t ret = smlParseValueFromJSON(metricVal, kv); - if (ret != TSDB_CODE_SUCCESS) { - return ret; +void clearColValArray(SArray *pCols) { + int32_t num = taosArrayGetSize(pCols); + for (int32_t i = 0; i < num; ++i) { + SColVal *pCol = taosArrayGet(pCols, i); + if (TSDB_DATA_TYPE_NCHAR == pCol->type) { + taosMemoryFreeClear(pCol->value.pData); + } } - return TSDB_CODE_SUCCESS; } -static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableName, SHashObj *dumplicateKey, - SSmlMsgBuf *msg) { - int32_t ret = TSDB_CODE_SUCCESS; - if (!pKVs) { - return TSDB_CODE_OUT_OF_MEMORY; - } - cJSON *tags = cJSON_GetObjectItem(root, "tags"); - if (tags == NULL || tags->type != cJSON_Object) { - return TSDB_CODE_TSC_INVALID_JSON; - } +void smlDestroyInfo(SSmlHandle *info) { + if (!info) return; + qDestroyQuery(info->pQuery); - size_t childTableNameLen = strlen(tsSmlChildTableName); - int32_t tagNum = cJSON_GetArraySize(tags); - for (int32_t i = 0; i < tagNum; ++i) { - cJSON *tag = cJSON_GetArrayItem(tags, i); - if (tag == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; - } - size_t keyLen = strlen(tag->string); - if (IS_INVALID_COL_LEN(keyLen)) { - uError("OTD:Tag key length is 0 or too large than 64"); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - // check duplicate keys - if (smlCheckDuplicateKey(tag->string, keyLen, dumplicateKey)) { - return TSDB_CODE_TSC_DUP_NAMES; + // destroy info->childTables + NodeList *tmp = info->childTables; + while (tmp) { + if (tmp->data.used) { + smlDestroyTableInfo(info, (SSmlTableInfo *)tmp->data.value); } + NodeList *t = tmp->next; + taosMemoryFree(tmp); + tmp = t; + } - // handle child table name - if (childTableNameLen != 0 && strcmp(tag->string, tsSmlChildTableName) == 0) { - if (!cJSON_IsString(tag)) { - uError("OTD:ID must be JSON string"); - return TSDB_CODE_TSC_INVALID_JSON; - } - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - tstrncpy(childTableName, tag->valuestring, TSDB_TABLE_NAME_LEN); - continue; + // destroy info->superTables + tmp = info->superTables; + while (tmp) { + if (tmp->data.used) { + smlDestroySTableMeta((SSmlSTableMeta *)tmp->data.value); } + NodeList *t = tmp->next; + taosMemoryFree(tmp); + tmp = t; + } - // add kv to SSmlKv - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - taosArrayPush(pKVs, &kv); + // destroy info->pVgHash + taosHashCleanup(info->pVgHash); - // key - kv->keyLen = keyLen; - kv->key = tag->string; + for (int i = 0; i < taosArrayGetSize(info->tagJsonArray); i++) { + cJSON *tags = (cJSON *)taosArrayGetP(info->tagJsonArray, i); + cJSON_Delete(tags); + } + taosArrayDestroy(info->tagJsonArray); - // value - ret = smlParseValueFromJSON(tag, kv); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + taosArrayDestroy(info->preLineTagKV); + taosArrayDestroy(info->maxTagKVs); + taosArrayDestroy(info->preLineColKV); + + if (!info->dataFormat) { + for (int i = 0; i < info->lineNum; i++) { + taosArrayDestroy(info->lines[i].colArray); + if (info->parseJsonByLib) { + taosMemoryFree(info->lines[i].tags); + } } + taosMemoryFree(info->lines); } - return ret; + cJSON_Delete(info->root); + taosMemoryFreeClear(info); } -static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo *tinfo, SArray *cols) { - int32_t ret = TSDB_CODE_SUCCESS; - - if (!cJSON_IsObject(root)) { - uError("OTD:0x%" PRIx64 " data point needs to be JSON object", info->id); - return TSDB_CODE_TSC_INVALID_JSON; +SSmlHandle *smlBuildSmlInfo(TAOS *taos) { + int32_t code = TSDB_CODE_SUCCESS; + SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle)); + if (NULL == info) { + return NULL; } - - int32_t size = cJSON_GetArraySize(root); - // outmost json fields has to be exactly 4 - if (size != OTD_JSON_FIELDS_NUM) { - uError("OTD:0x%" PRIx64 " Invalid number of JSON fields in data point %d", info->id, size); - return TSDB_CODE_TSC_INVALID_JSON; + if (taos != NULL) { + info->taos = acquireTscObj(*(int64_t *)taos); + code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); + goto cleanup; + } } - // Parse metric - ret = smlParseMetricFromJSON(info, root, tinfo); - if (ret != TSDB_CODE_SUCCESS) { - uError("OTD:0x%" PRIx64 " Unable to parse metric from JSON payload", info->id); - return ret; - } - uDebug("OTD:0x%" PRIx64 " Parse metric from JSON payload finished", info->id); + info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + info->id = smlGenId(); + info->pQuery = smlInitHandle(); + info->dataFormat = true; - // Parse timestamp - ret = smlParseTSFromJSON(info, root, cols); - if (ret) { - uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); - return ret; - } - uDebug("OTD:0x%" PRIx64 " Parse timestamp from JSON payload finished", info->id); + info->tagJsonArray = taosArrayInit(8, POINTER_BYTES); + info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); + info->maxTagKVs = taosArrayInit(8, sizeof(SSmlKv)); + info->preLineColKV = taosArrayInit(8, sizeof(SSmlKv)); - // Parse metric value - ret = smlParseColsFromJSON(root, cols); - if (ret) { - uError("OTD:0x%" PRIx64 " Unable to parse metric value from JSON payload", info->id); - return ret; + if (NULL == info->pVgHash) { + uError("create SSmlHandle failed"); + goto cleanup; } - uDebug("OTD:0x%" PRIx64 " Parse metric value from JSON payload finished", info->id); - // Parse tags - ret = smlParseTagsFromJSON(root, tinfo->tags, tinfo->childTableName, info->dumplicateKey, &info->msgBuf); - if (ret) { - uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); - return ret; + return info; + +cleanup: + smlDestroyInfo(info); + return NULL; +} + +static int32_t smlPushCols(SArray *colsArray, SArray *cols) { + SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (!kvHash) { + uError("SML:smlDealCols failed to allocate memory"); + return TSDB_CODE_OUT_OF_MEMORY; + } + for (size_t i = 0; i < taosArrayGetSize(cols); i++) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); } - uDebug("OTD:0x%" PRIx64 " Parse tags from JSON payload finished", info->id); + taosArrayPush(colsArray, &kvHash); return TSDB_CODE_SUCCESS; } -/************* TSDB_SML_JSON_PROTOCOL function end **************/ - -static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql, const int len) { - SSmlLineInfo elements = {0}; - uDebug("SML:0x%" PRIx64 " smlParseInfluxLine raw:%d, len:%d, sql:%s", info->id, info->isRawLine, len, - (info->isRawLine ? "rawdata" : sql)); - int ret = smlParseInfluxString(sql, sql + len, &elements, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseInfluxLine failed", info->id); - return ret; - } +static int32_t smlParseLineBottom(SSmlHandle *info) { + if (info->dataFormat) return TSDB_CODE_SUCCESS; - SArray *cols = NULL; - if (info->dataFormat) { // if dataFormat, cols need new memory to save data - cols = taosArrayInit(16, POINTER_BYTES); - if (cols == NULL) { - uError("SML:0x%" PRIx64 " smlParseInfluxLine failed to allocate memory", info->id); - return TSDB_CODE_OUT_OF_MEMORY; - } - } else { // if dataFormat is false, cols do not need to save data, there is another new memory to save data - cols = info->colsContainer; - } - - ret = smlParseTS(info, elements.timestamp, elements.timestampLen, cols); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseTS failed", info->id); - if (info->dataFormat) taosArrayDestroy(cols); - return ret; - } - ret = smlParseCols(elements.cols, elements.colsLen, cols, NULL, false, info->dumplicateKey, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseCols parse cloums fields failed", info->id); - smlDestroyCols(cols); - if (info->dataFormat) taosArrayDestroy(cols); - return ret; - } - - bool hasTable = true; - SSmlTableInfo *tinfo = NULL; - SSmlTableInfo **oneTable = - (SSmlTableInfo **)taosHashGet(info->childTables, elements.measure, elements.measureTagsLen); - if (!oneTable) { - tinfo = smlBuildTableInfo(); - if (!tinfo) { - smlDestroyCols(cols); - if (info->dataFormat) taosArrayDestroy(cols); - return TSDB_CODE_OUT_OF_MEMORY; + for (int32_t i = 0; i < info->lineNum; i++) { + SSmlLineInfo *elements = info->lines + i; + SSmlTableInfo *tinfo = NULL; + if (info->protocol == TSDB_SML_LINE_PROTOCOL) { + tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements->measure, elements->measureTagsLen, NULL); + } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { + tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); + } else { + tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); } - taosHashPut(info->childTables, elements.measure, elements.measureTagsLen, &tinfo, POINTER_BYTES); - oneTable = &tinfo; - hasTable = false; - } - ret = smlDealCols(*oneTable, info->dataFormat, cols); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - if (!hasTable) { - ret = smlParseCols(elements.tags, elements.tagsLen, (*oneTable)->tags, (*oneTable)->childTableName, true, - info->dumplicateKey, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseCols parse tag fields failed", info->id); - return ret; + if (tinfo == NULL) { + uError("SML:0x%" PRIx64 "get oneTable failed, line num:%d", info->id, i); + smlBuildInvalidDataMsg(&info->msgBuf, "get oneTable failed", elements->measure); + return TSDB_CODE_SML_INVALID_DATA; } - if (taosArrayGetSize((*oneTable)->tags) > TSDB_MAX_TAGS) { + if (taosArrayGetSize(tinfo->tags) > TSDB_MAX_TAGS) { smlBuildInvalidDataMsg(&info->msgBuf, "too many tags than 128", NULL); return TSDB_CODE_PAR_INVALID_TAGS_NUM; } - if (taosArrayGetSize(cols) + taosArrayGetSize((*oneTable)->tags) > TSDB_MAX_COLUMNS) { + if (taosArrayGetSize(elements->colArray) + taosArrayGetSize(tinfo->tags) > TSDB_MAX_COLUMNS) { smlBuildInvalidDataMsg(&info->msgBuf, "too many columns than 4096", NULL); return TSDB_CODE_PAR_TOO_MANY_COLUMNS; } - (*oneTable)->sTableName = elements.measure; - (*oneTable)->sTableNameLen = elements.measureLen; - if (strlen((*oneTable)->childTableName) == 0) { - RandTableName rName = {(*oneTable)->tags, (*oneTable)->sTableName, (uint8_t)(*oneTable)->sTableNameLen, - (*oneTable)->childTableName, 0}; - - buildChildTableName(&rName); - (*oneTable)->uid = rName.uid; - } else { - (*oneTable)->uid = *(uint64_t *)((*oneTable)->childTableName); - } - } - - SSmlSTableMeta **tableMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, elements.measure, elements.measureLen); - if (tableMeta) { // update meta - ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, cols, &info->msgBuf); - if (!hasTable && ret == TSDB_CODE_SUCCESS) { - ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, (*oneTable)->tags, &info->msgBuf); - } + int ret = smlPushCols(tinfo->cols, elements->colArray); if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); return ret; } - } else { - SSmlSTableMeta *meta = smlBuildSTableMeta(); - smlInsertMeta(meta->tagHash, meta->tags, (*oneTable)->tags); - smlInsertMeta(meta->colHash, meta->cols, cols); - taosHashPut(info->superTables, elements.measure, elements.measureLen, &meta, POINTER_BYTES); - } - - if (!info->dataFormat) { - taosArrayClear(info->colsContainer); - } - taosHashClear(info->dumplicateKey); - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseTelnetLine(SSmlHandle *info, void *data, const int len) { - int ret = TSDB_CODE_SUCCESS; - SSmlTableInfo *tinfo = smlBuildTableInfo(); - if (!tinfo) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - SArray *cols = taosArrayInit(16, POINTER_BYTES); - if (cols == NULL) { - uError("SML:0x%" PRIx64 " smlParseTelnetLine failed to allocate memory", info->id); - return TSDB_CODE_OUT_OF_MEMORY; - } - - if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - ret = smlParseTelnetString(info, (const char *)data, (char *)data + len, tinfo, cols); - } else if (info->protocol == TSDB_SML_JSON_PROTOCOL) { - ret = smlParseJSONString(info, (cJSON *)data, tinfo, cols); - } else { - ASSERT(0); - } - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseTelnetLine failed", info->id); - smlDestroyTableInfo(info, tinfo); - smlDestroyCols(cols); - taosArrayDestroy(cols); - return ret; - } - - if (taosArrayGetSize(tinfo->tags) <= 0 || taosArrayGetSize(tinfo->tags) > TSDB_MAX_TAGS) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalidate tags length:[1,128]", NULL); - smlDestroyTableInfo(info, tinfo); - smlDestroyCols(cols); - taosArrayDestroy(cols); - return TSDB_CODE_PAR_INVALID_TAGS_NUM; - } - taosHashClear(info->dumplicateKey); - - if (strlen(tinfo->childTableName) == 0) { - RandTableName rName = {tinfo->tags, tinfo->sTableName, (uint8_t)tinfo->sTableNameLen, tinfo->childTableName, 0}; - buildChildTableName(&rName); - tinfo->uid = rName.uid; - } else { - tinfo->uid = *(uint64_t *)(tinfo->childTableName); // generate uid by name simple - } - bool hasTable = true; - SSmlTableInfo **oneTable = - (SSmlTableInfo **)taosHashGet(info->childTables, tinfo->childTableName, strlen(tinfo->childTableName)); - if (!oneTable) { - taosHashPut(info->childTables, tinfo->childTableName, strlen(tinfo->childTableName), &tinfo, POINTER_BYTES); - oneTable = &tinfo; - hasTable = false; - } else { - smlDestroyTableInfo(info, tinfo); - } + SSmlSTableMeta *tableMeta = + (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + if (tableMeta) { // update meta + ret = smlUpdateMeta(tableMeta->colHash, tableMeta->cols, elements->colArray, false, &info->msgBuf); + if (ret == TSDB_CODE_SUCCESS) { + ret = smlUpdateMeta(tableMeta->tagHash, tableMeta->tags, tinfo->tags, true, &info->msgBuf); + } + if (ret != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); + return ret; + } + } else { + // ret = smlJudgeDupColName(elements->colArray, tinfo->tags, &info->msgBuf); + // if (ret != TSDB_CODE_SUCCESS) { + // uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); + // return ret; + // } - taosArrayPush((*oneTable)->cols, &cols); - SSmlSTableMeta **tableMeta = - (SSmlSTableMeta **)taosHashGet(info->superTables, (*oneTable)->sTableName, (*oneTable)->sTableNameLen); - if (tableMeta) { // update meta - ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, cols, &info->msgBuf); - if (!hasTable && ret == TSDB_CODE_SUCCESS) { - ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, (*oneTable)->tags, &info->msgBuf); - } - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); - return ret; + SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); + smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); + smlInsertMeta(meta->colHash, meta->cols, elements->colArray); + nodeListSet(&info->superTables, elements->measure, elements->measureLen, meta, NULL); } - } else { - SSmlSTableMeta *meta = smlBuildSTableMeta(); - smlInsertMeta(meta->tagHash, meta->tags, (*oneTable)->tags); - smlInsertMeta(meta->colHash, meta->cols, cols); - taosHashPut(info->superTables, (*oneTable)->sTableName, (*oneTable)->sTableNameLen, &meta, POINTER_BYTES); } return TSDB_CODE_SUCCESS; } -static int32_t smlParseJSON(SSmlHandle *info, char *payload) { - int32_t payloadNum = 0; - int32_t ret = TSDB_CODE_SUCCESS; - - if (payload == NULL) { - uError("SML:0x%" PRIx64 " empty JSON Payload", info->id); - return TSDB_CODE_TSC_INVALID_JSON; - } - - info->root = cJSON_Parse(payload); - if (info->root == NULL) { - uError("SML:0x%" PRIx64 " parse json failed:%s", info->id, payload); - return TSDB_CODE_TSC_INVALID_JSON; - } - // multiple data points must be sent in JSON array - if (cJSON_IsObject(info->root)) { - payloadNum = 1; - } else if (cJSON_IsArray(info->root)) { - payloadNum = cJSON_GetArraySize(info->root); - } else { - uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); - ret = TSDB_CODE_TSC_INVALID_JSON; - goto end; - } - - for (int32_t i = 0; i < payloadNum; ++i) { - cJSON *dataPoint = (payloadNum == 1 && cJSON_IsObject(info->root)) ? info->root : cJSON_GetArrayItem(info->root, i); - ret = smlParseTelnetLine(info, dataPoint, -1); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); - goto end; - } - } - -end: - return ret; -} - static int32_t smlInsertData(SSmlHandle *info) { int32_t code = TSDB_CODE_SUCCESS; - SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); - while (oneTable) { - SSmlTableInfo *tableData = *oneTable; + NodeList *tmp = info->childTables; + while (tmp) { + SSmlTableInfo *tableData = (SSmlTableInfo *)tmp->data.value; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); @@ -2330,45 +1230,39 @@ static int32_t smlInsertData(SSmlHandle *info) { } taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg)); - SSmlSTableMeta **pMeta = - (SSmlSTableMeta **)taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); - ASSERT(NULL != pMeta && NULL != *pMeta); + SSmlSTableMeta *pMeta = + (SSmlSTableMeta *)nodeListGet(info->superTables, tableData->sTableName, tableData->sTableNameLen, NULL); + if (unlikely(NULL == pMeta || NULL == pMeta->tableMeta)) { + uError("SML:0x%" PRIx64 " NULL == pMeta. table name: %s", info->id, tableData->childTableName); + return TSDB_CODE_SML_INTERNAL_ERROR; + } // use tablemeta of stable to save vgid and uid of child table - (*pMeta)->tableMeta->vgId = vg.vgId; - (*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid + pMeta->tableMeta->vgId = vg.vgId; + pMeta->tableMeta->uid = tableData->uid; // one table merge data block together according uid - code = smlBindData(info->exec, tableData->tags, (*pMeta)->cols, tableData->cols, info->dataFormat, - (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, - info->ttl, info->msgBuf.buf, info->msgBuf.len); + code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, pMeta->cols, tableData->cols, pMeta->tableMeta, + tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, info->ttl, + info->msgBuf.buf, info->msgBuf.len); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBindData failed", info->id); return code; } - oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); + tmp = tmp->next; } - code = smlBuildOutput(info->exec, info->pVgHash); + code = smlBuildOutput(info->pQuery, info->pVgHash); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildOutput failed", info->id); return code; } info->cost.insertRpcTime = taosGetTimestampUs(); - // launchQueryImpl(info->pRequest, info->pQuery, false, NULL); - // info->affectedRows = taos_affected_rows(info->pRequest); - // return info->pRequest->code; - SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary; atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); - SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper)); - if (pWrapper == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pWrapper->pRequest = info->pRequest; - launchAsyncQuery(info->pRequest, info->pQuery, NULL, pWrapper); - return TSDB_CODE_SUCCESS; + launchQueryImpl(info->pRequest, info->pQuery, true, NULL); + return info->pRequest->code; } static void smlPrintStatisticInfo(SSmlHandle *info) { @@ -2384,6 +1278,46 @@ static void smlPrintStatisticInfo(SSmlHandle *info) { info->cost.endTime - info->cost.parseTime); } +int32_t smlClearForRerun(SSmlHandle *info) { + info->reRun = false; + // clear info->childTables + NodeList *pList = info->childTables; + while (pList) { + if (pList->data.used) { + smlDestroyTableInfo(info, (SSmlTableInfo *)pList->data.value); + pList->data.used = false; + } + pList = pList->next; + } + + // clear info->superTables + pList = info->superTables; + while (pList) { + if (pList->data.used) { + smlDestroySTableMeta((SSmlSTableMeta *)pList->data.value); + pList->data.used = false; + } + pList = pList->next; + } + + if (!info->dataFormat) { + if (unlikely(info->lines != NULL)) { + uError("SML:0x%" PRIx64 " info->lines != NULL", info->id); + return TSDB_CODE_SML_INVALID_DATA; + } + info->lines = (SSmlLineInfo *)taosMemoryCalloc(info->lineNum, sizeof(SSmlLineInfo)); + } + + memset(&info->preLine, 0, sizeof(SSmlLineInfo)); + info->currSTableMeta = NULL; + info->currTableDataCtx = NULL; + + SVnodeModifyOpStmt *stmt = (SVnodeModifyOpStmt *)(info->pQuery->pRoot); + stmt->freeHashFunc(stmt->pTableBlockHashObj); + stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + return TSDB_CODE_SUCCESS; +} + static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { int32_t code = TSDB_CODE_SUCCESS; if (info->protocol == TSDB_SML_JSON_PROTOCOL) { @@ -2399,7 +1333,9 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char return code; } - for (int32_t i = 0; i < numLines; ++i) { + char *oldRaw = rawLine; + int32_t i = 0; + while (i < numLines) { char *tmp = NULL; int len = 0; if (lines) { @@ -2418,18 +1354,42 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char } } + uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, len:%d, sql:%s", info->id, info->isRawLine, len, + (info->isRawLine ? "rawdata" : tmp)); + if (info->protocol == TSDB_SML_LINE_PROTOCOL) { - code = smlParseInfluxLine(info, tmp, len); + if (info->dataFormat) { + SSmlLineInfo element = {0}; + code = smlParseInfluxString(info, tmp, tmp + len, &element); + } else { + code = smlParseInfluxString(info, tmp, tmp + len, info->lines + i); + } } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - code = smlParseTelnetLine(info, tmp, len); + if (info->dataFormat) { + SSmlLineInfo element = {0}; + code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, &element); + } else { + code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, info->lines + i); + } } else { - ASSERT(0); + code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; } if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); return code; } + if (info->reRun) { + i = 0; + rawLine = oldRaw; + code = smlClearForRerun(info); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + continue; + } + i++; } + return code; } @@ -2444,17 +1404,22 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL uError("SML:0x%" PRIx64 " smlParseLine error : %s", info->id, tstrerror(code)); return code; } + code = smlParseLineBottom(info); + if (code != 0) { + uError("SML:0x%" PRIx64 " smlParseLineBottom error : %s", info->id, tstrerror(code)); + return code; + } - info->cost.lineNum = numLines; - info->cost.numOfSTables = taosHashGetSize(info->superTables); - info->cost.numOfCTables = taosHashGetSize(info->childTables); + info->cost.lineNum = info->lineNum; + info->cost.numOfSTables = nodeListSize(info->superTables); + info->cost.numOfCTables = nodeListSize(info->childTables); info->cost.schemaTime = taosGetTimestampUs(); do { code = smlModifyDBSchemas(info); if (code == 0) break; - } while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES); + } while (retryNum++ < nodeListSize(info->superTables) * MAX_RETRY_TIMES); if (code != 0) { uError("SML:0x%" PRIx64 " smlModifyDBSchemas error : %s", info->id, tstrerror(code)); @@ -2471,92 +1436,42 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL return code; } -static int32_t isSchemalessDb(STscObj *taos, SRequestObj *request) { - // SCatalog *catalog = NULL; - // int32_t code = catalogGetHandle(((STscObj *)taos)->pAppInfo->clusterId, &catalog); - // if (code != TSDB_CODE_SUCCESS) { - // uError("SML get catalog error %d", code); - // return code; - // } - // - // SName name; - // tNameSetDbName(&name, taos->acctId, taos->db, strlen(taos->db)); - // char dbFname[TSDB_DB_FNAME_LEN] = {0}; - // tNameGetFullDbName(&name, dbFname); - // SDbCfgInfo pInfo = {0}; - // - // SRequestConnInfo conn = {0}; - // conn.pTrans = taos->pAppInfo->pTransporter; - // conn.requestId = request->requestId; - // conn.requestObjRefId = request->self; - // conn.mgmtEps = getEpSet_s(&taos->pAppInfo->mgmtEp); - // - // code = catalogGetDBCfg(catalog, &conn, dbFname, &pInfo); - // if (code != TSDB_CODE_SUCCESS) { - // return code; - // } - // taosArrayDestroy(pInfo.pRetensions); - // - // if (!pInfo.schemaless) { - // return TSDB_CODE_SML_INVALID_DB_CONF; - // } - return TSDB_CODE_SUCCESS; -} - -static void smlInsertCallback(void *param, void *res, int32_t code) { - SRequestObj *pRequest = (SRequestObj *)res; - SSmlHandle *info = (SSmlHandle *)param; - int32_t rows = taos_affected_rows(pRequest); - - uDebug("SML:0x%" PRIx64 " result. code:%d, msg:%s", info->id, pRequest->code, pRequest->msgBuf); - Params *pParam = info->params; - // lock - taosThreadSpinLock(&pParam->lock); - pParam->cnt++; - if (code != TSDB_CODE_SUCCESS) { - pParam->request->code = code; - pParam->request->body.resInfo.numOfRows += rows; - } else { - pParam->request->body.resInfo.numOfRows += info->affectedRows; +TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, char *rawLineEnd, int numLines, + int protocol, int precision, int32_t ttl, int64_t reqid) { + int32_t code = TSDB_CODE_SUCCESS; + if (NULL == taos) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return NULL; } - // unlock - taosThreadSpinUnlock(&pParam->lock); - if (pParam->cnt == pParam->total) { - tsem_post(&pParam->sem); + SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); + if (request == NULL) { + uError("SML:taos_schemaless_insert error request is null"); + return NULL; } - uDebug("SML:0x%" PRIx64 " insert finished, code: %d, rows: %d, total: %d", info->id, code, rows, info->affectedRows); - info->cost.endTime = taosGetTimestampUs(); - info->cost.code = code; - smlPrintStatisticInfo(info); - smlDestroyInfo(info); -} -TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char *rawLine, char *rawLineEnd, - int numLines, int protocol, int precision, int32_t ttl) { - int batchs = 0; - STscObj *pTscObj = request->pTscObj; + SSmlHandle *info = smlBuildSmlInfo(taos); + if (info == NULL) { + request->code = TSDB_CODE_OUT_OF_MEMORY; + uError("SML:taos_schemaless_insert error SSmlHandle is null"); + return (TAOS_RES *)request; + } + info->pRequest = request; + info->isRawLine = rawLine != NULL; + info->ttl = ttl; + info->precision = precision; + info->protocol = (TSDB_SML_PROTOCOL_TYPE)protocol; + info->msgBuf.buf = info->pRequest->msgBuf; + info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; + info->lineNum = numLines; - pTscObj->schemalessType = 1; SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - - Params params = {0}; - params.request = request; - tsem_init(¶ms.sem, 0, 0); - taosThreadSpinInit(&(params.lock), 0); - if (request->pDb == NULL) { request->code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; smlBuildInvalidDataMsg(&msg, "Database not specified", NULL); goto end; } - if (isSchemalessDb(pTscObj, request) != TSDB_CODE_SUCCESS) { - request->code = TSDB_CODE_SML_INVALID_DB_CONF; - smlBuildInvalidDataMsg(&msg, "Cannot write data to a non schemaless database", NULL); - goto end; - } - if (protocol < TSDB_SML_LINE_PROTOCOL || protocol > TSDB_SML_JSON_PROTOCOL) { request->code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; smlBuildInvalidDataMsg(&msg, "protocol invalidate", NULL); @@ -2578,65 +1493,14 @@ TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char goto end; } - batchs = ceil(((double)numLines) / tsSmlBatchSize); - params.total = batchs; - for (int i = 0; i < batchs; ++i) { - SRequestObj *req = (SRequestObj *)createRequest(pTscObj->id, TSDB_SQL_INSERT, 0); - if (!req) { - request->code = TSDB_CODE_OUT_OF_MEMORY; - uError("SML:taos_schemaless_insert error request is null"); - goto end; - } - SSmlHandle *info = smlBuildSmlInfo(pTscObj, req, (SMLProtocolType)protocol, precision); - if (!info) { - request->code = TSDB_CODE_OUT_OF_MEMORY; - uError("SML:taos_schemaless_insert error SSmlHandle is null"); - goto end; - } - - info->isRawLine = (rawLine == NULL); - info->ttl = ttl; - - int32_t perBatch = tsSmlBatchSize; - - if (numLines > perBatch) { - numLines -= perBatch; - } else { - perBatch = numLines; - numLines = 0; - } - - info->params = ¶ms; - info->affectedRows = perBatch; - info->pRequest->body.queryFp = smlInsertCallback; - info->pRequest->body.param = info; - int32_t code = smlProcess(info, lines, rawLine, rawLineEnd, perBatch); - if (lines) { - lines += perBatch; - } - if (rawLine) { - int num = 0; - while (rawLine < rawLineEnd) { - if (*(rawLine++) == '\n') { - num++; - } - if (num == perBatch) { - break; - } - } - } - if (code != TSDB_CODE_SUCCESS) { - info->pRequest->body.queryFp(info, req, code); - } - } - tsem_wait(¶ms.sem); + code = smlProcess(info, lines, rawLine, rawLineEnd, numLines); + request->code = code; + info->cost.endTime = taosGetTimestampUs(); + info->cost.code = code; + // smlPrintStatisticInfo(info); end: - taosThreadSpinDestroy(¶ms.lock); - tsem_destroy(¶ms.sem); - // ((STscObj *)taos)->schemalessType = 0; - pTscObj->schemalessType = 1; - uDebug("resultend:%s", request->msgBuf); + smlDestroyInfo(info); return (TAOS_RES *)request; } @@ -2661,25 +1525,7 @@ end: TAOS_RES *taos_schemaless_insert_ttl_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int32_t ttl, int64_t reqid) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); - if (!request) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } - - if (!lines) { - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&msg, "lines is null", NULL); - return (TAOS_RES *)request; - } - - return taos_schemaless_insert_inner(request, lines, NULL, NULL, numLines, protocol, precision, ttl); + return taos_schemaless_insert_inner(taos, lines, NULL, NULL, numLines, protocol, precision, ttl, reqid); } TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) { @@ -2699,24 +1545,6 @@ TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLi TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int32_t ttl, int64_t reqid) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); - if (!request) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } - - if (!lines || len <= 0) { - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&msg, "lines is null", NULL); - return (TAOS_RES *)request; - } - int numLines = 0; *totalRows = 0; char *tmp = lines; @@ -2729,7 +1557,7 @@ TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int tmp = lines + i + 1; } } - return taos_schemaless_insert_inner(request, NULL, lines, lines + len, numLines, protocol, precision, ttl); + return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, *totalRows, protocol, precision, ttl, reqid); } TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c new file mode 100644 index 0000000000000000000000000000000000000000..e6b71c4aaa74eb8df1af8cfa56150b57fe4f9fe4 --- /dev/null +++ b/source/client/src/clientSmlJson.c @@ -0,0 +1,1220 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include "clientSml.h" + +#define OTD_JSON_SUB_FIELDS_NUM 2 + +#define JUMP_JSON_SPACE(start) \ +while(*(start)){\ + if(unlikely(*(start) > 32))\ + break;\ + else\ + (start)++;\ + } + +//SArray *smlJsonParseTags(char *start, char *end){ +// SArray *tags = taosArrayInit(4, sizeof(SSmlKv)); +// while(start < end){ +// SSmlKv kv = {0}; +// kv.type = TSDB_DATA_TYPE_NCHAR; +// bool isInQuote = false; +// while(start < end){ +// if(unlikely(!isInQuote && *start == '"')){ +// start++; +// kv.key = start; +// isInQuote = true; +// continue; +// } +// if(unlikely(isInQuote && *start == '"')){ +// kv.keyLen = start - kv.key; +// start++; +// break; +// } +// start++; +// } +// bool hasColon = false; +// while(start < end){ +// if(unlikely(!hasColon && *start == ':')){ +// start++; +// hasColon = true; +// continue; +// } +// if(unlikely(hasColon && kv.value == NULL && (*start > 32 && *start != '"'))){ +// kv.value = start; +// start++; +// continue; +// } +// +// if(unlikely(hasColon && kv.value != NULL && (*start == '"' || *start == ',' || *start == '}'))){ +// kv.length = start - kv.value; +// taosArrayPush(tags, &kv); +// start++; +// break; +// } +// start++; +// } +// } +// return tags; +//} + +//static int32_t smlParseTagsFromJSON(SSmlHandle *info, SSmlLineInfo *elements) { +// int32_t ret = TSDB_CODE_SUCCESS; +// +// if(is_same_child_table_telnet(elements, &info->preLine) == 0){ +// return TSDB_CODE_SUCCESS; +// } +// +// bool isSameMeasure = IS_SAME_SUPER_TABLE; +// +// int cnt = 0; +// SArray *preLineKV = info->preLineTagKV; +// bool isSuperKVInit = true; +// SArray *superKV = NULL; +// if(info->dataFormat){ +// if(unlikely(!isSameMeasure)){ +// SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); +// +// if(unlikely(sMeta == NULL)){ +// sMeta = smlBuildSTableMeta(info->dataFormat); +// STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); +// sMeta->tableMeta = pTableMeta; +// if(pTableMeta == NULL){ +// info->dataFormat = false; +// info->reRun = true; +// return TSDB_CODE_SUCCESS; +// } +// nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL); +// } +// info->currSTableMeta = sMeta->tableMeta; +// superKV = sMeta->tags; +// +// if(unlikely(taosArrayGetSize(superKV) == 0)){ +// isSuperKVInit = false; +// } +// taosArraySetSize(preLineKV, 0); +// } +// }else{ +// taosArraySetSize(preLineKV, 0); +// } +// +// SArray *tags = smlJsonParseTags(elements->tags, elements->tags + elements->tagsLen); +// int32_t tagNum = taosArrayGetSize(tags); +// if (tagNum == 0) { +// uError("SML:tag is empty:%s", elements->tags) +// taosArrayDestroy(tags); +// return TSDB_CODE_SML_INVALID_DATA; +// } +// for (int32_t i = 0; i < tagNum; ++i) { +// SSmlKv kv = *(SSmlKv*)taosArrayGet(tags, i); +// +// if(info->dataFormat){ +// if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ +// info->dataFormat = false; +// info->reRun = true; +// taosArrayDestroy(tags); +// return TSDB_CODE_SUCCESS; +// } +// +// if(isSameMeasure){ +// if(unlikely(cnt >= taosArrayGetSize(preLineKV))) { +// info->dataFormat = false; +// info->reRun = true; +// taosArrayDestroy(tags); +// return TSDB_CODE_SUCCESS; +// } +// SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); +// if(unlikely(kv.length > preKV->length)){ +// preKV->length = kv.length; +// SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); +// if(unlikely(NULL == tableMeta)){ +// uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); +// return TSDB_CODE_SML_INTERNAL_ERROR; +// } +// +// SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt); +// oldKV->length = kv.length; +// info->needModifySchema = true; +// } +// if(unlikely(!IS_SAME_KEY)){ +// info->dataFormat = false; +// info->reRun = true; +// taosArrayDestroy(tags); +// return TSDB_CODE_SUCCESS; +// } +// }else{ +// if(isSuperKVInit){ +// if(unlikely(cnt >= taosArrayGetSize(superKV))) { +// info->dataFormat = false; +// info->reRun = true; +// taosArrayDestroy(tags); +// return TSDB_CODE_SUCCESS; +// } +// SSmlKv *preKV = (SSmlKv *)taosArrayGet(superKV, cnt); +// if(unlikely(kv.length > preKV->length)) { +// preKV->length = kv.length; +// }else{ +// kv.length = preKV->length; +// } +// info->needModifySchema = true; +// +// if(unlikely(!IS_SAME_KEY)){ +// info->dataFormat = false; +// info->reRun = true; +// taosArrayDestroy(tags); +// return TSDB_CODE_SUCCESS; +// } +// }else{ +// taosArrayPush(superKV, &kv); +// } +// taosArrayPush(preLineKV, &kv); +// } +// }else{ +// taosArrayPush(preLineKV, &kv); +// } +// cnt++; +// } +// taosArrayDestroy(tags); +// +// SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); +// if (unlikely(tinfo == NULL)) { +// tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); +// if (unlikely(!tinfo)) { +// return TSDB_CODE_OUT_OF_MEMORY; +// } +// tinfo->tags = taosArrayDup(preLineKV, NULL); +// +// smlSetCTableName(tinfo); +// if (info->dataFormat) { +// info->currSTableMeta->uid = tinfo->uid; +// tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); +// if (tinfo->tableDataCtx == NULL) { +// smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); +// return TSDB_CODE_SML_INVALID_DATA; +// } +// } +// +// SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); +// *key = *elements; +// tinfo->key = key; +// nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet); +// } +// if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; +// +// return ret; +//} + +static char* smlJsonGetObj(char *payload){ + int leftBracketCnt = 0; + while(*payload) { + if (unlikely(*payload == '{')) { + leftBracketCnt++; + payload++; + continue; + } + if (unlikely(*payload == '}')) { + leftBracketCnt--; + payload++; + if (leftBracketCnt == 0) { + return payload; + } else if (leftBracketCnt < 0) { + return NULL; + } + continue; + } + payload++; + } + return NULL; +} + +int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset){ + int index = 0; + while(*(*start)){ + if((*start)[0] != '"'){ + (*start)++; + continue; + } + + if(unlikely(index >= OTD_JSON_FIELDS_NUM)) { + uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start) + break; + } + + char *sTmp = *start; + if((*start)[1] == 'm' && (*start)[2] == 'e' && (*start)[3] == 't' + && (*start)[4] == 'r' && (*start)[5] == 'i' && (*start)[6] == 'c' && (*start)[7] == '"'){ + + (*start) += 8; + bool isInQuote = false; + while(*(*start)){ + if(unlikely(!isInQuote && *(*start) == '"')){ + (*start)++; + offset[index++] = *start - sTmp; + element->measure = (*start); + isInQuote = true; + continue; + } + if(unlikely(isInQuote && *(*start) == '"')){ + element->measureLen = (*start) - element->measure; + break; + } + (*start)++; + } + }else if((*start)[1] == 't' && (*start)[2] == 'i' && (*start)[3] == 'm' + && (*start)[4] == 'e' && (*start)[5] == 's' && (*start)[6] == 't' + && (*start)[7] == 'a' && (*start)[8] == 'm' && (*start)[9] == 'p' && (*start)[10] == '"'){ + + (*start) += 11; + bool hasColon = false; + while(*(*start)){ + if(unlikely(!hasColon && *(*start) == ':')){ + (*start)++; + JUMP_JSON_SPACE((*start)) + offset[index++] = *start - sTmp; + element->timestamp = (*start); + hasColon = true; + continue; + } + if(unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32))){ + element->timestampLen = (*start) - element->timestamp; + break; + } + (*start)++; + } + }else if((*start)[1] == 'v' && (*start)[2] == 'a' && (*start)[3] == 'l' + && (*start)[4] == 'u' && (*start)[5] == 'e' && (*start)[6] == '"'){ + + (*start) += 7; + + bool hasColon = false; + while(*(*start)){ + if(unlikely(!hasColon && *(*start) == ':')){ + (*start)++; + JUMP_JSON_SPACE((*start)) + offset[index++] = *start - sTmp; + element->cols = (*start); + hasColon = true; + continue; + } + if(unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32))){ + element->colsLen = (*start) - element->cols; + break; + } + (*start)++; + } + }else if((*start)[1] == 't' && (*start)[2] == 'a' && (*start)[3] == 'g' + && (*start)[4] == 's' && (*start)[5] == '"'){ + (*start) += 6; + + while(*(*start)){ + if(unlikely(*(*start) == ':')){ + (*start)++; + JUMP_JSON_SPACE((*start)) + offset[index++] = *start - sTmp; + element->tags = (*start); + char* tmp = smlJsonGetObj((*start)); + if(tmp){ + element->tagsLen = tmp - (*start); + *start = tmp; + } + break; + } + (*start)++; + } + } + if(*(*start) == '\0'){ + break; + } + if(*(*start) == '}'){ + (*start)++; + break; + } + (*start)++; + } + + if(unlikely(index != OTD_JSON_FIELDS_NUM) || element->tags == NULL || element->cols == NULL || element->measure == NULL || element->timestamp == NULL) { + uError("elements != %d or element parse null", OTD_JSON_FIELDS_NUM) + return -1; + } + return 0; +} + +int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset){ + int index = 0; + while(*(*start)){ + if((*start)[0] != '"'){ + (*start)++; + continue; + } + + if(unlikely(index >= OTD_JSON_FIELDS_NUM)) { + uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start) + break; + } + + if((*start)[1] == 'm'){ + (*start) += offset[index++]; + element->measure = *start; + while(*(*start)){ + if(unlikely(*(*start) == '"')){ + element->measureLen = (*start) - element->measure; + break; + } + (*start)++; + } + }else if((*start)[1] == 't' && (*start)[2] == 'i'){ + (*start) += offset[index++]; + element->timestamp = *start; + while(*(*start)){ + if(unlikely(*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)){ + element->timestampLen = (*start) - element->timestamp; + break; + } + (*start)++; + } + }else if((*start)[1] == 'v'){ + (*start) += offset[index++]; + element->cols = *start; + while(*(*start)){ + if(unlikely( *(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)){ + element->colsLen = (*start) - element->cols; + break; + } + (*start)++; + } + }else if((*start)[1] == 't' && (*start)[2] == 'a'){ + (*start) += offset[index++]; + element->tags = (*start); + char* tmp = smlJsonGetObj((*start)); + if(tmp){ + element->tagsLen = tmp - (*start); + *start = tmp; + } + break; + } + if(*(*start) == '}'){ + (*start)++; + break; + } + (*start)++; + } + + if(unlikely(index != 0 && index != OTD_JSON_FIELDS_NUM)) { + uError("elements != %d", OTD_JSON_FIELDS_NUM) + return -1; + } + return 0; +} + +static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) { + elements->measureLen = strlen(metric->valuestring); + if (IS_INVALID_TABLE_LEN(elements->measureLen)) { + uError("OTD:0x%" PRIx64 " Metric lenght is 0 or large than 192", info->id); + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + + elements->measure = metric->valuestring; + return TSDB_CODE_SUCCESS; +} + +const char *jsonName[OTD_JSON_FIELDS_NUM] = {"metric", "timestamp", "value", "tags"}; +static int32_t smlGetJsonElements(cJSON *root, cJSON ***marks){ + for (int i = 0; i < OTD_JSON_FIELDS_NUM; ++i) { + cJSON *child = root->child; + while(child != NULL) + { + if(strcasecmp(child->string, jsonName[i]) == 0){ + *marks[i] = child; + break; + } + child = child->next; + } + if(*marks[i] == NULL){ + uError("smlGetJsonElements error, not find mark:%d:%s", i, jsonName[i]); + return -1; + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t smlConvertJSONBool(SSmlKv *pVal, char *typeStr, cJSON *value) { + if (strcasecmp(typeStr, "bool") != 0) { + uError("OTD:invalid type(%s) for JSON Bool", typeStr); + return TSDB_CODE_TSC_INVALID_JSON_TYPE; + } + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->i = value->valueint; + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { + // tinyint + if (strcasecmp(typeStr, "i8") == 0 || strcasecmp(typeStr, "tinyint") == 0) { + if (!IS_VALID_TINYINT(value->valuedouble)) { + uError("OTD:JSON value(%f) cannot fit in type(tinyint)", value->valuedouble); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_TINYINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->i = value->valuedouble; + return TSDB_CODE_SUCCESS; + } + // smallint + if (strcasecmp(typeStr, "i16") == 0 || strcasecmp(typeStr, "smallint") == 0) { + if (!IS_VALID_SMALLINT(value->valuedouble)) { + uError("OTD:JSON value(%f) cannot fit in type(smallint)", value->valuedouble); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_SMALLINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->i = value->valuedouble; + return TSDB_CODE_SUCCESS; + } + // int + if (strcasecmp(typeStr, "i32") == 0 || strcasecmp(typeStr, "int") == 0) { + if (!IS_VALID_INT(value->valuedouble)) { + uError("OTD:JSON value(%f) cannot fit in type(int)", value->valuedouble); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_INT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->i = value->valuedouble; + return TSDB_CODE_SUCCESS; + } + // bigint + if (strcasecmp(typeStr, "i64") == 0 || strcasecmp(typeStr, "bigint") == 0) { + pVal->type = TSDB_DATA_TYPE_BIGINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + if (value->valuedouble >= (double)INT64_MAX) { + pVal->i = INT64_MAX; + } else if (value->valuedouble <= (double)INT64_MIN) { + pVal->i = INT64_MIN; + } else { + pVal->i = value->valuedouble; + } + return TSDB_CODE_SUCCESS; + } + // float + if (strcasecmp(typeStr, "f32") == 0 || strcasecmp(typeStr, "float") == 0) { + if (!IS_VALID_FLOAT(value->valuedouble)) { + uError("OTD:JSON value(%f) cannot fit in type(float)", value->valuedouble); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_FLOAT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->f = value->valuedouble; + return TSDB_CODE_SUCCESS; + } + // double + if (strcasecmp(typeStr, "f64") == 0 || strcasecmp(typeStr, "double") == 0) { + pVal->type = TSDB_DATA_TYPE_DOUBLE; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->d = value->valuedouble; + return TSDB_CODE_SUCCESS; + } + + // if reach here means type is unsupported + uError("OTD:invalid type(%s) for JSON Number", typeStr); + return TSDB_CODE_TSC_INVALID_JSON_TYPE; +} + +static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) { + if (strcasecmp(typeStr, "binary") == 0) { + pVal->type = TSDB_DATA_TYPE_BINARY; + } else if (strcasecmp(typeStr, "nchar") == 0) { + pVal->type = TSDB_DATA_TYPE_NCHAR; + } else { + uError("OTD:invalid type(%s) for JSON String", typeStr); + return TSDB_CODE_TSC_INVALID_JSON_TYPE; + } + pVal->length = (int16_t)strlen(value->valuestring); + + if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + if (pVal->type == TSDB_DATA_TYPE_NCHAR && + pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + + pVal->value = value->valuestring; + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseValueFromJSONObj(cJSON *root, SSmlKv *kv) { + int32_t ret = TSDB_CODE_SUCCESS; + int32_t size = cJSON_GetArraySize(root); + + if (size != OTD_JSON_SUB_FIELDS_NUM) { + return TSDB_CODE_TSC_INVALID_JSON; + } + + cJSON *value = cJSON_GetObjectItem(root, "value"); + if (value == NULL) { + return TSDB_CODE_TSC_INVALID_JSON; + } + + cJSON *type = cJSON_GetObjectItem(root, "type"); + if (!cJSON_IsString(type)) { + return TSDB_CODE_TSC_INVALID_JSON; + } + + switch (value->type) { + case cJSON_True: + case cJSON_False: { + ret = smlConvertJSONBool(kv, type->valuestring, value); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + break; + } + case cJSON_Number: { + ret = smlConvertJSONNumber(kv, type->valuestring, value); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + break; + } + case cJSON_String: { + ret = smlConvertJSONString(kv, type->valuestring, value); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + break; + } + default: + return TSDB_CODE_TSC_INVALID_JSON_TYPE; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { + switch (root->type) { + case cJSON_True: + case cJSON_False: { + kv->type = TSDB_DATA_TYPE_BOOL; + kv->length = (int16_t)tDataTypes[kv->type].bytes; + kv->i = root->valueint; + break; + } + case cJSON_Number: { + kv->type = TSDB_DATA_TYPE_DOUBLE; + kv->length = (int16_t)tDataTypes[kv->type].bytes; + kv->d = root->valuedouble; + break; + } + case cJSON_String: { + /* set default JSON type to binary/nchar according to + * user configured parameter tsDefaultJSONStrType + */ + + char *tsDefaultJSONStrType = "nchar"; // todo + smlConvertJSONString(kv, tsDefaultJSONStrType, root); + break; + } + case cJSON_Object: { + int32_t ret = smlParseValueFromJSONObj(root, kv); + if (ret != TSDB_CODE_SUCCESS) { + uError("OTD:Failed to parse value from JSON Obj"); + return ret; + } + break; + } + default: + return TSDB_CODE_TSC_INVALID_JSON; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { + int32_t ret = TSDB_CODE_SUCCESS; + + bool isSameMeasure = IS_SAME_SUPER_TABLE; + + int cnt = 0; + SArray *preLineKV = info->preLineTagKV; + SArray *maxKVs = info->maxTagKVs; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if(info->dataFormat){ + if(unlikely(!isSameMeasure)){ + SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + + if(unlikely(sMeta == NULL)){ + STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); + if(pTableMeta == NULL){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + sMeta = smlBuildSTableMeta(info->dataFormat); + sMeta->tableMeta = pTableMeta; + nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL); + } + info->currSTableMeta = sMeta->tableMeta; + superKV = sMeta->tags; + + if(unlikely(taosArrayGetSize(superKV) == 0)){ + isSuperKVInit = false; + } + taosArraySetSize(maxKVs, 0); + } + }else{ + taosArraySetSize(maxKVs, 0); + } + taosArraySetSize(preLineKV, 0); + + int32_t tagNum = cJSON_GetArraySize(tags); + if(unlikely(tagNum == 0)){ + uError("SML:Tag should not be empty"); + return TSDB_CODE_TSC_INVALID_JSON; + } + for (int32_t i = 0; i < tagNum; ++i) { + cJSON *tag = cJSON_GetArrayItem(tags, i); + if (unlikely(tag == NULL)) { + return TSDB_CODE_TSC_INVALID_JSON; + } +// if(unlikely(tag == cMeasure)) continue; + size_t keyLen = strlen(tag->string); + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + uError("OTD:Tag key length is 0 or too large than 64"); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } + + // add kv to SSmlKv + SSmlKv kv ={.key = tag->string, .keyLen = keyLen}; + // value + ret = smlParseValueFromJSON(tag, &kv); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + return ret; + } + + if(info->dataFormat){ + if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(isSameMeasure){ + if(unlikely(cnt >= taosArrayGetSize(maxKVs))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(maxKVs, cnt); + if(unlikely(kv.length > maxKV->length)){ + maxKV->length = kv.length; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + if(unlikely(NULL == tableMeta)){ + uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); + return TSDB_CODE_SML_INTERNAL_ERROR; + } + + SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + if(isSuperKVInit){ + if(unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(superKV, cnt); + if(unlikely(kv.length > maxKV->length)) { + maxKV->length = kv.length; + }else{ + kv.length = maxKV->length; + } + info->needModifySchema = true; + + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + taosArrayPush(superKV, &kv); + } + taosArrayPush(maxKVs, &kv); + } + }else{ + taosArrayPush(maxKVs, &kv); + } + taosArrayPush(preLineKV, &kv); + cnt++; + } + + SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); + if (unlikely(tinfo == NULL)) { + tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); + if (unlikely(!tinfo)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tinfo->tags = taosArrayDup(preLineKV, NULL); + + smlSetCTableName(tinfo); + tinfo->uid = info->uid++; + if (info->dataFormat) { + info->currSTableMeta->uid = tinfo->uid; + tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); + if (tinfo->tableDataCtx == NULL) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); + smlDestroyTableInfo(info, tinfo); + return TSDB_CODE_SML_INVALID_DATA; + } + } + + SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); + *key = *elements; + if(info->parseJsonByLib){ + key->tags = taosMemoryMalloc(elements->tagsLen + 1); + memcpy(key->tags, elements->tags, elements->tagsLen); + key->tags[elements->tagsLen] = 0; + } + tinfo->key = key; + nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet); + } + if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; + + return ret; +} + +static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPrecision) { + int32_t size = cJSON_GetArraySize(root); + if (unlikely(size != OTD_JSON_SUB_FIELDS_NUM)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); + return -1; + } + + cJSON *value = cJSON_GetObjectItem(root, "value"); + if (unlikely(!cJSON_IsNumber(value))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); + return -1; + } + + cJSON *type = cJSON_GetObjectItem(root, "type"); + if (unlikely(!cJSON_IsString(type))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); + return -1; + } + + double timeDouble = value->valuedouble; + if (unlikely(smlDoubleToInt64OverFlow(timeDouble))) { + smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); + return -1; + } + + if (timeDouble == 0) { + return taosGetTimestampNs()/smlFactorNS[toPrecision]; + } + + if (timeDouble < 0) { + return timeDouble; + } + + int64_t tsInt64 = timeDouble; + size_t typeLen = strlen(type->valuestring); + if (typeLen == 1 && (type->valuestring[0] == 's' || type->valuestring[0] == 'S')) { + // seconds + int8_t fromPrecision = TSDB_TIME_PRECISION_SECONDS; + if(smlFactorS[toPrecision] < INT64_MAX / tsInt64){ + return tsInt64 * smlFactorS[toPrecision]; + } + return -1; + } else if (typeLen == 2 && (type->valuestring[1] == 's' || type->valuestring[1] == 'S')) { + switch (type->valuestring[0]) { + case 'm': + case 'M': + // milliseconds + return convertTimePrecision(tsInt64, TSDB_TIME_PRECISION_MILLI, toPrecision); + break; + case 'u': + case 'U': + // microseconds + return convertTimePrecision(tsInt64, TSDB_TIME_PRECISION_MICRO, toPrecision); + break; + case 'n': + case 'N': + return convertTimePrecision(tsInt64, TSDB_TIME_PRECISION_NANO, toPrecision); + break; + default: + return -1; + } + } else { + return -1; + } +} + +uint8_t smlGetTimestampLen(int64_t num) { + uint8_t len = 0; + while ((num /= 10) != 0) { + len++; + } + len++; + return len; +} + +static int64_t smlParseTSFromJSON(SSmlHandle *info, cJSON *timestamp) { + // Timestamp must be the first KV to parse + int32_t toPrecision = info->currSTableMeta ? info->currSTableMeta->tableInfo.precision : TSDB_TIME_PRECISION_NANO; + if (cJSON_IsNumber(timestamp)) { + // timestamp value 0 indicates current system time + double timeDouble = timestamp->valuedouble; + if (unlikely(smlDoubleToInt64OverFlow(timeDouble))) { + smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); + return -1; + } + + if (unlikely(timeDouble < 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, + "timestamp is negative", NULL); + return timeDouble; + }else if (unlikely(timeDouble == 0)) { + return taosGetTimestampNs()/smlFactorNS[toPrecision]; + } + + uint8_t tsLen = smlGetTimestampLen((int64_t)timeDouble); + + int8_t fromPrecision = smlGetTsTypeByLen(tsLen); + if (unlikely(fromPrecision == -1)) { + smlBuildInvalidDataMsg(&info->msgBuf, + "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", NULL); + return -1; + } + int64_t tsInt64 = timeDouble; + if(fromPrecision == TSDB_TIME_PRECISION_SECONDS){ + if(smlFactorS[toPrecision] < INT64_MAX / tsInt64){ + return tsInt64 * smlFactorS[toPrecision]; + } + return -1; + }else{ + return convertTimePrecision(timeDouble, fromPrecision, toPrecision); + } + } else if (cJSON_IsObject(timestamp)) { + return smlParseTSFromJSONObj(info, timestamp, toPrecision); + } else { + smlBuildInvalidDataMsg(&info->msgBuf, + "invalidate json", NULL); + return -1; + } +} + +static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo *elements) { + int32_t ret = TSDB_CODE_SUCCESS; + + cJSON *metricJson = NULL; + cJSON *tsJson = NULL; + cJSON *valueJson = NULL; + cJSON *tagsJson = NULL; + + int32_t size = cJSON_GetArraySize(root); + // outmost json fields has to be exactly 4 + if (size != OTD_JSON_FIELDS_NUM) { + uError("OTD:0x%" PRIx64 " Invalid number of JSON fields in data point %d", info->id, size); + return TSDB_CODE_TSC_INVALID_JSON; + } + + cJSON **marks[OTD_JSON_FIELDS_NUM] = {&metricJson, &tsJson, &valueJson, &tagsJson}; + ret = smlGetJsonElements(root, marks); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + return ret; + } + + // Parse metric + ret = smlParseMetricFromJSON(info, metricJson, elements); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uError("OTD:0x%" PRIx64 " Unable to parse metric from JSON payload", info->id); + return ret; + } + + // Parse metric value + SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN}; + ret = smlParseValueFromJSON(valueJson, &kv); + if (unlikely(ret)) { + uError("OTD:0x%" PRIx64 " Unable to parse metric value from JSON payload", info->id); + return ret; + } + + // Parse tags + bool needFree = info->dataFormat; + elements->tags = cJSON_PrintUnformatted(tagsJson); + elements->tagsLen = strlen(elements->tags); + if(is_same_child_table_telnet(elements, &info->preLine) != 0) { + ret = smlParseTagsFromJSON(info, tagsJson, elements); + if (unlikely(ret)) { + uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); + taosMemoryFree(elements->tags); + elements->tags = NULL; + return ret; + } + } + + if(needFree){ + taosMemoryFree(elements->tags); + elements->tags = NULL; + } + + if(unlikely(info->reRun)){ + return TSDB_CODE_SUCCESS; + } + + // Parse timestamp + // notice!!! put ts back to tag to ensure get meta->precision + int64_t ts = smlParseTSFromJSON(info, tsJson); + if (unlikely(ts < 0)) { + uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); + return TSDB_CODE_INVALID_TIMESTAMP; + } + SSmlKv kvTs = { .key = TS, .keyLen = TS_LEN, .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + + if(info->dataFormat){ + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); + } + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildRow(info->currTableDataCtx); + } + clearColValArray(info->currTableDataCtx->pValues); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + return ret; + } + }else{ + if(elements->colArray == NULL){ + elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + } + taosArrayPush(elements->colArray, &kvTs); + taosArrayPush(elements->colArray, &kv); + } + info->preLine = *elements; + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { + int32_t payloadNum = 0; + int32_t ret = TSDB_CODE_SUCCESS; + + if (unlikely(payload == NULL)) { + uError("SML:0x%" PRIx64 " empty JSON Payload", info->id); + return TSDB_CODE_TSC_INVALID_JSON; + } + + info->root = cJSON_Parse(payload); + if (unlikely(info->root == NULL)) { + uError("SML:0x%" PRIx64 " parse json failed:%s", info->id, payload); + return TSDB_CODE_TSC_INVALID_JSON; + } + + // multiple data points must be sent in JSON array + if (cJSON_IsArray(info->root)) { + payloadNum = cJSON_GetArraySize(info->root); + } else if (cJSON_IsObject(info->root)) { + payloadNum = 1; + } else { + uError("SML:0x%" PRIx64 " Invalid JSON Payload 3:%s", info->id, payload); + return TSDB_CODE_TSC_INVALID_JSON; + } + + info->lineNum = payloadNum; + info->dataFormat = true; + if (unlikely(info->lines != NULL)) { + taosMemoryFree(info->lines); + info->lines = NULL; + } + ret = smlClearForRerun(info); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } + + info->parseJsonByLib = true; + cJSON *head = (payloadNum == 1 && cJSON_IsObject(info->root)) ? info->root : info->root->child; + + int cnt = 0; + cJSON *dataPoint = head; + while (dataPoint) { + if(info->dataFormat) { + SSmlLineInfo element = {0}; + ret = smlParseJSONStringExt(info, dataPoint, &element); + }else{ + ret = smlParseJSONStringExt(info, dataPoint, info->lines + cnt); + } + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uError("SML:0x%" PRIx64 " Invalid JSON Payload 2:%s", info->id, payload); + return ret; + } + + if(unlikely(info->reRun)){ + cnt = 0; + dataPoint = head; + info->lineNum = payloadNum; + ret = smlClearForRerun(info); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } + continue; + } + cnt++; + dataPoint = dataPoint->next; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *elements) { + int32_t ret = TSDB_CODE_SUCCESS; + + if(info->offset[0] == 0){ + ret = smlJsonParseObjFirst(start, elements, info->offset); + }else{ + ret = smlJsonParseObj(start, elements, info->offset); + } + + if (ret != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if(unlikely(**start == '\0' && elements->measure == NULL)) return TSDB_CODE_SUCCESS; + + if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL); + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + + SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; + if (elements->colsLen == 0 || smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { + uError("SML:cols invalidate:%s", elements->cols); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + // Parse tags + if(is_same_child_table_telnet(elements, &info->preLine) != 0){ + char tmp = *(elements->tags + elements->tagsLen); + *(elements->tags + elements->tagsLen) = 0; + cJSON* tagsJson = cJSON_Parse(elements->tags); + *(elements->tags + elements->tagsLen) = tmp; + if (unlikely(tagsJson == NULL)) { + uError("SML:0x%" PRIx64 " parse json failed:%s", info->id, elements->tags); + return TSDB_CODE_TSC_INVALID_JSON; + } + + taosArrayPush(info->tagJsonArray, &tagsJson); + ret = smlParseTagsFromJSON(info, tagsJson, elements); + if (unlikely(ret)) { + uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); + return ret; + } + } + + if(unlikely(info->reRun)){ + return TSDB_CODE_SUCCESS; + } + + // Parse timestamp + // notice!!! put ts back to tag to ensure get meta->precision + int64_t ts = smlParseOpenTsdbTime(info, elements->timestamp, elements->timestampLen); + if (unlikely(ts < 0)) { + uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); + return TSDB_CODE_INVALID_TIMESTAMP; + } + SSmlKv kvTs = { .key = TS, .keyLen = TS_LEN, .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + + if(info->dataFormat){ + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); + } + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildRow(info->currTableDataCtx); + } + clearColValArray(info->currTableDataCtx->pValues); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + return ret; + } + }else{ + if(elements->colArray == NULL){ + elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + } + taosArrayPush(elements->colArray, &kvTs); + taosArrayPush(elements->colArray, &kv); + } + info->preLine = *elements; + + return TSDB_CODE_SUCCESS; +} + +int32_t smlParseJSON(SSmlHandle *info, char *payload) { + int32_t payloadNum = 1 << 15; + int32_t ret = TSDB_CODE_SUCCESS; + + uDebug("SML:0x%" PRIx64 "json:%s", info->id, payload); + int cnt = 0; + char *dataPointStart = payload; + while (1) { + if(info->dataFormat) { + SSmlLineInfo element = {0}; + ret = smlParseJSONString(info, &dataPointStart, &element); + }else{ + if(cnt >= payloadNum){ + payloadNum = payloadNum << 1; + void* tmp = taosMemoryRealloc(info->lines, payloadNum * sizeof(SSmlLineInfo)); + if(tmp != NULL){ + info->lines = (SSmlLineInfo*)tmp; + memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo)); + } + } + ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt); + } + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uError("SML:0x%" PRIx64 " Invalid JSON Payload 1:%s", info->id, payload); + return smlParseJSONExt(info, payload); + } + + if(unlikely(info->reRun)){ + cnt = 0; + dataPointStart = payload; + info->lineNum = payloadNum; + ret = smlClearForRerun(info); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } + continue; + } + + if(*dataPointStart == '\0') break; + cnt++; + } + info->lineNum = cnt; + + return TSDB_CODE_SUCCESS; +} diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c new file mode 100644 index 0000000000000000000000000000000000000000..890245f53d5f7cbf52613ca7804da3c43e9543f8 --- /dev/null +++ b/source/client/src/clientSmlLine.c @@ -0,0 +1,664 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include + +#include "clientSml.h" + +// comma , +//#define IS_SLASH_COMMA(sql) (*(sql) == COMMA && *((sql)-1) == SLASH) +#define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH) +// space +//#define IS_SLASH_SPACE(sql) (*(sql) == SPACE && *((sql)-1) == SLASH) +#define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH) +// equal = +//#define IS_SLASH_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) == SLASH) +#define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH) +// quote " +//#define IS_SLASH_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) == SLASH) +#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH) +// SLASH +//#define IS_SLASH_SLASH(sql) (*(sql) == SLASH && *((sql)-1) == SLASH) + +#define IS_SLASH_LETTER(sql) \ + (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL || *(sql) == QUOTE || *(sql) == SLASH)) \ +// (IS_SLASH_COMMA(sql) || IS_SLASH_SPACE(sql) || IS_SLASH_EQUAL(sql) || IS_SLASH_QUOTE(sql) || IS_SLASH_SLASH(sql)) + +#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) + +#define PROCESS_SLASH(key, keyLen) \ + for (int i = 1; i < keyLen; ++i) { \ + if (IS_SLASH_LETTER(key + i)) { \ + MOVE_FORWARD_ONE(key + i, keyLen - i); \ + i--; \ + keyLen--; \ + } \ + } + +#define BINARY_ADD_LEN 2 // "binary" 2 means " " +#define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" " + +uint8_t smlPrecisionConvert[7] = {TSDB_TIME_PRECISION_NANO, TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES, + TSDB_TIME_PRECISION_SECONDS, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO, + TSDB_TIME_PRECISION_NANO}; + +static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t len) { + uint8_t toPrecision = info->currSTableMeta ? info->currSTableMeta->tableInfo.precision : TSDB_TIME_PRECISION_NANO; + + if(unlikely(len == 0 || (len == 1 && data[0] == '0'))){ + return taosGetTimestampNs()/smlFactorNS[toPrecision]; + } + + uint8_t fromPrecision = smlPrecisionConvert[info->precision]; + + int64_t ts = smlGetTimeValue(data, len, fromPrecision, toPrecision); + if (unlikely(ts == -1)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); + return -1; + } + return ts; +} + +int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { + if (pVal->value[0] == '"'){ // binary + if (pVal->length >= 2 && pVal->value[pVal->length - 1] == '"') { + pVal->type = TSDB_DATA_TYPE_BINARY; + pVal->length -= BINARY_ADD_LEN; + if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + pVal->value += (BINARY_ADD_LEN - 1); + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if(pVal->value[0] == 'l' || pVal->value[0] == 'L'){ // nchar + if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= 3){ + pVal->type = TSDB_DATA_TYPE_NCHAR; + pVal->length -= NCHAR_ADD_LEN; + if (pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + pVal->value += (NCHAR_ADD_LEN - 1); + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pVal->value[0] == 't' || pVal->value[0] == 'T'){ + if(pVal->length == 1 || (pVal->length == 4 && (pVal->value[1] == 'r' || pVal->value[1] == 'R') + && (pVal->value[2] == 'u' || pVal->value[2] == 'U') + && (pVal->value[3] == 'e' || pVal->value[3] == 'E'))){ + pVal->i = TSDB_TRUE; + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pVal->value[0] == 'f' || pVal->value[0] == 'F'){ + if(pVal->length == 1 || (pVal->length == 5 && (pVal->value[1] == 'a' || pVal->value[1] == 'A') + && (pVal->value[2] == 'l' || pVal->value[2] == 'L') + && (pVal->value[3] == 's' || pVal->value[3] == 'S') + && (pVal->value[4] == 'e' || pVal->value[4] == 'E'))){ + pVal->i = TSDB_FALSE; + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + + // number + if (smlParseNumber(pVal, msg)) { + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return TSDB_CODE_SUCCESS; + } + + return TSDB_CODE_TSC_INVALID_VALUE; +} + +static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, + SSmlLineInfo* currElement, bool isSameMeasure, bool isSameCTable){ + if(isSameCTable){ + return TSDB_CODE_SUCCESS; + } + + int cnt = 0; + SArray *preLineKV = info->preLineTagKV; + SArray *maxKVs = info->maxTagKVs; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if(info->dataFormat){ + if(unlikely(!isSameMeasure)){ + SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); + + if(unlikely(sMeta == NULL)){ + STableMeta * pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); + if(pTableMeta == NULL){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + sMeta = smlBuildSTableMeta(info->dataFormat); + sMeta->tableMeta = pTableMeta; + nodeListSet(&info->superTables, currElement->measure, currElement->measureLen, sMeta, NULL); + } + info->currSTableMeta = sMeta->tableMeta; + superKV = sMeta->tags; + + if(unlikely(taosArrayGetSize(superKV) == 0)){ + isSuperKVInit = false; + } + taosArraySetSize(maxKVs, 0); + } + }else{ + taosArraySetSize(maxKVs, 0); + } + taosArraySetSize(preLineKV, 0); + + while (*sql < sqlEnd) { + if (unlikely(IS_SPACE(*sql))) { + break; + } + + bool hasSlash = false; + // parse key + const char *key = *sql; + size_t keyLen = 0; + while (*sql < sqlEnd) { + if (unlikely(IS_COMMA(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(IS_EQUAL(*sql))) { + keyLen = *sql - key; + (*sql)++; + break; + } + if(!hasSlash){ + hasSlash = (*(*sql) == SLASH); + } + (*sql)++; + } + if(unlikely(hasSlash)) { + PROCESS_SLASH(key, keyLen) + } + + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } + + // parse value + const char *value = *sql; + size_t valueLen = 0; + hasSlash = false; + while (*sql < sqlEnd) { + // parse value + if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { + break; + }else if (unlikely(IS_EQUAL(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + + if(!hasSlash){ + hasSlash = (*(*sql) == SLASH); + } + + (*sql)++; + } + valueLen = *sql - value; + + if (unlikely(valueLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); + return TSDB_CODE_SML_INVALID_DATA; + } + + if(unlikely(hasSlash)) { + PROCESS_SLASH(value, valueLen) + } + + if (unlikely(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + + SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen}; + if(info->dataFormat){ + if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(isSameMeasure){ + if(unlikely(cnt >= taosArrayGetSize(maxKVs))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(maxKVs, cnt); + if(unlikely(kv.length > maxKV->length)){ + maxKV->length = kv.length; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); + if(unlikely(NULL == tableMeta)){ + uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); + return TSDB_CODE_SML_INTERNAL_ERROR; + } + + SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + if(isSuperKVInit){ + if(unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(superKV, cnt); + if(unlikely(kv.length > maxKV->length)) { + maxKV->length = kv.length; + }else{ + kv.length = maxKV->length; + } + info->needModifySchema = true; + + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + taosArrayPush(superKV, &kv); + } + taosArrayPush(maxKVs, &kv); + } + }else{ + taosArrayPush(maxKVs, &kv); + } + taosArrayPush(preLineKV, &kv); + + cnt++; + if(IS_SPACE(*sql)){ + break; + } + (*sql)++; + } + + void* oneTable = nodeListGet(info->childTables, currElement->measure, currElement->measureTagsLen, NULL); + if ((oneTable != NULL)) { + return TSDB_CODE_SUCCESS; + } + + SSmlTableInfo *tinfo = smlBuildTableInfo(1, currElement->measure, currElement->measureLen); + if (unlikely(!tinfo)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tinfo->tags = taosArrayDup(preLineKV, NULL); + + smlSetCTableName(tinfo); + tinfo->uid = info->uid++; + if(info->dataFormat) { + info->currSTableMeta->uid = tinfo->uid; + tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); + if(tinfo->tableDataCtx == NULL){ + smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } + } + + nodeListSet(&info->childTables, currElement->measure, currElement->measureTagsLen, tinfo, NULL); + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, + SSmlLineInfo* currElement, bool isSameMeasure, bool isSameCTable){ + int cnt = 0; + SArray *preLineKV = info->preLineColKV; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if(info->dataFormat){ + if(unlikely(!isSameCTable)){ + SSmlTableInfo *oneTable = (SSmlTableInfo *)nodeListGet(info->childTables, currElement->measure, currElement->measureTagsLen, NULL); + if (unlikely(oneTable == NULL)) { + smlBuildInvalidDataMsg(&info->msgBuf, "child table should inside", currElement->measure); + return TSDB_CODE_SML_INVALID_DATA; + } + info->currTableDataCtx = oneTable->tableDataCtx; + } + + if(unlikely(!isSameMeasure)){ + SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); + + if(unlikely(sMeta == NULL)){ + STableMeta * pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); + if(pTableMeta == NULL){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + sMeta = smlBuildSTableMeta(info->dataFormat); + sMeta->tableMeta = pTableMeta; + nodeListSet(&info->superTables, currElement->measure, currElement->measureLen, sMeta, NULL); + } + info->currSTableMeta = sMeta->tableMeta; + superKV = sMeta->cols; + if(unlikely(taosArrayGetSize(superKV) == 0)){ + isSuperKVInit = false; + } + taosArraySetSize(preLineKV, 0); + } + } + + while (*sql < sqlEnd) { + if (unlikely(IS_SPACE(*sql))) { + break; + } + + bool hasSlash = false; + // parse key + const char *key = *sql; + size_t keyLen = 0; + while (*sql < sqlEnd) { + if (unlikely(IS_COMMA(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(IS_EQUAL(*sql))) { + keyLen = *sql - key; + (*sql)++; + break; + } + if(!hasSlash){ + hasSlash = (*(*sql) == SLASH); + } + (*sql)++; + } + if(unlikely(hasSlash)) { + PROCESS_SLASH(key, keyLen) + } + + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } + + // parse value + const char *value = *sql; + size_t valueLen = 0; + hasSlash = false; + bool isInQuote = false; + while (*sql < sqlEnd) { + // parse value + if (unlikely(IS_QUOTE(*sql))) { + isInQuote = !isInQuote; + (*sql)++; + continue; + } + if (!isInQuote){ + if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { + break; + } else if (unlikely(IS_EQUAL(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + } + if(!hasSlash){ + hasSlash = (*(*sql) == SLASH); + } + + (*sql)++; + } + valueLen = *sql - value; + + if (unlikely(isInQuote)) { + smlBuildInvalidDataMsg(&info->msgBuf, "only one quote", value); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(valueLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); + return TSDB_CODE_SML_INVALID_DATA; + } + if(unlikely(hasSlash)) { + PROCESS_SLASH(value, valueLen) + } + + SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen}; + int32_t ret = smlParseValue(&kv, &info->msgBuf); + if (ret != TSDB_CODE_SUCCESS) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlParseValue error", value); + return ret; + } + + if(info->dataFormat){ + //cnt begin 0, add ts so + 2 + if(unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + // bind data + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, cnt + 1); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uError("smlBuildCol error, retry"); + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(isSameMeasure){ + if(cnt >= taosArrayGetSize(preLineKV)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); + if(kv.type != maxKV->type){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > maxKV->length)){ + maxKV->length = kv.length; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); + if(unlikely(NULL == tableMeta)){ + uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); + return TSDB_CODE_SML_INTERNAL_ERROR; + } + + SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->cols, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + if(isSuperKVInit){ + if(unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(superKV, cnt); + if(unlikely(kv.type != maxKV->type)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(IS_VAR_DATA_TYPE(kv.type)){ + if(kv.length > maxKV->length) { + maxKV->length = kv.length; + }else{ + kv.length = maxKV->length; + } + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + taosArrayPush(superKV, &kv); + } + taosArrayPush(preLineKV, &kv); + } + }else{ + if(currElement->colArray == NULL){ + currElement->colArray = taosArrayInit(16, sizeof(SSmlKv)); + taosArraySetSize(currElement->colArray, 1); + } + taosArrayPush(currElement->colArray, &kv); //reserve for timestamp + } + + cnt++; + if(IS_SPACE(*sql)){ + break; + } + (*sql)++; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements) { + if (!sql) return TSDB_CODE_SML_INVALID_DATA; + JUMP_SPACE(sql, sqlEnd) + if (unlikely(*sql == COMMA)) return TSDB_CODE_SML_INVALID_DATA; + elements->measure = sql; + + // parse measure + while (sql < sqlEnd) { + if (unlikely((sql != elements->measure) && IS_SLASH_LETTER(sql))) { + MOVE_FORWARD_ONE(sql, sqlEnd - sql); + sqlEnd--; + continue; + } + if (unlikely(IS_COMMA(sql))) { + break; + } + + if (unlikely(IS_SPACE(sql))) { + break; + } + sql++; + } + elements->measureLen = sql - elements->measure; + if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL); + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + + // to get measureTagsLen before + const char* tmp = sql; + while (tmp < sqlEnd){ + if (unlikely(IS_SPACE(tmp))) { + break; + } + tmp++; + } + elements->measureTagsLen = tmp - elements->measure; + + bool isSameCTable = false; + bool isSameMeasure = false; + if(IS_SAME_CHILD_TABLE){ + isSameCTable = true; + isSameMeasure = true; + }else if(info->dataFormat) { + isSameMeasure = IS_SAME_SUPER_TABLE; + } + // parse tag + if (*sql == COMMA) sql++; + elements->tags = sql; + + int ret = smlParseTagKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable); + if(unlikely(ret != TSDB_CODE_SUCCESS)){ + return ret; + } + if(unlikely(info->reRun)){ + return TSDB_CODE_SUCCESS; + } + + sql = elements->measure + elements->measureTagsLen; + elements->tagsLen = sql - elements->tags; + + // parse cols + JUMP_SPACE(sql, sqlEnd) + elements->cols = sql; + + ret = smlParseColKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable); + if(unlikely(ret != TSDB_CODE_SUCCESS)){ + return ret; + } + + if(unlikely(info->reRun)){ + return TSDB_CODE_SUCCESS; + } + + elements->colsLen = sql - elements->cols; + if (unlikely(elements->colsLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "cols is empty", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } + + // parse timestamp + JUMP_SPACE(sql, sqlEnd) + elements->timestamp = sql; + while (sql < sqlEnd) { + if (unlikely(isspace(*sql))) { + break; + } + sql++; + } + elements->timestampLen = sql - elements->timestamp; + + int64_t ts = smlParseInfluxTime(info, elements->timestamp, elements->timestampLen); + if (unlikely(ts <= 0)) { + uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); + return TSDB_CODE_INVALID_TIMESTAMP; + } + // add ts to + SSmlKv kv = { .key = TS, .keyLen = TS_LEN, .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + if(info->dataFormat){ + smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0); + smlBuildRow(info->currTableDataCtx); + clearColValArray(info->currTableDataCtx->pValues); + }else{ + taosArraySet(elements->colArray, 0, &kv); + } + info->preLine = *elements; + + return ret; +} + diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c new file mode 100644 index 0000000000000000000000000000000000000000..d43ab6c9f97ca7d1c4523f03ff2bb9c73556fc4b --- /dev/null +++ b/source/client/src/clientSmlTelnet.c @@ -0,0 +1,347 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include + +#include "clientSml.h" + +int32_t is_same_child_table_telnet(const void *a, const void *b){ + SSmlLineInfo *t1 = (SSmlLineInfo *)a; + SSmlLineInfo *t2 = (SSmlLineInfo *)b; +// uError("is_same_child_table_telnet len:%d,%d %s,%s @@@ len:%d,%d %s,%s", t1->measureLen, t2->measureLen, +// t1->measure, t2->measure, t1->tagsLen, t2->tagsLen, t1->tags, t2->tags); + if(t1 == NULL || t2 == NULL || t1->measure == NULL || t2->measure == NULL + || t1->tags == NULL || t2->tags == NULL) + return 1; + return (((t1->measureLen == t2->measureLen) && memcmp(t1->measure, t2->measure, t1->measureLen) == 0) + && ((t1->tagsLen == t2->tagsLen) && memcmp(t1->tags, t2->tags, t1->tagsLen) == 0)) ? 0 : 1; +} + +int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len) { + uint8_t toPrecision = info->currSTableMeta ? info->currSTableMeta->tableInfo.precision : TSDB_TIME_PRECISION_NANO; + + if (unlikely(!data)) { + smlBuildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL); + return -1; + } + if (unlikely(len == 1 && data[0] == '0')) { + return taosGetTimestampNs()/smlFactorNS[toPrecision]; + } + int8_t fromPrecision = smlGetTsTypeByLen(len); + if (unlikely(fromPrecision == -1)) { + smlBuildInvalidDataMsg(&info->msgBuf, + "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data); + return -1; + } + int64_t ts = smlGetTimeValue(data, len, fromPrecision, toPrecision); + if (unlikely(ts == -1)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); + return -1; + } + return ts; +} + + +static void smlParseTelnetElement(char **sql, char *sqlEnd, char **data, int32_t *len) { + while (*sql < sqlEnd) { + if (unlikely((**sql != SPACE && !(*data)))) { + *data = *sql; + } else if (unlikely(**sql == SPACE && *data)) { + *len = *sql - *data; + break; + } + (*sql)++; + } +} + +static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) { + if(is_same_child_table_telnet(elements, &info->preLine) == 0){ + return TSDB_CODE_SUCCESS; + } + + bool isSameMeasure = IS_SAME_SUPER_TABLE; + + int cnt = 0; + SArray *preLineKV = info->preLineTagKV; + SArray *maxKVs = info->maxTagKVs; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if(info->dataFormat){ + if(!isSameMeasure){ + SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + + if(unlikely(sMeta == NULL)){ + STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); + if(pTableMeta == NULL){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + sMeta = smlBuildSTableMeta(info->dataFormat); + sMeta->tableMeta = pTableMeta; + nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL); + } + info->currSTableMeta = sMeta->tableMeta; + superKV = sMeta->tags; + + if(unlikely(taosArrayGetSize(superKV) == 0)){ + isSuperKVInit = false; + } + taosArraySetSize(maxKVs, 0); + } + }else{ + taosArraySetSize(maxKVs, 0); + } + + taosArraySetSize(preLineKV, 0); + const char *sql = data; + while (sql < sqlEnd) { + JUMP_SPACE(sql, sqlEnd) + if (unlikely(*sql == '\0')) break; + + const char *key = sql; + size_t keyLen = 0; + + // parse key + while (sql < sqlEnd) { + if (unlikely(*sql == SPACE)) { + smlBuildInvalidDataMsg(msg, "invalid data", sql); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(*sql == EQUAL)) { + keyLen = sql - key; + sql++; + break; + } + sql++; + } + + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } +// if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { +// smlBuildInvalidDataMsg(msg, "dumplicate key", key); +// return TSDB_CODE_TSC_DUP_NAMES; +// } + + // parse value + const char *value = sql; + size_t valueLen = 0; + while (sql < sqlEnd) { + // parse value + if (unlikely(*sql == SPACE)) { + break; + } + if (unlikely(*sql == EQUAL)) { + smlBuildInvalidDataMsg(msg, "invalid data", sql); + return TSDB_CODE_SML_INVALID_DATA; + } + sql++; + } + valueLen = sql - value; + + if (unlikely(valueLen == 0)) { + smlBuildInvalidDataMsg(msg, "invalid value", value); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (unlikely(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + + SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen}; + + if(info->dataFormat){ + if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(isSameMeasure){ + if(unlikely(cnt >= taosArrayGetSize(maxKVs))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(maxKVs, cnt); + if(unlikely(kv.length > maxKV->length)){ + maxKV->length = kv.length; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + if(unlikely(NULL == tableMeta)){ + uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); + return TSDB_CODE_SML_INTERNAL_ERROR; + } + + SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + if(isSuperKVInit){ + if(unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(superKV, cnt); + if(unlikely(kv.length > maxKV->length)) { + maxKV->length = kv.length; + }else{ + kv.length = maxKV->length; + } + info->needModifySchema = true; + + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + taosArrayPush(superKV, &kv); + } + taosArrayPush(maxKVs, &kv); + } + }else{ + taosArrayPush(maxKVs, &kv); + } + taosArrayPush(preLineKV, &kv); + cnt++; + } + SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); + if (unlikely(tinfo == NULL)) { + tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); + if (!tinfo) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tinfo->tags = taosArrayDup(preLineKV, NULL); + + smlSetCTableName(tinfo); + tinfo->uid = info->uid++; + if (info->dataFormat) { + info->currSTableMeta->uid = tinfo->uid; + tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); + if (tinfo->tableDataCtx == NULL) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); + smlDestroyTableInfo(info, tinfo); + return TSDB_CODE_SML_INVALID_DATA; + } + } + + SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); + *key = *elements; + tinfo->key = key; + nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet); + } + if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; + + return TSDB_CODE_SUCCESS; +} + +// format: =[ =] +int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements) { + if (!sql) return TSDB_CODE_SML_INVALID_DATA; + + // parse metric + smlParseTelnetElement(&sql, sqlEnd, &elements->measure, &elements->measureLen); + if (unlikely((!(elements->measure) || IS_INVALID_TABLE_LEN(elements->measureLen)))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + + // parse timestamp + smlParseTelnetElement(&sql, sqlEnd, &elements->timestamp, &elements->timestampLen); + if (unlikely(!elements->timestamp || elements->timestampLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); + return TSDB_CODE_SML_INVALID_DATA; + } + + bool needConverTime = false; // get TS before parse tag(get meta), so need conver time + if(info->dataFormat && info->currSTableMeta == NULL){ + needConverTime = true; + } + int64_t ts = smlParseOpenTsdbTime(info, elements->timestamp, elements->timestampLen); + if (unlikely(ts < 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); + return TSDB_CODE_INVALID_TIMESTAMP; + } + SSmlKv kvTs = { .key = TS, .keyLen = TS_LEN, .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + + // parse value + smlParseTelnetElement(&sql, sqlEnd, &elements->cols, &elements->colsLen); + if (unlikely(!elements->cols || elements->colsLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; + if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_VALUE; + } + + JUMP_SPACE(sql, sqlEnd) + + elements->tags = sql; + elements->tagsLen = sqlEnd - sql; + if (unlikely(!elements->tags || elements->tagsLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + int ret = smlParseTelnetTags(info, sql, sqlEnd, elements, &info->msgBuf); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + return ret; + } + + if(unlikely(info->reRun)){ + return TSDB_CODE_SUCCESS; + } + + if(info->dataFormat){ + if(needConverTime) { + kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision); + } + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); + } + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildRow(info->currTableDataCtx); + } + clearColValArray(info->currTableDataCtx->pValues); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + return ret; + } + }else{ + if(elements->colArray == NULL){ + elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + } + taosArrayPush(elements->colArray, &kvTs); + taosArrayPush(elements->colArray, &kv); + } + info->preLine = *elements; + + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 82ea9e0d8f8f6e70829567ce34e9b9d1ae444da4..f5b65371a7c90393b72ee059afb8060f1b373891 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -152,9 +152,10 @@ int32_t stmtRestoreQueryFields(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } -int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, SName* tbName, const char* sTableName, bool autoCreateTbl) { +int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, SName* tbName, const char* sTableName, + bool autoCreateTbl) { STscStmt* pStmt = (STscStmt*)stmt; - char tbFName[TSDB_TABLE_FNAME_LEN]; + char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(tbName, tbFName); memcpy(&pStmt->bInfo.sname, tbName, sizeof(*tbName)); @@ -171,12 +172,11 @@ int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, return TSDB_CODE_SUCCESS; } -int32_t stmtUpdateExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockHash, bool autoCreateTbl) { +int32_t stmtUpdateExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockHash) { STscStmt* pStmt = (STscStmt*)stmt; pStmt->sql.pVgHash = pVgHash; pStmt->exec.pBlockHash = pBlockHash; - pStmt->exec.autoCreateTbl = autoCreateTbl; return TSDB_CODE_SUCCESS; } @@ -186,7 +186,7 @@ int32_t stmtUpdateInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, SNam STscStmt* pStmt = (STscStmt*)stmt; STMT_ERR_RET(stmtUpdateBindInfo(stmt, pTableMeta, tags, tbName, sTableName, autoCreateTbl)); - STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash, autoCreateTbl)); + STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash)); pStmt->sql.autoCreateTbl = autoCreateTbl; @@ -214,16 +214,16 @@ int32_t stmtCacheBlock(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } - STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + STableDataCxt** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (!pSrc) { return TSDB_CODE_OUT_OF_MEMORY; } - STableDataBlocks* pDst = NULL; + STableDataCxt* pDst = NULL; - STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc)); + STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc, true)); SStmtTableCache cache = { - .pDataBlock = pDst, + .pDataCtx = pDst, .boundTags = pStmt->bInfo.boundTags, }; @@ -241,6 +241,8 @@ int32_t stmtCacheBlock(STscStmt* pStmt) { } int32_t stmtParseSql(STscStmt* pStmt) { + pStmt->exec.pCurrBlock = NULL; + SStmtCallback stmtCb = { .pStmt = pStmt, .getTbNameFn = stmtGetTbName, @@ -273,7 +275,7 @@ int32_t stmtCleanBindInfo(STscStmt* pStmt) { pStmt->bInfo.tbName[0] = 0; pStmt->bInfo.tbFName[0] = 0; if (!pStmt->bInfo.tagsCached) { - destroyBoundColumnInfo(pStmt->bInfo.boundTags); + qDestroyBoundColInfo(pStmt->bInfo.boundTags); taosMemoryFreeClear(pStmt->bInfo.boundTags); } memset(pStmt->bInfo.stbFName, 0, TSDB_TABLE_FNAME_LEN); @@ -289,29 +291,24 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) { size_t keyLen = 0; void* pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); while (pIter) { - STableDataBlocks* pBlocks = *(STableDataBlocks**)pIter; - char* key = taosHashGetKey(pIter, &keyLen); - STableMeta* pMeta = qGetTableMetaInDataBlock(pBlocks); + STableDataCxt* pBlocks = *(STableDataCxt**)pIter; + char* key = taosHashGetKey(pIter, &keyLen); + STableMeta* pMeta = qGetTableMetaInDataBlock(pBlocks); - if (keepTable && (strlen(pStmt->bInfo.tbFName) == keyLen) && strncmp(pStmt->bInfo.tbFName, key, keyLen) == 0) { - STMT_ERR_RET(qResetStmtDataBlock(pBlocks, true)); + if (keepTable && pBlocks == pStmt->exec.pCurrBlock) { + TSWAP(pBlocks->pData, pStmt->exec.pCurrTbData); + STMT_ERR_RET(qResetStmtDataBlock(pBlocks, false)); pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); continue; } - if (STMT_TYPE_MULTI_INSERT == pStmt->sql.type) { - qFreeStmtDataBlock(pBlocks); - } else { - qDestroyStmtDataBlock(pBlocks); - } + qDestroyStmtDataBlock(pBlocks); taosHashRemove(pStmt->exec.pBlockHash, key, keyLen); pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); } - pStmt->exec.autoCreateTbl = false; - if (keepTable) { return TSDB_CODE_SUCCESS; } @@ -319,6 +316,9 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) { taosHashCleanup(pStmt->exec.pBlockHash); pStmt->exec.pBlockHash = NULL; + tDestroySSubmitTbData(pStmt->exec.pCurrTbData, TSDB_MSG_FLG_ENCODE); + taosMemoryFreeClear(pStmt->exec.pCurrTbData); + STMT_ERR_RET(stmtCleanBindInfo(pStmt)); return TSDB_CODE_SUCCESS; @@ -337,8 +337,8 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { while (pIter) { SStmtTableCache* pCache = (SStmtTableCache*)pIter; - qDestroyStmtDataBlock(pCache->pDataBlock); - destroyBoundColumnInfo(pCache->boundTags); + qDestroyStmtDataBlock(pCache->pDataCtx); + qDestroyBoundColInfo(pCache->boundTags); taosMemoryFreeClear(pCache->boundTags); pIter = taosHashIterate(pStmt->sql.pTableCache, pIter); @@ -354,7 +354,8 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } -int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataBlocks* pDataBlock, STableDataBlocks** newBlock, uint64_t uid) { +int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataCxt* pDataBlock, STableDataCxt** newBlock, uint64_t uid, + uint64_t suid) { SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); SVgroupInfo vgInfo = {0}; SRequestConnInfo conn = {.pTrans = pStmt->taos->pAppInfo->pTransporter, @@ -366,7 +367,9 @@ int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataBlocks* pDataBlock, STab STMT_ERR_RET( taosHashPut(pStmt->sql.pVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo))); - STMT_ERR_RET(qRebuildStmtDataBlock(newBlock, pDataBlock, uid, vgInfo.vgId)); + STMT_ERR_RET(qRebuildStmtDataBlock(newBlock, pDataBlock, uid, suid, vgInfo.vgId, pStmt->sql.autoCreateTbl)); + + STMT_DLOG("tableDataCxt rebuilt, uid:%" PRId64 ", vgId:%d", uid, vgInfo.vgId); return TSDB_CODE_SUCCESS; } @@ -375,12 +378,13 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.needParse = true; pStmt->bInfo.inExecCache = false; - STableDataBlocks* pBlockInExec = - taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); - if (pBlockInExec) { + STableDataCxt** pCxtInExec = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + if (pCxtInExec) { pStmt->bInfo.needParse = false; pStmt->bInfo.inExecCache = true; + pStmt->exec.pCurrBlock = *pCxtInExec; + if (pStmt->sql.autoCreateTbl) { tscDebug("reuse stmt block for tb %s in execBlock", pStmt->bInfo.tbFName); return TSDB_CODE_SUCCESS; @@ -389,7 +393,10 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { if (NULL == pStmt->sql.pTableCache || taosHashGetSize(pStmt->sql.pTableCache) <= 0) { if (pStmt->bInfo.inExecCache) { - ASSERT(taosHashGetSize(pStmt->exec.pBlockHash) == 1); + if(ASSERT(taosHashGetSize(pStmt->exec.pBlockHash) == 1)){ + tscError("stmtGetFromCache error"); + return TSDB_CODE_TSC_STMT_CACHE_ERROR; + } pStmt->bInfo.needParse = false; tscDebug("reuse stmt block for tb %s in execBlock", pStmt->bInfo.tbFName); return TSDB_CODE_SUCCESS; @@ -407,18 +414,18 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pStmt->bInfo.tbSuid, sizeof(pStmt->bInfo.tbSuid)); if (pCache) { pStmt->bInfo.needParse = false; - pStmt->exec.autoCreateTbl = true; - pStmt->bInfo.tbUid = 0; - STableDataBlocks* pNewBlock = NULL; - STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataBlock, &pNewBlock, 0)); + STableDataCxt* pNewBlock = NULL; + STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataCtx, &pNewBlock, 0, pStmt->bInfo.tbSuid)); if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + pStmt->exec.pCurrBlock = pNewBlock; + tscDebug("reuse stmt block for tb %s in sqlBlock, suid:0x%" PRIx64, pStmt->bInfo.tbFName, pStmt->bInfo.tbSuid); return TSDB_CODE_SUCCESS; @@ -489,14 +496,16 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.boundTags = pCache->boundTags; pStmt->bInfo.tagsCached = true; - STableDataBlocks* pNewBlock = NULL; - STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataBlock, &pNewBlock, uid)); + STableDataCxt* pNewBlock = NULL; + STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataCtx, &pNewBlock, uid, suid)); if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + pStmt->exec.pCurrBlock = pNewBlock; + tscDebug("tb %s in sqlBlock list, set to current", pStmt->bInfo.tbFName); return TSDB_CODE_SUCCESS; @@ -614,8 +623,8 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { return TSDB_CODE_SUCCESS; } - STableDataBlocks** pDataBlock = - (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + STableDataCxt** pDataBlock = + (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_APP_ERROR); @@ -626,8 +635,6 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); - pStmt->exec.autoCreateTbl = true; - return TSDB_CODE_SUCCESS; } @@ -637,8 +644,8 @@ int stmtFetchTagFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); } - STableDataBlocks** pDataBlock = - (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + STableDataCxt** pDataBlock = + (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_APP_ERROR); @@ -655,8 +662,8 @@ int stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); } - STableDataBlocks** pDataBlock = - (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + STableDataCxt** pDataBlock = + (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_APP_ERROR); @@ -729,11 +736,18 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { return TSDB_CODE_SUCCESS; } - STableDataBlocks** pDataBlock = - (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); - if (NULL == pDataBlock) { - tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); - STMT_ERR_RET(TSDB_CODE_APP_ERROR); + STableDataCxt** pDataBlock = NULL; + + if (pStmt->exec.pCurrBlock) { + pDataBlock = &pStmt->exec.pCurrBlock; + } else { + pDataBlock = + (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + if (NULL == pDataBlock) { + tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); + STMT_ERR_RET(TSDB_CODE_TSC_STMT_CACHE_ERROR); + } + pStmt->exec.pCurrBlock = *pDataBlock; } if (colIdx < 0) { @@ -779,10 +793,10 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) { int32_t code = 0; int32_t finalCode = 0; size_t keyLen = 0; - STableDataBlocks** pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); + void* pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); while (pIter) { - STableDataBlocks* pBlock = *pIter; - char* key = taosHashGetKey(pIter, &keyLen); + STableDataCxt* pBlock = *(STableDataCxt**)pIter; + char* key = taosHashGetKey(pIter, &keyLen); STableMeta* pMeta = qGetTableMetaInDataBlock(pBlock); if (pMeta->uid) { @@ -848,7 +862,7 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) { pMeta->uid = pTableMeta->uid; pStmt->bInfo.tbUid = pTableMeta->uid; - taosMemoryFree(pTableMeta); + taosMemoryFree(pTableMeta); } pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); @@ -861,7 +875,6 @@ int stmtExec(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; int32_t code = 0; SSubmitRsp* pRsp = NULL; - bool autoCreateTbl = pStmt->exec.autoCreateTbl; STMT_DLOG_E("start to exec"); @@ -870,8 +883,13 @@ int stmtExec(TAOS_STMT* stmt) { if (STMT_TYPE_QUERY == pStmt->sql.type) { launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); } else { + tDestroySSubmitTbData(pStmt->exec.pCurrTbData, TSDB_MSG_FLG_ENCODE); + taosMemoryFreeClear(pStmt->exec.pCurrTbData); + + STMT_ERR_RET(qCloneCurrentTbData(pStmt->exec.pCurrBlock, &pStmt->exec.pCurrTbData)); + STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, pStmt->exec.pBlockHash)); - launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, (autoCreateTbl ? (void**)&pRsp : NULL)); + launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); } if (pStmt->exec.pRequest->code && NEED_CLIENT_HANDLE_ERROR(pStmt->exec.pRequest->code)) { @@ -894,15 +912,6 @@ _return: stmtCleanExecInfo(pStmt, (code ? false : true), false); - if (TSDB_CODE_SUCCESS == code && autoCreateTbl) { - if (NULL == pRsp) { - tscError("no submit resp got for auto create table"); - code = TSDB_CODE_APP_ERROR; - } else { - code = stmtUpdateTableUid(pStmt, pRsp); - } - } - tFreeSSubmitRsp(pRsp); ++pStmt->sql.runTimes; diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 8905dc21320322695aa0cb64c8433206fca5bec1..01c99c6e9ed16048a715839dc4ccb18ca04c3852 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -418,7 +418,10 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { static void tmqCommitRspCountDown(SMqCommitCbParamSet* pParamSet) { int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); - ASSERT(waitingRspNum >= 0); + if(ASSERT(waitingRspNum >= 0)){ + tscError("tmqCommitRspCountDown error:%d", waitingRspNum); + return; + } if (waitingRspNum == 0) { tmqCommitDone(pParamSet); } @@ -909,10 +912,12 @@ void tmqFreeImpl(void* handle) { tmq_t* tmq = (tmq_t*)handle; // TODO stop timer - tmqClearUnhandleMsg(tmq); - if (tmq->mqueue) taosCloseQueue(tmq->mqueue); + if (tmq->mqueue) { + tmqClearUnhandleMsg(tmq); + taosCloseQueue(tmq->mqueue); + } if (tmq->delayedTask) taosCloseQueue(tmq->delayedTask); - if (tmq->qall) taosFreeQall(tmq->qall); + taosFreeQall(tmq->qall); tsem_destroy(&tmq->rspSem); diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 54778e87a788190c55fafc0e4d0db732677d795d..76911e229a569403ccdcc36b4b90470d3f02618e 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -25,7 +25,7 @@ #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" -#include "../src/clientSml.c" +#include "../inc/clientSml.h" #include "taos.h" int main(int argc, char **argv) { @@ -40,11 +40,14 @@ TEST(testCase, smlParseInfluxString_Test) { msgBuf.len = 256; SSmlLineInfo elements = {0}; + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->protocol = TSDB_SML_LINE_PROTOCOL; + info->dataFormat = false; // case 1 char *tmp = "\\,st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; char *sql = (char *)taosMemoryCalloc(256, 1); memcpy(sql, tmp, strlen(tmp) + 1); - int ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measureLen, strlen(",st")); @@ -58,28 +61,23 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 2 false tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); - - // case 3 false - tmp = "st, t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); - ASSERT_EQ(ret, 0); - ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 1); - ASSERT_EQ(elements.colsLen, strlen("t1=3,t2=4,t3=t3")); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 4 tag is null tmp = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measureLen, strlen("st")); @@ -93,12 +91,14 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 5 tag is null tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql + 1); ASSERT_EQ(elements.measureLen, strlen("st")); @@ -111,91 +111,104 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + 1 + elements.measureTagsLen + 3 + elements.colsLen + 2); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 6 tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; + smlClearForRerun(info); // case 7 tmp = " st , "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); - ASSERT_EQ(ret, 0); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); + ASSERT_NE(ret, 0); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 8 false tmp = ", st , "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; + taosMemoryFree(sql); + smlDestroyInfo(info); + } TEST(testCase, smlParseCols_Error_Test) { - const char *data[] = {"c=\"89sd", // binary, nchar - "c=j\"89sd\"", - "c=\"89sd\"k", - "c=u", // bool - "c=truet", - "c=f64", // double - "c=8f64f", - "c=8ef64", - "c=f32", // float - "c=8f32f", - "c=8wef32", - "c=-3.402823466e+39f32", - "c=", // double - "c=8f", - "c=8we", - "c=i8", // tiny int - "c=-8i8f", - "c=8wei8", - "c=-999i8", - "c=u8", // u tiny int - "c=8fu8", - "c=8weu8", - "c=999u8", - "c=-8u8", - "c=i16", // small int - "c=8fi16u", - "c=8wei16", - "c=-67787i16", - "c=u16", // u small int - "c=8u16f", - "c=8weu16", - "c=-9u16", - "c=67787u16", - "c=i32", // int - "c=8i32f", - "c=8wei32", - "c=2147483649i32", - "c=u32", // u int - "c=8u32f", - "c=8weu32", - "c=-4u32", - "c=42949672958u32", - "c=i64", // big int - "c=8i64i", - "c=8wei64", - "c=-9223372036854775809i64", - "c=i", // big int - "c=8fi", - "c=8wei", - "c=9223372036854775808i", - "c=u64", // u big int - "c=8u64f", - "c=8weu64", - "c=-3.402823466e+39u64", - "c=-339u64", - "c=18446744073709551616u64", - "c=1,c=2", - "c=1=2"}; - - SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + const char *data[] = {"st,t=1 c=\"89sd 1626006833639000000", // binary, nchar + "st,t=1 c=j\"89sd\" 1626006833639000000", + "st,t=1 c=\"89sd\"k 1626006833639000000", + "st,t=1 c=u 1626006833639000000", // bool + "st,t=1 c=truet 1626006833639000000", + "st,t=1 c=f64 1626006833639000000", // double + "st,t=1 c=8f64f 1626006833639000000", + "st,t=1 c=8ef64 1626006833639000000", + "st,t=1 c=f32 1626006833639000000", // float + "st,t=1 c=8f32f 1626006833639000000", + "st,t=1 c=8wef32 1626006833639000000", + "st,t=1 c=-3.402823466e+39f32 1626006833639000000", + "st,t=1 c= 1626006833639000000", // double + "st,t=1 c=8f 1626006833639000000", + "st,t=1 c=8we 1626006833639000000", + "st,t=1 c=i8 1626006833639000000", // tiny int + "st,t=1 c=-8i8f 1626006833639000000", + "st,t=1 c=8wei8 1626006833639000000", + "st,t=1 c=-999i8 1626006833639000000", + "st,t=1 c=u8 1626006833639000000", // u tiny int + "st,t=1 c=8fu8 1626006833639000000", + "st,t=1 c=8weu8 1626006833639000000", + "st,t=1 c=999u8 1626006833639000000", + "st,t=1 c=-8u8 1626006833639000000", + "st,t=1 c=i16 1626006833639000000", // small int + "st,t=1 c=8fi16u 1626006833639000000", + "st,t=1 c=8wei16 1626006833639000000", + "st,t=1 c=-67787i16 1626006833639000000", + "st,t=1 c=u16 1626006833639000000", // u small int + "st,t=1 c=8u16f 1626006833639000000", + "st,t=1 c=8weu16 1626006833639000000", + "st,t=1 c=-9u16 1626006833639000000", + "st,t=1 c=67787u16 1626006833639000000", + "st,t=1 c=i32 1626006833639000000", // int + "st,t=1 c=8i32f 1626006833639000000", + "st,t=1 c=8wei32 1626006833639000000", + "st,t=1 c=2147483649i32 1626006833639000000", + "st,t=1 c=u32 1626006833639000000", // u int + "st,t=1 c=8u32f 1626006833639000000", + "st,t=1 c=8weu32 1626006833639000000", + "st,t=1 c=-4u32 1626006833639000000", + "st,t=1 c=42949672958u32 1626006833639000000", + "st,t=1 c=i64 1626006833639000000", // big int + "st,t=1 c=8i64i 1626006833639000000", + "st,t=1 c=8wei64 1626006833639000000", + "st,t=1 c=-9223372036854775809i64 1626006833639000000", + "st,t=1 c=i 1626006833639000000", // big int + "st,t=1 c=8fi 1626006833639000000", + "st,t=1 c=8wei 1626006833639000000", + "st,t=1 c=9223372036854775808i 1626006833639000000", + "st,t=1 c=u64 1626006833639000000", // u big int + "st,t=1 c=8u64f 1626006833639000000", + "st,t=1 c=8weu64 1626006833639000000", + "st,t=1 c=-3.402823466e+39u64 1626006833639000000", + "st,t=1 c=-339u64 1626006833639000000", + "st,t=1 c=18446744073709551616u64 1626006833639000000", + "st,t=1 c=1=2 1626006833639000000"}; + + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->protocol = TSDB_SML_LINE_PROTOCOL; + info->dataFormat = false; for (int i = 0; i < sizeof(data) / sizeof(data[0]); i++) { char msg[256] = {0}; SSmlMsgBuf msgBuf; @@ -204,76 +217,14 @@ TEST(testCase, smlParseCols_Error_Test) { int32_t len = strlen(data[i]); char *sql = (char *)taosMemoryCalloc(256, 1); memcpy(sql, data[i], len + 1); - SArray *cols = taosArrayInit(8, POINTER_BYTES); - int32_t ret = smlParseCols(sql, len, cols, NULL, false, dumplicateKey, &msgBuf); - printf("i:%d\n", i); + SSmlLineInfo elements = {0}; + int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); +// printf("i:%d\n", i); ASSERT_NE(ret, TSDB_CODE_SUCCESS); - taosHashClear(dumplicateKey); taosMemoryFree(sql); - for (int j = 0; j < taosArrayGetSize(cols); j++) { - void *kv = taosArrayGetP(cols, j); - taosMemoryFree(kv); - } - taosArrayDestroy(cols); - } - taosHashCleanup(dumplicateKey); -} - -TEST(testCase, smlParseCols_tag_Test) { - char msg[256] = {0}; - SSmlMsgBuf msgBuf; - msgBuf.buf = msg; - msgBuf.len = 256; - - SArray *cols = taosArrayInit(16, POINTER_BYTES); - ASSERT_NE(cols, nullptr); - SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - - const char *data = - "cbin=\"passit " - "helloc\",cnch=L\"iisdfsf\",cbool=false,cf64=4.31f64,cf64_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=" - "898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_" - "=l\"iuwq\""; - int32_t len = strlen(data); - int32_t ret = smlParseCols(data, len, cols, NULL, true, dumplicateKey, &msgBuf); - ASSERT_EQ(ret, TSDB_CODE_SUCCESS); - int32_t size = taosArrayGetSize(cols); - ASSERT_EQ(size, 19); - - // nchar - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, 0); - ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0); - ASSERT_EQ(kv->keyLen, 4); - ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); - ASSERT_EQ(kv->length, 15); - ASSERT_EQ(strncasecmp(kv->value, "\"passit", 7), 0); - - // nchar - kv = (SSmlKv *)taosArrayGetP(cols, 3); - ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); - ASSERT_EQ(kv->keyLen, 4); - ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); - ASSERT_EQ(kv->length, 7); - ASSERT_EQ(strncasecmp(kv->value, "4.31f64", 7), 0); - - for (int i = 0; i < size; i++) { - void *tmp = taosArrayGetP(cols, i); - taosMemoryFree(tmp); + taosArrayDestroy(elements.colArray); } - taosArrayClear(cols); - - // test tag is null - data = "t=3e"; - len = 0; - memset(msgBuf.buf, 0, msgBuf.len); - taosHashClear(dumplicateKey); - ret = smlParseCols(data, len, cols, NULL, true, dumplicateKey, &msgBuf); - ASSERT_EQ(ret, TSDB_CODE_SUCCESS); - size = taosArrayGetSize(cols); - ASSERT_EQ(size, 0); - - taosArrayDestroy(cols); - taosHashCleanup(dumplicateKey); + smlDestroyInfo(info); } TEST(testCase, smlParseCols_Test) { @@ -281,202 +232,183 @@ TEST(testCase, smlParseCols_Test) { SSmlMsgBuf msgBuf; msgBuf.buf = msg; msgBuf.len = 256; - - SArray *cols = taosArrayInit(16, POINTER_BYTES); - ASSERT_NE(cols, nullptr); - - SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->protocol = TSDB_SML_LINE_PROTOCOL; + info->dataFormat = false; + SSmlLineInfo elements = {0}; + info->msgBuf = msgBuf; const char *data = - "cb\\=in=\"pass\\,it " + "st,t=1 cb\\=in=\"pass\\,it " "hello,c=2\",cnch=L\"ii\\=sdfsf\",cbool=false,cf64=4.31f64,cf64_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=" "233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t," - "cboolf=f,cnch_=l\"iuwq\""; + "cboolf=f,cnch_=l\"iuwq\" 1626006833639000000"; int32_t len = strlen(data); char *sql = (char *)taosMemoryCalloc(1024, 1); memcpy(sql, data, len + 1); - int32_t ret = smlParseCols(sql, len, cols, NULL, false, dumplicateKey, &msgBuf); + int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); ASSERT_EQ(ret, TSDB_CODE_SUCCESS); - int32_t size = taosArrayGetSize(cols); - ASSERT_EQ(size, 19); + int32_t size = taosArrayGetSize(elements.colArray); + ASSERT_EQ(size, 20); // binary - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, 0); + SSmlKv *kv = (SSmlKv *)taosArrayGet(elements.colArray, 1); ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY); ASSERT_EQ(kv->length, 17); ASSERT_EQ(strncasecmp(kv->value, "pass,it ", 8), 0); - taosMemoryFree(kv); // nchar - kv = (SSmlKv *)taosArrayGetP(cols, 1); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 2); ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->length, 8); ASSERT_EQ(strncasecmp(kv->value, "ii=sd", 5), 0); - taosMemoryFree(kv); // bool - kv = (SSmlKv *)taosArrayGetP(cols, 2); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 3); ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, false); - taosMemoryFree(kv); // double - kv = (SSmlKv *)taosArrayGetP(cols, 3); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 4); ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); ASSERT_EQ(kv->length, 8); // ASSERT_EQ(kv->d, 4.31); printf("4.31 = kv->d:%f\n", kv->d); - taosMemoryFree(kv); // float - kv = (SSmlKv *)taosArrayGetP(cols, 4); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 5); ASSERT_EQ(strncasecmp(kv->key, "cf64_", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); ASSERT_EQ(kv->length, 8); // ASSERT_EQ(kv->f, 8.32); printf("8.32 = kv->d:%f\n", kv->d); - taosMemoryFree(kv); // float - kv = (SSmlKv *)taosArrayGetP(cols, 5); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 6); ASSERT_EQ(strncasecmp(kv->key, "cf32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); ASSERT_EQ(kv->length, 4); // ASSERT_EQ(kv->f, 8.23); printf("8.23 = kv->f:%f\n", kv->f); - taosMemoryFree(kv); // tiny int - kv = (SSmlKv *)taosArrayGetP(cols, 6); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 7); ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0); ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, -34); - taosMemoryFree(kv); // unsigned tiny int - kv = (SSmlKv *)taosArrayGetP(cols, 7); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 8); ASSERT_EQ(strncasecmp(kv->key, "cu8", 3), 0); ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UTINYINT); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->u, 89); - taosMemoryFree(kv); // small int - kv = (SSmlKv *)taosArrayGetP(cols, 8); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 9); ASSERT_EQ(strncasecmp(kv->key, "ci16", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_SMALLINT); ASSERT_EQ(kv->length, 2); ASSERT_EQ(kv->u, 233); - taosMemoryFree(kv); // unsigned smallint - kv = (SSmlKv *)taosArrayGetP(cols, 9); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 10); ASSERT_EQ(strncasecmp(kv->key, "cu16", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_USMALLINT); ASSERT_EQ(kv->length, 2); ASSERT_EQ(kv->u, 898); - taosMemoryFree(kv); // int - kv = (SSmlKv *)taosArrayGetP(cols, 10); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 11); ASSERT_EQ(strncasecmp(kv->key, "ci32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_INT); ASSERT_EQ(kv->length, 4); ASSERT_EQ(kv->u, 98289); - taosMemoryFree(kv); // unsigned int - kv = (SSmlKv *)taosArrayGetP(cols, 11); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 12); ASSERT_EQ(strncasecmp(kv->key, "cu32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UINT); ASSERT_EQ(kv->length, 4); ASSERT_EQ(kv->u, 12323); - taosMemoryFree(kv); // bigint - kv = (SSmlKv *)taosArrayGetP(cols, 12); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 13); ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(kv->length, 8); ASSERT_EQ(kv->i, -89238); - taosMemoryFree(kv); // bigint - kv = (SSmlKv *)taosArrayGetP(cols, 13); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 14); ASSERT_EQ(strncasecmp(kv->key, "ci", 2), 0); ASSERT_EQ(kv->keyLen, 2); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(kv->length, 8); ASSERT_EQ(kv->i, 989); - taosMemoryFree(kv); // unsigned bigint - kv = (SSmlKv *)taosArrayGetP(cols, 14); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 15); ASSERT_EQ(strncasecmp(kv->key, "cu64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UBIGINT); ASSERT_EQ(kv->length, 8); ASSERT_EQ(kv->u, 8989323); - taosMemoryFree(kv); // bool - kv = (SSmlKv *)taosArrayGetP(cols, 15); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 16); ASSERT_EQ(strncasecmp(kv->key, "cbooltrue", 9), 0); ASSERT_EQ(kv->keyLen, 9); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, true); - taosMemoryFree(kv); // bool - kv = (SSmlKv *)taosArrayGetP(cols, 16); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 17); ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, true); - taosMemoryFree(kv); // bool - kv = (SSmlKv *)taosArrayGetP(cols, 17); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 18); ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, false); - taosMemoryFree(kv); // nchar - kv = (SSmlKv *)taosArrayGetP(cols, 18); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 19); ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->length, 4); ASSERT_EQ(strncasecmp(kv->value, "iuwq", 4), 0); - taosMemoryFree(kv); - taosArrayDestroy(cols); - taosHashCleanup(dumplicateKey); + taosArrayDestroy(elements.colArray); taosMemoryFree(sql); + smlDestroyInfo(info); } TEST(testCase, smlGetTimestampLen_Test) { @@ -515,7 +447,9 @@ TEST(testCase, smlParseNumber_Test) { } TEST(testCase, smlParseTelnetLine_error_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->dataFormat = false; + info->protocol = TSDB_SML_TELNET_PROTOCOL; ASSERT_NE(info, nullptr); const char *sql[] = { @@ -532,479 +466,95 @@ TEST(testCase, smlParseTelnetLine_error_Test) { "sys.procs.running 1479496100 42 ", "sys.procs.running 1479496100 42 host= ", "sys.procs.running 1479496100 42or host=web01", - "sys.procs.running 1479496100 true host=web01", - "sys.procs.running 1479496100 \"binary\" host=web01", - "sys.procs.running 1479496100 L\"rfr\" host=web01", +// "sys.procs.running 1479496100 true host=web01", +// "sys.procs.running 1479496100 \"binary\" host=web01", +// "sys.procs.running 1479496100 L\"rfr\" host=web01", "sys.procs.running 1479496100 42 host=web01 cpu= ", - "sys.procs.running 1479496100 42 host=web01 host=w2", "sys.procs.running 1479496100 42 host=web01 host", "sys.procs.running 1479496100 42 host=web01=er", "sys.procs.running 1479496100 42 host= web01", }; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - int ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); + SSmlLineInfo elements = {0}; + int ret = smlParseTelnetString(info, (char*)sql[i], (char*)(sql[i] + strlen(sql[i])), &elements); +// printf("i:%d\n", i); ASSERT_NE(ret, 0); } smlDestroyInfo(info); } -TEST(testCase, smlParseTelnetLine_diff_type_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - - const char *sql[] = {"sys.procs.running 1479496104000 42 host=web01", - "sys.procs.running 1479496104000 42u8 host=web01", - "appywjnuct 1626006833641 True id=\"appywjnuct_40601_49808_1\" t0=t t1=127i8 " - "id=\"appywjnuct_40601_49808_2\" t2=32767i16 t3=2147483647i32 t4=9223372036854775807i64 " - "t5=11.12345f32 t6=22.123456789f64 t7=\"binaryTagValue\" t8=L\"ncharTagValue\""}; - - int ret = TSDB_CODE_SUCCESS; - for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); - if (ret != TSDB_CODE_SUCCESS) break; - } - ASSERT_NE(ret, 0); - smlDestroyInfo(info); -} - -TEST(testCase, smlParseTelnetLine_json_error_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); +TEST(testCase, smlParseTelnetLine_Test) { + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->dataFormat = false; + info->protocol = TSDB_SML_TELNET_PROTOCOL; ASSERT_NE(info, nullptr); const char *sql[] = { - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 13468464009999333322222223,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"web01\",\n" - " \"dc\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400i,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"web01\",\n" - " \"dc\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"groupid\": { \n" - " \"value\" : 2,\n" - " \"type\" : \"nchar\"\n" - " },\n" - " \"location\": { \n" - " \"value\" : \"北京\",\n" - " \"type\" : \"binary\"\n" - " },\n" - " \"id\": \"d1001\"\n" - " }\n" - " },\n" - "]", + "twudyr 1626006833641 \"abcd`~!@#$%^&*()_-{[}]|:;<.>?lfjal\" id=twudyr_17102_17825 t0=t t1=127i8 t2=32767i16 t3=2147483647i32 t4=9223372036854775807i64 t5=11.12345f32 t6=22.123456789f64 t7=\"abcd`~!@#$%^&*()_-{[}]|:;<.>?lfjal\" t8=L\"abcd`~!@#$%^&*()_-{[}]|:;<.>?lfjal\"", }; - - int ret = TSDB_CODE_SUCCESS; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); - ASSERT_NE(ret, 0); + SSmlLineInfo elements = {0}; + int ret = smlParseTelnetString(info, (char*)sql[i], (char*)(sql[i] + strlen(sql[i])), &elements); +// printf("i:%d\n", i); + ASSERT_EQ(ret, 0); } smlDestroyInfo(info); } -TEST(testCase, smlParseTelnetLine_diff_json_type1_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - - const char *sql[] = { - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": 8\n" - " }\n" - " },\n" - "]", - }; - - int ret = TSDB_CODE_SUCCESS; - for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); - if (ret != TSDB_CODE_SUCCESS) break; - } - ASSERT_NE(ret, 0); - smlDestroyInfo(info); -} - -TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - - const char *sql[] = { - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": \"18\",\n" - " \"tags\": {\n" - " \"host\": \"fff\"\n" - " }\n" - " },\n" - "]", - }; - int ret = TSDB_CODE_SUCCESS; - for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); - if (ret != TSDB_CODE_SUCCESS) break; - } - ASSERT_NE(ret, 0); - smlDestroyInfo(info); -} - -TEST(testCase, sml_col_4096_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - - const char *sql[] = { - "spgwgvldxv,id=spgwgvldxv_1,t0=f " - "c0=t,c1=t,c2=t,c3=t,c4=t,c5=t,c6=t,c7=t,c8=t,c9=t,c10=t,c11=t,c12=t,c13=t,c14=t,c15=t,c16=t,c17=t,c18=t,c19=t," - "c20=t,c21=t,c22=t,c23=t,c24=t,c25=t,c26=t,c27=t,c28=t,c29=t,c30=t,c31=t,c32=t,c33=t,c34=t,c35=t,c36=t,c37=t,c38=" - "t,c39=t,c40=t,c41=t,c42=t,c43=t,c44=t,c45=t,c46=t,c47=t,c48=t,c49=t,c50=t,c51=t,c52=t,c53=t,c54=t,c55=t,c56=t," - "c57=t,c58=t,c59=t,c60=t,c61=t,c62=t,c63=t,c64=t,c65=t,c66=t,c67=t,c68=t,c69=t,c70=t,c71=t,c72=t,c73=t,c74=t,c75=" - "t,c76=t,c77=t,c78=t,c79=t,c80=t,c81=t,c82=t,c83=t,c84=t,c85=t,c86=t,c87=t,c88=t,c89=t,c90=t,c91=t,c92=t,c93=t," - "c94=t,c95=t,c96=t,c97=t,c98=t,c99=t,c100=t," - "c101=t,c102=t,c103=t,c104=t,c105=t,c106=t,c107=t,c108=t,c109=t,c110=t,c111=t,c112=t,c113=t,c114=t,c115=t,c116=t," - "c117=t,c118=t,c119=t,c120=t,c121=t,c122=t,c123=t,c124=t,c125=t,c126=t,c127=t,c128=t,c129=t,c130=t,c131=t,c132=t," - "c133=t,c134=t,c135=t,c136=t,c137=t,c138=t,c139=t,c140=t,c141=t,c142=t,c143=t,c144=t,c145=t,c146=t,c147=t,c148=t," - "c149=t,c150=t,c151=t,c152=t,c153=t,c154=t,c155=t,c156=t,c157=t,c158=t,c159=t,c160=t,c161=t,c162=t,c163=t,c164=t," - "c165=t,c166=t,c167=t,c168=t,c169=t,c170=t,c171=t,c172=t,c173=t,c174=t,c175=t,c176=t,c177=t,c178=t,c179=t,c180=t," - "c181=t,c182=t,c183=t,c184=t,c185=t,c186=t,c187=t,c188=t,c189=t," - "c190=t,c191=t,c192=t,c193=t,c194=t,c195=t,c196=t,c197=t,c198=t,c199=t,c200=t,c201=t,c202=t,c203=t,c204=t,c205=t," - "c206=t,c207=t,c208=t,c209=t,c210=t,c211=t,c212=t,c213=t,c214=t,c215=t,c216=t,c217=t,c218=t,c219=t,c220=t,c221=t," - "c222=t,c223=t,c224=t,c225=t,c226=t,c227=t,c228=t,c229=t,c230=t,c231=t,c232=t,c233=t,c234=t,c235=t,c236=t,c237=t," - "c238=t,c239=t,c240=t,c241=t,c242=t,c243=t,c244=t,c245=t,c246=t,c247=t,c248=t,c249=t,c250=t,c251=t,c252=t,c253=t," - "c254=t,c255=t,c256=t,c257=t,c258=t,c259=t,c260=t,c261=t,c262=t,c263=t,c264=t,c265=t,c266=t,c267=t,c268=t,c269=t," - "c270=t,c271=t,c272=t,c273=t,c274=t,c275=t,c276=t,c277=t,c278=t," - "c279=t,c280=t,c281=t,c282=t,c283=t,c284=t,c285=t,c286=t,c287=t,c288=t,c289=t,c290=t,c291=t,c292=t,c293=t,c294=t," - "c295=t,c296=t,c297=t,c298=t,c299=t,c300=t,c301=t,c302=t,c303=t,c304=t,c305=t,c306=t,c307=t,c308=t,c309=t,c310=t," - "c311=t,c312=t,c313=t,c314=t,c315=t,c316=t,c317=t,c318=t,c319=t,c320=t,c321=t,c322=t,c323=t,c324=t,c325=t,c326=t," - "c327=t,c328=t,c329=t,c330=t,c331=t,c332=t,c333=t,c334=t,c335=t,c336=t,c337=t,c338=t,c339=t,c340=t,c341=t,c342=t," - "c343=t,c344=t,c345=t,c346=t,c347=t,c348=t,c349=t,c350=t,c351=t,c352=t,c353=t,c354=t,c355=t,c356=t,c357=t,c358=t," - "c359=t,c360=t,c361=t,c362=t,c363=t,c364=t,c365=t,c366=t,c367=t,c368=t,c369=t,c370=t,c371=t,c372=t,c373=t,c374=t," - "c375=t,c376=t,c377=t,c378=t,c379=t,c380=t,c381=t,c382=t,c383=t,c384=t,c385=t,c386=t,c387=t,c388=t,c389=t,c390=t," - "c391=t,c392=t,c393=t,c394=t,c395=t,c396=t,c397=t,c398=t,c399=t,c400=t,c401=t,c402=t,c403=t,c404=t,c405=t,c406=t," - "c407=t,c408=t,c409=t,c410=t,c411=t,c412=t,c413=t,c414=t,c415=t,c416=t,c417=t,c418=t,c419=t,c420=t,c421=t,c422=t," - "c423=t,c424=t,c425=t,c426=t,c427=t,c428=t,c429=t,c430=t,c431=t,c432=t,c433=t,c434=t,c435=t,c436=t,c437=t,c438=t," - "c439=t,c440=t,c441=t,c442=t,c443=t,c444=t,c445=t,c446=t," - "c447=t,c448=t,c449=t,c450=t,c451=t,c452=t,c453=t,c454=t,c455=t,c456=t,c457=t,c458=t,c459=t,c460=t,c461=t,c462=t," - "c463=t,c464=t,c465=t,c466=t,c467=t,c468=t,c469=t,c470=t,c471=t,c472=t,c473=t,c474=t,c475=t,c476=t,c477=t,c478=t," - "c479=t,c480=t,c481=t,c482=t,c483=t,c484=t,c485=t,c486=t,c487=t,c488=t,c489=t,c490=t,c491=t,c492=t,c493=t,c494=t," - "c495=t,c496=t,c497=t,c498=t,c499=t,c500=t,c501=t,c502=t,c503=t,c504=t,c505=t,c506=t,c507=t,c508=t,c509=t,c510=t," - "c511=t,c512=t,c513=t,c514=t,c515=t,c516=t,c517=t,c518=t,c519=t,c520=t,c521=t,c522=t,c523=t,c524=t,c525=t,c526=t," - "c527=t,c528=t,c529=t,c530=t,c531=t,c532=t,c533=t,c534=t,c535=t,c536=t,c537=t,c538=t,c539=t,c540=t,c541=t,c542=t," - "c543=t,c544=t,c545=t,c546=t,c547=t,c548=t,c549=t,c550=t,c551=t,c552=t,c553=t,c554=t,c555=t,c556=t,c557=t,c558=t," - "c559=t,c560=t,c561=t,c562=t,c563=t,c564=t,c565=t,c566=t,c567=t,c568=t,c569=t,c570=t,c571=t,c572=t,c573=t,c574=t," - "c575=t,c576=t,c577=t,c578=t,c579=t,c580=t,c581=t,c582=t,c583=t,c584=t,c585=t,c586=t,c587=t,c588=t,c589=t,c590=t," - "c591=t,c592=t,c593=t,c594=t,c595=t,c596=t,c597=t,c598=t,c599=t,c600=t,c601=t,c602=t,c603=t,c604=t,c605=t,c606=t," - "c607=t,c608=t,c609=t,c610=t,c611=t,c612=t,c613=t,c614=t," - "c615=t,c616=t,c617=t,c618=t,c619=t,c620=t,c621=t,c622=t,c623=t,c624=t,c625=t,c626=t,c627=t,c628=t,c629=t,c630=t," - "c631=t,c632=t,c633=t,c634=t,c635=t,c636=t,c637=t,c638=t,c639=t,c640=t,c641=t,c642=t,c643=t,c644=t,c645=t,c646=t," - "c647=t,c648=t,c649=t,c650=t,c651=t,c652=t,c653=t,c654=t,c655=t,c656=t,c657=t,c658=t,c659=t,c660=t,c661=t,c662=t," - "c663=t,c664=t,c665=t,c666=t,c667=t,c668=t,c669=t,c670=t,c671=t,c672=t,c673=t,c674=t,c675=t,c676=t,c677=t,c678=t," - "c679=t,c680=t,c681=t,c682=t,c683=t,c684=t,c685=t,c686=t,c687=t,c688=t,c689=t,c690=t,c691=t,c692=t,c693=t,c694=t," - "c695=t,c696=t,c697=t,c698=t,c699=t,c700=t,c701=t,c702=t,c703=t,c704=t,c705=t,c706=t,c707=t,c708=t,c709=t,c710=t," - "c711=t,c712=t,c713=t,c714=t,c715=t,c716=t,c717=t,c718=t,c719=t,c720=t,c721=t,c722=t,c723=t,c724=t,c725=t,c726=t," - "c727=t,c728=t,c729=t,c730=t,c731=t,c732=t,c733=t,c734=t,c735=t,c736=t,c737=t,c738=t,c739=t,c740=t,c741=t,c742=t," - "c743=t,c744=t,c745=t,c746=t,c747=t,c748=t,c749=t,c750=t,c751=t,c752=t,c753=t,c754=t,c755=t,c756=t,c757=t,c758=t," - "c759=t,c760=t,c761=t,c762=t,c763=t,c764=t,c765=t,c766=t,c767=t,c768=t,c769=t,c770=t,c771=t,c772=t,c773=t,c774=t," - "c775=t,c776=t,c777=t,c778=t,c779=t,c780=t,c781=t,c782=t," - "c783=t,c784=t,c785=t,c786=t,c787=t,c788=t,c789=t,c790=t,c791=t,c792=t,c793=t,c794=t,c795=t,c796=t,c797=t,c798=t," - "c799=t,c800=t,c801=t,c802=t,c803=t,c804=t,c805=t,c806=t,c807=t,c808=t,c809=t,c810=t,c811=t,c812=t,c813=t," - "c814=t,c815=t,c816=t,c817=t,c818=t,c819=t,c820=t,c821=t,c822=t,c823=t,c824=t,c825=t,c826=t,c827=t,c828=t,c829=t," - "c830=t,c831=t,c832=t,c833=t,c834=t,c835=t,c836=t,c837=t,c838=t,c839=t,c840=t,c841=t,c842=t,c843=t,c844=t,c845=t," - "c846=t,c847=t,c848=t,c849=t,c850=t,c851=t,c852=t,c853=t,c854=t,c855=t,c856=t,c857=t,c858=t,c859=t,c860=t,c861=t," - "c862=t," - "c863=t,c864=t,c865=t,c866=t,c867=t,c868=t,c869=t,c870=t,c871=t,c872=t,c873=t,c874=t,c875=t,c876=t,c877=t,c878=t," - "c879=t,c880=t,c881=t,c882=t,c883=t,c884=t,c885=t,c886=t,c887=t,c888=t,c889=t,c890=t,c891=t,c892=t,c893=t,c894=t," - "c895=t,c896=t,c897=t,c898=t,c899=t,c900=t,c901=t,c902=t,c903=t,c904=t,c905=t,c906=t,c907=t,c908=t,c909=t,c910=t," - "c911=t,c912=t,c913=t,c914=t,c915=t,c916=t,c917=t,c918=t,c919=t,c920=t,c921=t,c922=t,c923=t,c924=t,c925=t,c926=t," - "c927=t,c928=t,c929=t,c930=t,c931=t,c932=t,c933=t,c934=t,c935=t,c936=t,c937=t,c938=t,c939=t,c940=t,c941=t,c942=t," - "c943=t,c944=t,c945=t,c946=t,c947=t,c948=t,c949=t,c950=t,c951=t,c952=t,c953=t,c954=t,c955=t,c956=t,c957=t,c958=t," - "c959=t,c960=t,c961=t,c962=t,c963=t,c964=t,c965=t,c966=t,c967=t,c968=t,c969=t,c970=t,c971=t,c972=t,c973=t,c974=t," - "c975=t,c976=t,c977=t,c978=t,c979=t,c980=t,c981=t,c982=t,c983=t,c984=t,c985=t,c986=t,c987=t,c988=t,c989=t,c990=t," - "c991=t,c992=t,c993=t,c994=t,c995=t,c996=t,c997=t,c998=t,c999=t,c1000=t,c1001=t,c1002=t,c1003=t,c1004=t,c1005=t," - "c1006=t,c1007=t,c1008=t,c1009=t,c1010=t,c1011=t,c1012=t,c1013=t,c1014=t,c1015=t,c1016=t,c1017=t,c1018=t,c1019=t," - "c1020=t,c1021=t,c1022=t,c1023=t,c1024=t,c1025=t,c1026=t," - "c1027=t,c1028=t,c1029=t,c1030=t,c1031=t,c1032=t,c1033=t,c1034=t,c1035=t,c1036=t,c1037=t,c1038=t,c1039=t,c1040=t," - "c1041=t,c1042=t,c1043=t,c1044=t,c1045=t,c1046=t,c1047=t,c1048=t,c1049=t,c1050=t,c1051=t,c1052=t,c1053=t,c1054=t," - "c1055=t,c1056=t,c1057=t,c1058=t,c1059=t,c1060=t,c1061=t,c1062=t,c1063=t,c1064=t,c1065=t,c1066=t,c1067=t,c1068=t," - "c1069=t,c1070=t,c1071=t,c1072=t,c1073=t,c1074=t,c1075=t,c1076=t,c1077=t,c1078=t,c1079=t,c1080=t,c1081=t,c1082=t," - "c1083=t,c1084=t,c1085=t,c1086=t,c1087=t,c1088=t,c1089=t,c1090=t,c1091=t,c1092=t,c1093=t,c1094=t,c1095=t,c1096=t," - "c1097=t,c1098=t,c1099=t,c1100=t,c1101=t,c1102=t,c1103=t,c1104=t,c1105=t,c1106=t,c1107=t,c1108=t,c1109=t,c1110=t," - "c1111=t,c1112=t,c1113=t,c1114=t,c1115=t,c1116=t,c1117=t,c1118=t,c1119=t,c1120=t,c1121=t,c1122=t,c1123=t,c1124=t," - "c1125=t,c1126=t,c1127=t,c1128=t,c1129=t,c1130=t,c1131=t,c1132=t,c1133=t,c1134=t,c1135=t,c1136=t,c1137=t,c1138=t," - "c1139=t,c1140=t,c1141=t,c1142=t,c1143=t,c1144=t,c1145=t,c1146=t,c1147=t,c1148=t,c1149=t,c1150=t,c1151=t,c1152=t," - "c1153=t,c1154=t,c1155=t,c1156=t,c1157=t,c1158=t,c1159=t,c1160=t,c1161=t,c1162=t,c1163=t,c1164=t,c1165=t,c1166=t," - "c1167=t,c1168=t,c1169=t,c1170=t,c1171=t,c1172=t,c1173=t," - "c1174=t,c1175=t,c1176=t,c1177=t,c1178=t,c1179=t,c1180=t,c1181=t,c1182=t,c1183=t,c1184=t,c1185=t,c1186=t,c1187=t," - "c1188=t,c1189=t,c1190=t,c1191=t,c1192=t,c1193=t,c1194=t,c1195=t,c1196=t,c1197=t,c1198=t,c1199=t,c1200=t,c1201=t," - "c1202=t,c1203=t,c1204=t,c1205=t,c1206=t,c1207=t,c1208=t,c1209=t,c1210=t,c1211=t,c1212=t,c1213=t,c1214=t,c1215=t," - "c1216=t,c1217=t,c1218=t,c1219=t,c1220=t,c1221=t,c1222=t,c1223=t,c1224=t,c1225=t,c1226=t,c1227=t,c1228=t,c1229=t," - "c1230=t,c1231=t,c1232=t,c1233=t,c1234=t,c1235=t,c1236=t,c1237=t,c1238=t,c1239=t,c1240=t,c1241=t,c1242=t,c1243=t," - "c1244=t,c1245=t,c1246=t,c1247=t,c1248=t,c1249=t,c1250=t,c1251=t,c1252=t,c1253=t,c1254=t,c1255=t,c1256=t,c1257=t," - "c1258=t,c1259=t,c1260=t,c1261=t,c1262=t,c1263=t,c1264=t,c1265=t,c1266=t,c1267=t,c1268=t,c1269=t,c1270=t,c1271=t," - "c1272=t,c1273=t,c1274=t,c1275=t,c1276=t,c1277=t,c1278=t,c1279=t,c1280=t,c1281=t,c1282=t,c1283=t,c1284=t,c1285=t," - "c1286=t,c1287=t,c1288=t,c1289=t,c1290=t,c1291=t,c1292=t,c1293=t,c1294=t,c1295=t,c1296=t,c1297=t,c1298=t,c1299=t," - "c1300=t,c1301=t,c1302=t,c1303=t,c1304=t,c1305=t,c1306=t,c1307=t,c1308=t,c1309=t,c1310=t,c1311=t,c1312=t,c1313=t," - "c1314=t,c1315=t,c1316=t,c1317=t,c1318=t,c1319=t,c1320=t," - "c1321=t,c1322=t,c1323=t,c1324=t,c1325=t,c1326=t,c1327=t,c1328=t,c1329=t,c1330=t,c1331=t,c1332=t,c1333=t,c1334=t," - "c1335=t,c1336=t,c1337=t,c1338=t,c1339=t,c1340=t,c1341=t,c1342=t,c1343=t,c1344=t,c1345=t,c1346=t,c1347=t," - "c1348=t,c1349=t,c1350=t,c1351=t,c1352=t,c1353=t,c1354=t,c1355=t,c1356=t,c1357=t,c1358=t,c1359=t,c1360=t,c1361=t," - "c1362=t,c1363=t,c1364=t,c1365=t,c1366=t,c1367=t,c1368=t,c1369=t,c1370=t,c1371=t,c1372=t,c1373=t,c1374=t,c1375=t," - "c1376=t,c1377=t,c1378=t,c1379=t,c1380=t,c1381=t,c1382=t,c1383=t,c1384=t,c1385=t,c1386=t,c1387=t,c1388=t,c1389=t," - "c1390=t,c1391=t,c1392=t,c1393=t,c1394=t,c1395=t,c1396=t,c1397=t,c1398=t,c1399=t,c1400=t,c1401=t,c1402=t,c1403=t," - "c1404=t,c1405=t,c1406=t,c1407=t,c1408=t,c1409=t,c1410=t,c1411=t,c1412=t,c1413=t,c1414=t,c1415=t,c1416=t,c1417=t," - "c1418=t,c1419=t,c1420=t,c1421=t,c1422=t,c1423=t,c1424=t,c1425=t,c1426=t,c1427=t,c1428=t,c1429=t,c1430=t,c1431=t," - "c1432=t,c1433=t,c1434=t,c1435=t,c1436=t,c1437=t,c1438=t,c1439=t,c1440=t,c1441=t,c1442=t,c1443=t,c1444=t,c1445=t," - "c1446=t,c1447=t,c1448=t,c1449=t,c1450=t,c1451=t,c1452=t,c1453=t,c1454=t,c1455=t,c1456=t,c1457=t,c1458=t,c1459=t," - "c1460=t,c1461=t,c1462=t,c1463=t,c1464=t,c1465=t,c1466=t,c1467=t,c1468=t,c1469=t,c1470=t,c1471=t,c1472=t,c1473=t," - "c1474=t,c1475=t,c1476=t,c1477=t,c1478=t,c1479=t,c1480=t,c1481=t,c1482=t,c1483=t,c1484=t,c1485=t,c1486=t,c1487=t," - "c1488=t,c1489=t,c1490=t,c1491=t,c1492=t,c1493=t,c1494=t," - "c1495=t,c1496=t,c1497=t,c1498=t,c1499=t,c1500=t,c1501=t,c1502=t,c1503=t,c1504=t,c1505=t,c1506=t,c1507=t,c1508=t," - "c1509=t,c1510=t,c1511=t,c1512=t,c1513=t,c1514=t,c1515=t,c1516=t,c1517=t,c1518=t,c1519=t,c1520=t,c1521=t,c1522=t," - "c1523=t,c1524=t,c1525=t,c1526=t,c1527=t,c1528=t,c1529=t,c1530=t,c1531=t,c1532=t,c1533=t,c1534=t,c1535=t,c1536=t," - "c1537=t,c1538=t,c1539=t,c1540=t,c1541=t,c1542=t,c1543=t,c1544=t,c1545=t,c1546=t,c1547=t,c1548=t,c1549=t,c1550=t," - "c1551=t,c1552=t,c1553=t,c1554=t,c1555=t,c1556=t,c1557=t,c1558=t,c1559=t,c1560=t,c1561=t,c1562=t,c1563=t,c1564=t," - "c1565=t,c1566=t,c1567=t,c1568=t,c1569=t,c1570=t,c1571=t,c1572=t,c1573=t,c1574=t,c1575=t,c1576=t,c1577=t,c1578=t," - "c1579=t,c1580=t,c1581=t,c1582=t,c1583=t,c1584=t,c1585=t,c1586=t,c1587=t,c1588=t,c1589=t,c1590=t,c1591=t,c1592=t," - "c1593=t,c1594=t,c1595=t,c1596=t,c1597=t,c1598=t,c1599=t,c1600=t,c1601=t,c1602=t,c1603=t,c1604=t,c1605=t,c1606=t," - "c1607=t,c1608=t,c1609=t,c1610=t,c1611=t,c1612=t,c1613=t,c1614=t,c1615=t,c1616=t,c1617=t,c1618=t,c1619=t,c1620=t," - "c1621=t,c1622=t,c1623=t,c1624=t,c1625=t,c1626=t,c1627=t,c1628=t,c1629=t,c1630=t,c1631=t,c1632=t,c1633=t,c1634=t," - "c1635=t,c1636=t,c1637=t,c1638=t,c1639=t,c1640=t,c1641=t," - "c1642=t,c1643=t,c1644=t,c1645=t,c1646=t,c1647=t,c1648=t,c1649=t,c1650=t,c1651=t,c1652=t,c1653=t,c1654=t,c1655=t," - "c1656=t,c1657=t,c1658=t,c1659=t,c1660=t,c1661=t,c1662=t,c1663=t,c1664=t,c1665=t,c1666=t,c1667=t,c1668=t,c1669=t," - "c1670=t,c1671=t,c1672=t,c1673=t,c1674=t,c1675=t,c1676=t,c1677=t,c1678=t,c1679=t,c1680=t,c1681=t,c1682=t,c1683=t," - "c1684=t,c1685=t,c1686=t,c1687=t,c1688=t,c1689=t,c1690=t,c1691=t,c1692=t,c1693=t,c1694=t,c1695=t,c1696=t,c1697=t," - "c1698=t,c1699=t,c1700=t,c1701=t,c1702=t,c1703=t,c1704=t,c1705=t,c1706=t,c1707=t,c1708=t,c1709=t,c1710=t,c1711=t," - "c1712=t,c1713=t,c1714=t,c1715=t,c1716=t,c1717=t,c1718=t,c1719=t,c1720=t,c1721=t,c1722=t,c1723=t,c1724=t,c1725=t," - "c1726=t,c1727=t,c1728=t,c1729=t,c1730=t,c1731=t,c1732=t,c1733=t,c1734=t,c1735=t,c1736=t,c1737=t,c1738=t,c1739=t," - "c1740=t,c1741=t,c1742=t,c1743=t,c1744=t,c1745=t,c1746=t,c1747=t,c1748=t,c1749=t,c1750=t,c1751=t,c1752=t,c1753=t," - "c1754=t,c1755=t,c1756=t,c1757=t,c1758=t,c1759=t,c1760=t,c1761=t,c1762=t,c1763=t,c1764=t,c1765=t,c1766=t,c1767=t," - "c1768=t,c1769=t,c1770=t,c1771=t,c1772=t,c1773=t,c1774=t,c1775=t,c1776=t,c1777=t,c1778=t,c1779=t,c1780=t,c1781=t," - "c1782=t,c1783=t,c1784=t,c1785=t,c1786=t,c1787=t,c1788=t," - "c1789=t,c1790=t,c1791=t,c1792=t,c1793=t,c1794=t,c1795=t,c1796=t,c1797=t,c1798=t,c1799=t,c1800=t,c1801=t,c1802=t," - "c1803=t,c1804=t,c1805=t,c1806=t,c1807=t,c1808=t,c1809=t,c1810=t,c1811=t,c1812=t,c1813=t,c1814=t,c1815=t," - "c1816=t,c1817=t,c1818=t,c1819=t,c1820=t,c1821=t,c1822=t,c1823=t,c1824=t,c1825=t,c1826=t,c1827=t,c1828=t,c1829=t," - "c1830=t,c1831=t,c1832=t,c1833=t,c1834=t,c1835=t,c1836=t,c1837=t,c1838=t,c1839=t,c1840=t,c1841=t,c1842=t,c1843=t," - "c1844=t,c1845=t,c1846=t,c1847=t,c1848=t,c1849=t,c1850=t,c1851=t,c1852=t,c1853=t,c1854=t,c1855=t,c1856=t,c1857=t," - "c1858=t,c1859=t,c1860=t,c1861=t,c1862=t,c1863=t,c1864=t,c1865=t,c1866=t,c1867=t,c1868=t,c1869=t,c1870=t,c1871=t," - "c1872=t,c1873=t,c1874=t,c1875=t,c1876=t,c1877=t,c1878=t,c1879=t,c1880=t,c1881=t,c1882=t,c1883=t,c1884=t,c1885=t," - "c1886=t,c1887=t,c1888=t,c1889=t,c1890=t,c1891=t,c1892=t,c1893=t,c1894=t,c1895=t,c1896=t,c1897=t,c1898=t,c1899=t," - "c1900=t,c1901=t,c1902=t,c1903=t,c1904=t,c1905=t,c1906=t,c1907=t,c1908=t,c1909=t,c1910=t,c1911=t,c1912=t,c1913=t," - "c1914=t,c1915=t,c1916=t,c1917=t,c1918=t,c1919=t,c1920=t,c1921=t,c1922=t,c1923=t,c1924=t,c1925=t,c1926=t,c1927=t," - "c1928=t,c1929=t,c1930=t,c1931=t,c1932=t,c1933=t,c1934=t,c1935=t,c1936=t,c1937=t,c1938=t,c1939=t,c1940=t,c1941=t," - "c1942=t,c1943=t,c1944=t,c1945=t,c1946=t,c1947=t,c1948=t,c1949=t,c1950=t,c1951=t,c1952=t,c1953=t,c1954=t,c1955=t," - "c1956=t,c1957=t,c1958=t,c1959=t,c1960=t,c1961=t,c1962=t," - "c1963=t,c1964=t,c1965=t,c1966=t,c1967=t,c1968=t,c1969=t,c1970=t,c1971=t,c1972=t,c1973=t,c1974=t,c1975=t,c1976=t," - "c1977=t,c1978=t,c1979=t,c1980=t,c1981=t,c1982=t,c1983=t,c1984=t,c1985=t,c1986=t,c1987=t,c1988=t,c1989=t,c1990=t," - "c1991=t,c1992=t,c1993=t,c1994=t,c1995=t,c1996=t,c1997=t,c1998=t,c1999=t,c2000=t,c2001=t,c2002=t,c2003=t,c2004=t," - "c2005=t,c2006=t,c2007=t,c2008=t,c2009=t,c2010=t,c2011=t,c2012=t,c2013=t,c2014=t,c2015=t,c2016=t,c2017=t,c2018=t," - "c2019=t,c2020=t,c2021=t,c2022=t,c2023=t,c2024=t,c2025=t,c2026=t,c2027=t,c2028=t,c2029=t,c2030=t,c2031=t,c2032=t," - "c2033=t,c2034=t,c2035=t,c2036=t,c2037=t,c2038=t,c2039=t,c2040=t,c2041=t,c2042=t,c2043=t,c2044=t,c2045=t,c2046=t," - "c2047=t,c2048=t,c2049=t,c2050=t,c2051=t,c2052=t,c2053=t,c2054=t,c2055=t,c2056=t,c2057=t,c2058=t,c2059=t,c2060=t," - "c2061=t,c2062=t,c2063=t,c2064=t,c2065=t,c2066=t,c2067=t,c2068=t,c2069=t,c2070=t,c2071=t,c2072=t,c2073=t,c2074=t," - "c2075=t,c2076=t,c2077=t,c2078=t,c2079=t,c2080=t,c2081=t,c2082=t,c2083=t,c2084=t,c2085=t,c2086=t,c2087=t,c2088=t," - "c2089=t,c2090=t,c2091=t,c2092=t,c2093=t,c2094=t,c2095=t,c2096=t,c2097=t,c2098=t,c2099=t,c2100=t,c2101=t,c2102=t," - "c2103=t,c2104=t,c2105=t,c2106=t,c2107=t,c2108=t,c2109=t," - "c2110=t,c2111=t,c2112=t,c2113=t,c2114=t,c2115=t,c2116=t,c2117=t,c2118=t,c2119=t,c2120=t,c2121=t,c2122=t,c2123=t," - "c2124=t,c2125=t,c2126=t,c2127=t,c2128=t,c2129=t,c2130=t,c2131=t,c2132=t,c2133=t,c2134=t,c2135=t,c2136=t,c2137=t," - "c2138=t,c2139=t,c2140=t,c2141=t,c2142=t,c2143=t,c2144=t,c2145=t,c2146=t,c2147=t,c2148=t,c2149=t,c2150=t,c2151=t," - "c2152=t,c2153=t,c2154=t,c2155=t,c2156=t,c2157=t,c2158=t,c2159=t,c2160=t,c2161=t,c2162=t,c2163=t,c2164=t,c2165=t," - "c2166=t,c2167=t,c2168=t,c2169=t,c2170=t,c2171=t,c2172=t,c2173=t,c2174=t,c2175=t,c2176=t,c2177=t,c2178=t,c2179=t," - "c2180=t,c2181=t,c2182=t,c2183=t,c2184=t,c2185=t,c2186=t,c2187=t,c2188=t,c2189=t,c2190=t,c2191=t,c2192=t,c2193=t," - "c2194=t,c2195=t,c2196=t,c2197=t,c2198=t,c2199=t,c2200=t,c2201=t,c2202=t,c2203=t,c2204=t,c2205=t,c2206=t,c2207=t," - "c2208=t,c2209=t,c2210=t,c2211=t,c2212=t,c2213=t,c2214=t,c2215=t,c2216=t,c2217=t,c2218=t,c2219=t,c2220=t,c2221=t," - "c2222=t,c2223=t,c2224=t,c2225=t,c2226=t,c2227=t,c2228=t,c2229=t,c2230=t,c2231=t,c2232=t,c2233=t,c2234=t,c2235=t," - "c2236=t,c2237=t,c2238=t,c2239=t,c2240=t,c2241=t,c2242=t,c2243=t,c2244=t,c2245=t,c2246=t,c2247=t,c2248=t,c2249=t," - "c2250=t,c2251=t,c2252=t,c2253=t,c2254=t,c2255=t,c2256=t," - "c2257=t,c2258=t,c2259=t,c2260=t,c2261=t,c2262=t,c2263=t,c2264=t,c2265=t,c2266=t,c2267=t,c2268=t,c2269=t,c2270=t," - "c2271=t,c2272=t,c2273=t,c2274=t,c2275=t,c2276=t,c2277=t,c2278=t,c2279=t,c2280=t,c2281=t,c2282=t,c2283=t," - "c2284=t,c2285=t,c2286=t,c2287=t,c2288=t,c2289=t,c2290=t,c2291=t,c2292=t,c2293=t,c2294=t,c2295=t,c2296=t,c2297=t," - "c2298=t,c2299=t,c2300=t,c2301=t,c2302=t,c2303=t,c2304=t,c2305=t,c2306=t,c2307=t,c2308=t,c2309=t,c2310=t,c2311=t," - "c2312=t,c2313=t,c2314=t,c2315=t,c2316=t,c2317=t,c2318=t,c2319=t,c2320=t,c2321=t,c2322=t,c2323=t,c2324=t,c2325=t," - "c2326=t,c2327=t,c2328=t,c2329=t,c2330=t,c2331=t,c2332=t,c2333=t,c2334=t,c2335=t,c2336=t,c2337=t,c2338=t,c2339=t," - "c2340=t,c2341=t,c2342=t,c2343=t,c2344=t,c2345=t,c2346=t,c2347=t,c2348=t,c2349=t,c2350=t,c2351=t,c2352=t,c2353=t," - "c2354=t,c2355=t,c2356=t,c2357=t,c2358=t,c2359=t,c2360=t,c2361=t,c2362=t,c2363=t,c2364=t,c2365=t,c2366=t,c2367=t," - "c2368=t,c2369=t,c2370=t,c2371=t,c2372=t,c2373=t,c2374=t,c2375=t,c2376=t,c2377=t,c2378=t,c2379=t,c2380=t,c2381=t," - "c2382=t,c2383=t,c2384=t,c2385=t,c2386=t,c2387=t,c2388=t,c2389=t,c2390=t,c2391=t,c2392=t,c2393=t,c2394=t,c2395=t," - "c2396=t,c2397=t,c2398=t,c2399=t,c2400=t,c2401=t,c2402=t,c2403=t,c2404=t,c2405=t,c2406=t,c2407=t,c2408=t,c2409=t," - "c2410=t,c2411=t,c2412=t,c2413=t,c2414=t,c2415=t,c2416=t,c2417=t,c2418=t,c2419=t,c2420=t,c2421=t,c2422=t,c2423=t," - "c2424=t,c2425=t,c2426=t,c2427=t,c2428=t,c2429=t,c2430=t," - "c2431=t,c2432=t,c2433=t,c2434=t,c2435=t,c2436=t,c2437=t,c2438=t,c2439=t,c2440=t,c2441=t,c2442=t,c2443=t,c2444=t," - "c2445=t,c2446=t,c2447=t,c2448=t,c2449=t,c2450=t,c2451=t,c2452=t,c2453=t,c2454=t,c2455=t,c2456=t,c2457=t,c2458=t," - "c2459=t,c2460=t,c2461=t,c2462=t,c2463=t,c2464=t,c2465=t,c2466=t,c2467=t,c2468=t,c2469=t,c2470=t,c2471=t,c2472=t," - "c2473=t,c2474=t,c2475=t,c2476=t,c2477=t,c2478=t,c2479=t,c2480=t,c2481=t,c2482=t,c2483=t,c2484=t,c2485=t,c2486=t," - "c2487=t,c2488=t,c2489=t,c2490=t,c2491=t,c2492=t,c2493=t,c2494=t,c2495=t,c2496=t,c2497=t,c2498=t,c2499=t,c2500=t," - "c2501=t,c2502=t,c2503=t,c2504=t,c2505=t,c2506=t,c2507=t,c2508=t,c2509=t,c2510=t,c2511=t,c2512=t,c2513=t,c2514=t," - "c2515=t,c2516=t,c2517=t,c2518=t,c2519=t,c2520=t,c2521=t,c2522=t,c2523=t,c2524=t,c2525=t,c2526=t,c2527=t,c2528=t," - "c2529=t,c2530=t,c2531=t,c2532=t,c2533=t,c2534=t,c2535=t,c2536=t,c2537=t,c2538=t,c2539=t,c2540=t,c2541=t,c2542=t," - "c2543=t,c2544=t,c2545=t,c2546=t,c2547=t,c2548=t,c2549=t,c2550=t,c2551=t,c2552=t,c2553=t,c2554=t,c2555=t,c2556=t," - "c2557=t,c2558=t,c2559=t,c2560=t,c2561=t,c2562=t,c2563=t,c2564=t,c2565=t,c2566=t,c2567=t,c2568=t,c2569=t,c2570=t," - "c2571=t,c2572=t,c2573=t,c2574=t,c2575=t,c2576=t,c2577=t," - "c2578=t,c2579=t,c2580=t,c2581=t,c2582=t,c2583=t,c2584=t,c2585=t,c2586=t,c2587=t,c2588=t,c2589=t,c2590=t,c2591=t," - "c2592=t,c2593=t,c2594=t,c2595=t,c2596=t,c2597=t,c2598=t,c2599=t,c2600=t,c2601=t,c2602=t,c2603=t,c2604=t,c2605=t," - "c2606=t,c2607=t,c2608=t,c2609=t,c2610=t,c2611=t,c2612=t,c2613=t,c2614=t,c2615=t,c2616=t,c2617=t,c2618=t,c2619=t," - "c2620=t,c2621=t,c2622=t,c2623=t,c2624=t,c2625=t,c2626=t,c2627=t,c2628=t,c2629=t,c2630=t,c2631=t,c2632=t,c2633=t," - "c2634=t,c2635=t,c2636=t,c2637=t,c2638=t,c2639=t,c2640=t,c2641=t,c2642=t,c2643=t,c2644=t,c2645=t,c2646=t,c2647=t," - "c2648=t,c2649=t,c2650=t,c2651=t,c2652=t,c2653=t,c2654=t,c2655=t,c2656=t,c2657=t,c2658=t,c2659=t,c2660=t,c2661=t," - "c2662=t,c2663=t,c2664=t,c2665=t,c2666=t,c2667=t,c2668=t,c2669=t,c2670=t,c2671=t,c2672=t,c2673=t,c2674=t,c2675=t," - "c2676=t,c2677=t,c2678=t,c2679=t,c2680=t,c2681=t,c2682=t,c2683=t,c2684=t,c2685=t,c2686=t,c2687=t,c2688=t,c2689=t," - "c2690=t,c2691=t,c2692=t,c2693=t,c2694=t,c2695=t,c2696=t,c2697=t,c2698=t,c2699=t,c2700=t,c2701=t,c2702=t,c2703=t," - "c2704=t,c2705=t,c2706=t,c2707=t,c2708=t,c2709=t,c2710=t,c2711=t,c2712=t,c2713=t,c2714=t,c2715=t,c2716=t,c2717=t," - "c2718=t,c2719=t,c2720=t,c2721=t,c2722=t,c2723=t,c2724=t," - "c2725=t,c2726=t,c2727=t,c2728=t,c2729=t,c2730=t,c2731=t,c2732=t,c2733=t,c2734=t,c2735=t,c2736=t,c2737=t,c2738=t," - "c2739=t,c2740=t,c2741=t,c2742=t,c2743=t,c2744=t,c2745=t,c2746=t,c2747=t,c2748=t,c2749=t,c2750=t,c2751=t,c2752=t," - "c2753=t,c2754=t,c2755=t,c2756=t,c2757=t,c2758=t,c2759=t,c2760=t,c2761=t,c2762=t,c2763=t,c2764=t,c2765=t,c2766=t," - "c2767=t,c2768=t,c2769=t,c2770=t,c2771=t,c2772=t,c2773=t,c2774=t,c2775=t,c2776=t,c2777=t,c2778=t,c2779=t,c2780=t," - "c2781=t,c2782=t,c2783=t,c2784=t,c2785=t,c2786=t,c2787=t,c2788=t,c2789=t,c2790=t,c2791=t,c2792=t,c2793=t,c2794=t," - "c2795=t,c2796=t,c2797=t,c2798=t,c2799=t,c2800=t,c2801=t,c2802=t,c2803=t,c2804=t,c2805=t,c2806=t,c2807=t,c2808=t," - "c2809=t,c2810=t,c2811=t,c2812=t,c2813=t,c2814=t,c2815=t,c2816=t,c2817=t,c2818=t,c2819=t,c2820=t,c2821=t,c2822=t," - "c2823=t,c2824=t,c2825=t,c2826=t,c2827=t,c2828=t,c2829=t,c2830=t,c2831=t,c2832=t,c2833=t,c2834=t,c2835=t,c2836=t," - "c2837=t,c2838=t,c2839=t,c2840=t,c2841=t,c2842=t,c2843=t,c2844=t,c2845=t,c2846=t,c2847=t,c2848=t,c2849=t,c2850=t," - "c2851=t,c2852=t,c2853=t,c2854=t,c2855=t,c2856=t,c2857=t,c2858=t,c2859=t,c2860=t,c2861=t,c2862=t,c2863=t,c2864=t," - "c2865=t,c2866=t,c2867=t,c2868=t,c2869=t,c2870=t,c2871=t," - "c2872=t,c2873=t,c2874=t,c2875=t,c2876=t,c2877=t,c2878=t,c2879=t,c2880=t,c2881=t,c2882=t,c2883=t,c2884=t,c2885=t," - "c2886=t,c2887=t,c2888=t,c2889=t,c2890=t,c2891=t,c2892=t,c2893=t,c2894=t,c2895=t,c2896=t,c2897=t,c2898=t,c2899=t," - "c2900=t,c2901=t,c2902=t,c2903=t,c2904=t,c2905=t,c2906=t,c2907=t,c2908=t,c2909=t,c2910=t,c2911=t,c2912=t,c2913=t," - "c2914=t,c2915=t,c2916=t,c2917=t,c2918=t,c2919=t,c2920=t,c2921=t,c2922=t,c2923=t,c2924=t,c2925=t,c2926=t,c2927=t," - "c2928=t,c2929=t,c2930=t,c2931=t,c2932=t,c2933=t,c2934=t,c2935=t,c2936=t,c2937=t,c2938=t,c2939=t,c2940=t,c2941=t," - "c2942=t,c2943=t,c2944=t,c2945=t,c2946=t,c2947=t,c2948=t,c2949=t,c2950=t,c2951=t,c2952=t,c2953=t,c2954=t,c2955=t," - "c2956=t,c2957=t,c2958=t,c2959=t,c2960=t,c2961=t,c2962=t,c2963=t,c2964=t,c2965=t,c2966=t,c2967=t,c2968=t,c2969=t," - "c2970=t,c2971=t,c2972=t,c2973=t,c2974=t,c2975=t,c2976=t,c2977=t,c2978=t,c2979=t,c2980=t,c2981=t,c2982=t,c2983=t," - "c2984=t,c2985=t,c2986=t,c2987=t,c2988=t,c2989=t,c2990=t,c2991=t,c2992=t,c2993=t,c2994=t,c2995=t,c2996=t,c2997=t," - "c2998=t,c2999=t,c3000=t,c3001=t,c3002=t,c3003=t,c3004=t,c3005=t,c3006=t,c3007=t,c3008=t,c3009=t,c3010=t,c3011=t," - "c3012=t,c3013=t,c3014=t,c3015=t,c3016=t,c3017=t,c3018=t," - "c3019=t,c3020=t,c3021=t,c3022=t,c3023=t,c3024=t,c3025=t,c3026=t,c3027=t,c3028=t,c3029=t,c3030=t,c3031=t,c3032=t," - "c3033=t,c3034=t,c3035=t,c3036=t,c3037=t,c3038=t,c3039=t,c3040=t,c3041=t,c3042=t,c3043=t,c3044=t,c3045=t,c3046=t," - "c3047=t,c3048=t,c3049=t,c3050=t,c3051=t,c3052=t,c3053=t,c3054=t,c3055=t,c3056=t,c3057=t,c3058=t,c3059=t,c3060=t," - "c3061=t,c3062=t,c3063=t,c3064=t,c3065=t,c3066=t,c3067=t,c3068=t,c3069=t,c3070=t,c3071=t,c3072=t,c3073=t,c3074=t," - "c3075=t,c3076=t,c3077=t,c3078=t,c3079=t,c3080=t,c3081=t,c3082=t,c3083=t,c3084=t,c3085=t,c3086=t,c3087=t,c3088=t," - "c3089=t,c3090=t,c3091=t,c3092=t,c3093=t,c3094=t,c3095=t,c3096=t,c3097=t,c3098=t,c3099=t,c3100=t,c3101=t,c3102=t," - "c3103=t,c3104=t,c3105=t,c3106=t,c3107=t,c3108=t,c3109=t,c3110=t,c3111=t,c3112=t,c3113=t,c3114=t,c3115=t,c3116=t," - "c3117=t,c3118=t,c3119=t,c3120=t,c3121=t,c3122=t,c3123=t,c3124=t,c3125=t,c3126=t,c3127=t,c3128=t,c3129=t,c3130=t," - "c3131=t,c3132=t,c3133=t,c3134=t,c3135=t,c3136=t,c3137=t,c3138=t,c3139=t,c3140=t,c3141=t,c3142=t,c3143=t,c3144=t," - "c3145=t,c3146=t,c3147=t,c3148=t,c3149=t,c3150=t,c3151=t,c3152=t,c3153=t,c3154=t,c3155=t,c3156=t,c3157=t,c3158=t," - "c3159=t,c3160=t,c3161=t,c3162=t,c3163=t,c3164=t,c3165=t," - "c3166=t,c3167=t,c3168=t,c3169=t,c3170=t,c3171=t,c3172=t,c3173=t,c3174=t,c3175=t,c3176=t,c3177=t,c3178=t,c3179=t," - "c3180=t,c3181=t,c3182=t,c3183=t,c3184=t,c3185=t,c3186=t,c3187=t,c3188=t,c3189=t,c3190=t,c3191=t,c3192=t,c3193=t," - "c3194=t,c3195=t,c3196=t,c3197=t,c3198=t,c3199=t,c3200=t,c3201=t,c3202=t,c3203=t,c3204=t,c3205=t,c3206=t,c3207=t," - "c3208=t,c3209=t,c3210=t,c3211=t,c3212=t,c3213=t,c3214=t,c3215=t,c3216=t,c3217=t,c3218=t,c3219=t,c3220=t,c3221=t," - "c3222=t,c3223=t,c3224=t,c3225=t,c3226=t,c3227=t,c3228=t,c3229=t,c3230=t,c3231=t,c3232=t,c3233=t,c3234=t,c3235=t," - "c3236=t,c3237=t,c3238=t,c3239=t,c3240=t,c3241=t,c3242=t,c3243=t,c3244=t,c3245=t,c3246=t,c3247=t,c3248=t,c3249=t," - "c3250=t,c3251=t,c3252=t,c3253=t,c3254=t,c3255=t,c3256=t,c3257=t,c3258=t,c3259=t,c3260=t,c3261=t,c3262=t,c3263=t," - "c3264=t,c3265=t,c3266=t,c3267=t,c3268=t,c3269=t,c3270=t,c3271=t,c3272=t,c3273=t,c3274=t,c3275=t,c3276=t,c3277=t," - "c3278=t,c3279=t,c3280=t,c3281=t,c3282=t,c3283=t,c3284=t,c3285=t,c3286=t,c3287=t,c3288=t,c3289=t,c3290=t,c3291=t," - "c3292=t,c3293=t,c3294=t,c3295=t,c3296=t,c3297=t,c3298=t,c3299=t,c3300=t,c3301=t,c3302=t,c3303=t,c3304=t,c3305=t," - "c3306=t,c3307=t,c3308=t,c3309=t,c3310=t,c3311=t,c3312=t," - "c3313=t,c3314=t,c3315=t,c3316=t,c3317=t,c3318=t,c3319=t,c3320=t,c3321=t,c3322=t,c3323=t,c3324=t,c3325=t,c3326=t," - "c3327=t,c3328=t,c3329=t,c3330=t,c3331=t,c3332=t,c3333=t,c3334=t,c3335=t,c3336=t,c3337=t,c3338=t,c3339=t,c3340=t," - "c3341=t,c3342=t,c3343=t,c3344=t,c3345=t,c3346=t,c3347=t,c3348=t,c3349=t,c3350=t,c3351=t,c3352=t,c3353=t,c3354=t," - "c3355=t,c3356=t,c3357=t,c3358=t,c3359=t,c3360=t,c3361=t,c3362=t,c3363=t,c3364=t,c3365=t,c3366=t,c3367=t,c3368=t," - "c3369=t,c3370=t,c3371=t,c3372=t,c3373=t,c3374=t,c3375=t,c3376=t,c3377=t,c3378=t,c3379=t,c3380=t,c3381=t,c3382=t," - "c3383=t,c3384=t,c3385=t,c3386=t,c3387=t,c3388=t,c3389=t,c3390=t,c3391=t,c3392=t,c3393=t,c3394=t,c3395=t,c3396=t," - "c3397=t,c3398=t,c3399=t,c3400=t,c3401=t,c3402=t,c3403=t,c3404=t,c3405=t,c3406=t,c3407=t,c3408=t,c3409=t,c3410=t," - "c3411=t,c3412=t,c3413=t,c3414=t,c3415=t,c3416=t,c3417=t,c3418=t,c3419=t,c3420=t,c3421=t,c3422=t,c3423=t,c3424=t," - "c3425=t,c3426=t,c3427=t,c3428=t,c3429=t,c3430=t,c3431=t,c3432=t,c3433=t,c3434=t,c3435=t,c3436=t,c3437=t,c3438=t," - "c3439=t,c3440=t,c3441=t,c3442=t,c3443=t,c3444=t,c3445=t,c3446=t,c3447=t,c3448=t,c3449=t,c3450=t,c3451=t,c3452=t," - "c3453=t,c3454=t,c3455=t,c3456=t,c3457=t,c3458=t,c3459=t," - "c3460=t,c3461=t,c3462=t,c3463=t,c3464=t,c3465=t,c3466=t,c3467=t,c3468=t,c3469=t,c3470=t,c3471=t,c3472=t,c3473=t," - "c3474=t,c3475=t,c3476=t,c3477=t,c3478=t,c3479=t,c3480=t,c3481=t,c3482=t,c3483=t,c3484=t,c3485=t,c3486=t,c3487=t," - "c3488=t,c3489=t,c3490=t,c3491=t,c3492=t,c3493=t,c3494=t,c3495=t,c3496=t,c3497=t,c3498=t,c3499=t,c3500=t,c3501=t," - "c3502=t,c3503=t,c3504=t,c3505=t,c3506=t,c3507=t,c3508=t,c3509=t,c3510=t,c3511=t,c3512=t,c3513=t," - "c3514=t,c3515=t,c3516=t,c3517=t,c3518=t,c3519=t,c3520=t,c3521=t,c3522=t,c3523=t,c3524=t,c3525=t,c3526=t,c3527=t," - "c3528=t,c3529=t,c3530=t,c3531=t,c3532=t,c3533=t,c3534=t,c3535=t,c3536=t,c3537=t,c3538=t,c3539=t,c3540=t,c3541=t," - "c3542=t,c3543=t,c3544=t,c3545=t,c3546=t,c3547=t,c3548=t,c3549=t,c3550=t,c3551=t,c3552=t,c3553=t,c3554=t,c3555=t," - "c3556=t,c3557=t,c3558=t,c3559=t,c3560=t,c3561=t,c3562=t,c3563=t,c3564=t,c3565=t,c3566=t,c3567=t,c3568=t,c3569=t," - "c3570=t,c3571=t,c3572=t,c3573=t,c3574=t,c3575=t,c3576=t,c3577=t,c3578=t,c3579=t,c3580=t,c3581=t,c3582=t,c3583=t," - "c3584=t,c3585=t,c3586=t,c3587=t,c3588=t,c3589=t,c3590=t,c3591=t,c3592=t,c3593=t,c3594=t,c3595=t,c3596=t,c3597=t," - "c3598=t,c3599=t,c3600=t,c3601=t,c3602=t,c3603=t,c3604=t,c3605=t,c3606=t,c3607=t,c3608=t,c3609=t,c3610=t,c3611=t," - "c3612=t,c3613=t,c3614=t,c3615=t,c3616=t,c3617=t,c3618=t,c3619=t,c3620=t,c3621=t,c3622=t,c3623=t,c3624=t,c3625=t," - "c3626=t,c3627=t,c3628=t,c3629=t,c3630=t,c3631=t,c3632=t,c3633=t,c3634=t,c3635=t,c3636=t,c3637=t,c3638=t,c3639=t," - "c3640=t,c3641=t,c3642=t,c3643=t,c3644=t,c3645=t,c3646=t,c3647=t,c3648=t,c3649=t,c3650=t,c3651=t,c3652=t,c3653=t," - "c3654=t,c3655=t,c3656=t,c3657=t,c3658=t,c3659=t,c3660=t," - "c3661=t,c3662=t,c3663=t,c3664=t,c3665=t,c3666=t,c3667=t,c3668=t,c3669=t,c3670=t,c3671=t,c3672=t,c3673=t,c3674=t," - "c3675=t,c3676=t,c3677=t,c3678=t,c3679=t,c3680=t,c3681=t,c3682=t,c3683=t,c3684=t,c3685=t,c3686=t,c3687=t,c3688=t," - "c3689=t,c3690=t,c3691=t,c3692=t,c3693=t,c3694=t,c3695=t,c3696=t,c3697=t,c3698=t,c3699=t,c3700=t,c3701=t,c3702=t," - "c3703=t,c3704=t,c3705=t,c3706=t,c3707=t,c3708=t,c3709=t,c3710=t,c3711=t,c3712=t,c3713=t,c3714=t,c3715=t,c3716=t," - "c3717=t,c3718=t,c3719=t,c3720=t,c3721=t,c3722=t,c3723=t,c3724=t,c3725=t,c3726=t,c3727=t,c3728=t,c3729=t,c3730=t," - "c3731=t,c3732=t,c3733=t,c3734=t,c3735=t,c3736=t,c3737=t,c3738=t,c3739=t,c3740=t,c3741=t,c3742=t,c3743=t,c3744=t," - "c3745=t,c3746=t,c3747=t,c3748=t,c3749=t,c3750=t,c3751=t,c3752=t,c3753=t,c3754=t,c3755=t,c3756=t,c3757=t,c3758=t," - "c3759=t,c3760=t,c3761=t,c3762=t,c3763=t,c3764=t,c3765=t,c3766=t,c3767=t,c3768=t,c3769=t,c3770=t,c3771=t,c3772=t," - "c3773=t,c3774=t,c3775=t,c3776=t,c3777=t,c3778=t,c3779=t,c3780=t,c3781=t,c3782=t,c3783=t,c3784=t,c3785=t,c3786=t," - "c3787=t,c3788=t,c3789=t,c3790=t,c3791=t,c3792=t,c3793=t,c3794=t,c3795=t,c3796=t,c3797=t,c3798=t,c3799=t,c3800=t," - "c3801=t,c3802=t,c3803=t,c3804=t,c3805=t,c3806=t,c3807=t," - "c3808=t,c3809=t,c3810=t,c3811=t,c3812=t,c3813=t,c3814=t,c3815=t,c3816=t,c3817=t,c3818=t,c3819=t,c3820=t,c3821=t," - "c3822=t,c3823=t,c3824=t,c3825=t,c3826=t,c3827=t,c3828=t,c3829=t,c3830=t,c3831=t,c3832=t,c3833=t,c3834=t,c3835=t," - "c3836=t,c3837=t,c3838=t,c3839=t,c3840=t,c3841=t,c3842=t,c3843=t,c3844=t,c3845=t,c3846=t,c3847=t,c3848=t,c3849=t," - "c3850=t,c3851=t,c3852=t,c3853=t,c3854=t,c3855=t,c3856=t,c3857=t,c3858=t,c3859=t,c3860=t,c3861=t,c3862=t,c3863=t," - "c3864=t,c3865=t,c3866=t,c3867=t,c3868=t,c3869=t,c3870=t,c3871=t,c3872=t,c3873=t,c3874=t,c3875=t,c3876=t,c3877=t," - "c3878=t,c3879=t,c3880=t,c3881=t,c3882=t,c3883=t,c3884=t,c3885=t,c3886=t,c3887=t,c3888=t,c3889=t,c3890=t,c3891=t," - "c3892=t,c3893=t,c3894=t,c3895=t,c3896=t,c3897=t,c3898=t,c3899=t,c3900=t,c3901=t,c3902=t,c3903=t,c3904=t,c3905=t," - "c3906=t,c3907=t,c3908=t,c3909=t,c3910=t,c3911=t,c3912=t,c3913=t,c3914=t,c3915=t,c3916=t,c3917=t,c3918=t,c3919=t," - "c3920=t,c3921=t,c3922=t,c3923=t,c3924=t,c3925=t,c3926=t,c3927=t,c3928=t,c3929=t,c3930=t,c3931=t,c3932=t,c3933=t," - "c3934=t,c3935=t,c3936=t,c3937=t,c3938=t,c3939=t,c3940=t,c3941=t,c3942=t,c3943=t,c3944=t,c3945=t,c3946=t,c3947=t," - "c3948=t,c3949=t,c3950=t,c3951=t,c3952=t,c3953=t,c3954=t," - "c3955=t,c3956=t,c3957=t,c3958=t,c3959=t,c3960=t,c3961=t,c3962=t,c3963=t,c3964=t,c3965=t,c3966=t,c3967=t,c3968=t," - "c3969=t,c3970=t,c3971=t,c3972=t,c3973=t,c3974=t,c3975=t,c3976=t,c3977=t,c3978=t,c3979=t,c3980=t,c3981=t,c3982=t," - "c3983=t,c3984=t,c3985=t,c3986=t,c3987=t,c3988=t,c3989=t,c3990=t,c3991=t,c3992=t,c3993=t,c3994=t,c3995=t,c3996=t," - "c3997=t,c3998=t,c3999=t,c4000=t,c4001=t,c4002=t,c4003=t,c4004=t,c4005=t,c4006=t,c4007=t,c4008=t,c4009=t,c4010=t," - "c4011=t,c4012=t,c4013=t,c4014=t,c4015=t,c4016=t,c4017=t,c4018=t,c4019=t,c4020=t,c4021=t,c4022=t,c4023=t,c4024=t," - "c4025=t,c4026=t,c4027=t,c4028=t,c4029=t,c4030=t,c4031=t,c4032=t,c4033=t,c4034=t,c4035=t,c4036=t,c4037=t,c4038=t," - "c4039=t,c4040=t,c4041=t,c4042=t,c4043=t,c4044=t,c4045=t,c4046=t,c4047=t,c4048=t,c4049=t,c4050=t,c4051=t,c4052=t," - "c4053=t,c4054=t,c4055=t,c4056=t,c4057=t,c4058=t,c4059=t,c4060=t,c4061=t,c4062=t,c4063=t,c4064=t,c4065=t,c4066=t," - "c4067=t,c4068=t,c4069=t,c4070=t,c4071=t,c4072=t,c4073=t,c4074=t,c4075=t,c4076=t,c4077=t,c4078=t,c4079=t,c4080=t," - "c4081=t,c4082=t,c4083=t,c4084=t,c4085=t,c4086=t,c4087=t,c4088=t,c4089=t,c4090=t,c4091=t,c4092=t,c4093=t " - "1626006833640000000"}; - - int ret = TSDB_CODE_SUCCESS; - for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseInfluxLine(info, sql[i], strlen(sql[i])); - if (ret != TSDB_CODE_SUCCESS) break; +//TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { +// SSmlHandle *info = smlBuildSmlInfo(NULL); +// info->protocol = TSDB_SML_JSON_PROTOCOL; +// ASSERT_NE(info, nullptr); +// +// const char *sql[] = { +// "[{\"metric\":\"sys.cpu.nice\",\"timestamp\": 1346846400,\"value\": 18,\"tags\": {\"host\": \"lga\"}},{\"metric\": \"sys.sdfa\",\"timestamp\": 1346846400,\"value\": \"18\",\"tags\": {\"host\": 8932}},]", +// }; +// for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { +// char *dataPointStart = (char *)sql[i]; +// int8_t offset[4] = {0}; +// while (1) { +// SSmlLineInfo elements = {0}; +// if(offset[0] == 0){ +// smlJsonParseObjFirst(&dataPointStart, &elements, offset); +// }else{ +// smlJsonParseObj(&dataPointStart, &elements, offset); +// } +// if(*dataPointStart == '\0') break; +// +// SArray *tags = smlJsonParseTags(elements.tags, elements.tags + elements.tagsLen); +// size_t num = taosArrayGetSize(tags); +// ASSERT_EQ(num, 1); +// +// taosArrayDestroy(tags); +// } +// } +// smlDestroyInfo(info); +//} + +TEST(testCase, smlParseNumber_performance_Test) { + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + SSmlKv kv; + + char* str[3] = {"2893f64", "2323u32", "93u8"}; + for (int i = 0; i < 3; ++i) { + int64_t t1 = taosGetTimestampUs(); + for (int j = 0; j < 10000000; ++j) { + kv.value = str[i]; + kv.length = strlen(str[i]); + smlParseNumber(&kv, &msgBuf); + } + printf("smlParseNumber:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t1); + printf("\n"); + int64_t t2 = taosGetTimestampUs(); + for (int j = 0; j < 10000000; ++j) { + kv.value = str[i]; + kv.length = strlen(str[i]); + smlParseNumberOld(&kv, &msgBuf); + } + printf("smlParseNumberOld:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t2); + printf("\n\n"); } - ASSERT_NE(ret, 0); - smlDestroyInfo(info); -} +} \ No newline at end of file diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 60a673ef9cf25b61e8889f4941802a8500b726a2..8791a81bbe7fbed021c13b772892f11ca109a41d 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -67,6 +67,8 @@ static const SSysDbTableSchema clusterSchema[] = { {.name = "name", .bytes = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "uptime", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, + {.name = "version", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "expire_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, }; static const SSysDbTableSchema userDBSchema[] = { @@ -176,6 +178,18 @@ static const SSysDbTableSchema userTagsSchema[] = { {.name = "tag_value", .bytes = TSDB_MAX_TAGS_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; +static const SSysDbTableSchema userColsSchema[] = { + {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "table_type", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "col_name", .bytes = TSDB_COL_NAME_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "col_type", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "col_length", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "col_precision", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "col_scale", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "col_nullable", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false} +}; + static const SSysDbTableSchema userTblDistSchema[] = { {.name = "db_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, @@ -292,6 +306,7 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_STABLES, userStbsSchema, tListLen(userStbsSchema), false}, {TSDB_INS_TABLE_TABLES, userTblsSchema, tListLen(userTblsSchema), false}, {TSDB_INS_TABLE_TAGS, userTagsSchema, tListLen(userTagsSchema), false}, + {TSDB_INS_TABLE_COLS, userColsSchema, tListLen(userColsSchema), false}, // {TSDB_INS_TABLE_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)}, {TSDB_INS_TABLE_USERS, userUsersSchema, tListLen(userUsersSchema), false}, {TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema), true}, diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 2b43229b838fffb14625f01ba98bfc5c196a8458..9b5b32cf695ff6d38ff09fe9eb778dbb2679a43c 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -19,10 +19,9 @@ #include "tlog.h" #include "tname.h" -#define MALLOC_ALIGN_BYTES 256 +#define MALLOC_ALIGN_BYTES 32 int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { - ASSERT(pColumnInfoData != NULL); if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { return pColumnInfoData->varmeta.length; } else { @@ -38,7 +37,8 @@ int32_t colDataGetFullLength(const SColumnInfoData* pColumnInfoData, int32_t num if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { return pColumnInfoData->varmeta.length + sizeof(int32_t) * numOfRows; } else { - return ((pColumnInfoData->info.type == TSDB_DATA_TYPE_NULL) ? 0 : pColumnInfoData->info.bytes * numOfRows) + BitmapLen(numOfRows); + return ((pColumnInfoData->info.type == TSDB_DATA_TYPE_NULL) ? 0 : pColumnInfoData->info.bytes * numOfRows) + + BitmapLen(numOfRows); } } @@ -65,14 +65,12 @@ int32_t getJsonValueLen(const char* data) { } int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull) { - ASSERT(pColumnInfoData != NULL); - if (isNull) { // There is a placehold for each NULL value of binary or nchar type. if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { pColumnInfoData->varmeta.offset[currentRow] = -1; // it is a null value of VAR type. } else { - colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow); + colDataSetNull_f_s(pColumnInfoData, currentRow); } pColumnInfoData->hasNull = true; @@ -111,7 +109,7 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con uint32_t len = pColumnInfoData->varmeta.length; pColumnInfoData->varmeta.offset[currentRow] = len; - memcpy(pColumnInfoData->pData + len, pData, dataLen); + memmove(pColumnInfoData->pData + len, pData, dataLen); pColumnInfoData->varmeta.length += dataLen; } else { memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow, pData, pColumnInfoData->info.bytes); @@ -177,8 +175,6 @@ static void doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t curren int32_t colDataAppendNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, uint32_t numOfRows) { - ASSERT(pData != NULL && pColumnInfoData != NULL); - int32_t len = pColumnInfoData->info.bytes; if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { len = varDataTLen(pData); @@ -236,7 +232,10 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity, const SColumnInfoData* pSource, int32_t numOfRow2) { - ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); + if (pColumnInfoData->info.type != pSource->info.type) { + return TSDB_CODE_FAILED; + } + if (numOfRow2 == 0) { return numOfRow1; } @@ -279,7 +278,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int pColumnInfoData->varmeta.allocLen = len + oldLen; } - if (pColumnInfoData->pData && pSource->pData) { // TD-20382 + if (pColumnInfoData->pData && pSource->pData) { // TD-20382 memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len); } pColumnInfoData->varmeta.length = len + oldLen; @@ -316,13 +315,12 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, const SDataBlockInfo* pBlockInfo) { - ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); - if (numOfRows <= 0) { - return numOfRows; + if (pColumnInfoData->info.type != pSource->info.type || (pBlockInfo != NULL && pBlockInfo->capacity < numOfRows)) { + return TSDB_CODE_FAILED; } - if (pBlockInfo != NULL) { - ASSERT(pBlockInfo->capacity >= numOfRows); + if (numOfRows <= 0) { + return numOfRows; } if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { @@ -358,14 +356,14 @@ size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { return taosArrayGetSiz size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; } int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex) { - if (pDataBlock->info.rows > 0) { -// ASSERT(pDataBlock->info.dataLoad == 1); - } - if (pDataBlock == NULL || pDataBlock->info.rows <= 0 || pDataBlock->info.dataLoad == 0) { return 0; } + if (pDataBlock->info.rows > 0) { + // ASSERT(pDataBlock->info.dataLoad == 1); + } + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); if (numOfCols <= 0) { return -1; @@ -388,7 +386,6 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex) } int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { - assert(pSrc != NULL && pDest != NULL); int32_t capacity = pDest->info.capacity; size_t numOfCols = taosArrayGetSize(pDest->pDataBlock); @@ -406,8 +403,6 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { } size_t blockDataGetSize(const SSDataBlock* pBlock) { - assert(pBlock != NULL); - size_t total = 0; size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { @@ -422,8 +417,6 @@ size_t blockDataGetSize(const SSDataBlock* pBlock) { // Actual data rows pluses the corresponding meta data must fit in one memory buffer of the given page size. int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, int32_t pageSize) { - ASSERT(pBlock != NULL && stopIndex != NULL); - size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int32_t numOfRows = pBlock->info.rows; @@ -437,7 +430,9 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd if (!hasVarCol) { size_t rowSize = blockDataGetRowSize(pBlock); int32_t capacity = payloadSize / (rowSize + numOfCols * bitmapChar / 8.0); - ASSERT(capacity > 0); + if (capacity <= 0) { + return TSDB_CODE_FAILED; + } *stopIndex = startIndex + capacity - 1; if (*stopIndex >= numOfRows) { @@ -469,7 +464,9 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd if (size > pageSize) { // pageSize must be able to hold one row *stopIndex = j - 1; - ASSERT(*stopIndex >= startIndex); + if (*stopIndex < startIndex) { + return TSDB_CODE_FAILED; + } return TSDB_CODE_SUCCESS; } @@ -540,8 +537,6 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 * @return */ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { - ASSERT(pBlock != NULL); - // write the number of rows *(uint32_t*)buf = pBlock->info.rows; @@ -612,7 +607,9 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { } pCol->varmeta.length = colLength; - ASSERT(pCol->varmeta.length <= pCol->varmeta.allocLen); + if (pCol->varmeta.length > pCol->varmeta.allocLen) { + return TSDB_CODE_FAILED; + } } memcpy(pCol->pData, pStart, colLength); @@ -659,7 +656,9 @@ int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity) } pCol->varmeta.length = colLength; - ASSERT(pCol->varmeta.length <= pCol->varmeta.allocLen); + if (pCol->varmeta.length > pCol->varmeta.allocLen) { + return TSDB_CODE_FAILED; + } } if (!colDataIsNNull_s(pCol, 0, pBlock->info.rows)) { @@ -673,7 +672,6 @@ int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity) } size_t blockDataGetRowSize(SSDataBlock* pBlock) { - ASSERT(pBlock != NULL); if (pBlock->info.rowSize == 0) { size_t rowSize = 0; @@ -702,7 +700,6 @@ size_t blockDataGetSerialMetaSize(uint32_t numOfCols) { } double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { - ASSERT(pBlock != NULL); double rowSize = 0; size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); @@ -828,7 +825,7 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB } else { for (int32_t j = 0; j < pDataBlock->info.rows; ++j) { if (colDataIsNull_f(pSrc->nullbitmap, index[j])) { - colDataSetNull_f(pDst->nullbitmap, j); + colDataSetNull_f_s(pDst, j); continue; } memcpy(pDst->pData + j * pDst->info.bytes, pSrc->pData + index[j] * pDst->info.bytes, pDst->info.bytes); @@ -905,7 +902,6 @@ static int32_t* createTupleIndex(size_t rows) { static void destroyTupleIndex(int32_t* index) { taosMemoryFreeClear(index); } int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { - ASSERT(pDataBlock != NULL && pOrderInfo != NULL); if (pDataBlock->info.rows <= 1) { return TSDB_CODE_SUCCESS; } @@ -1149,8 +1145,7 @@ void blockDataCleanup(SSDataBlock* pDataBlock) { void blockDataEmpty(SSDataBlock* pDataBlock) { SDataBlockInfo* pInfo = &pDataBlock->info; - ASSERT(pInfo->rows <= pDataBlock->info.capacity); - if (pInfo->capacity == 0) { + if (pInfo->capacity == 0 || pInfo->rows > pDataBlock->info.capacity) { return; } @@ -1166,16 +1161,17 @@ void blockDataEmpty(SSDataBlock* pDataBlock) { pInfo->window.skey = 0; } -// todo temporarily disable it -static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows, bool clearPayload) { - ASSERT(numOfRows > 0); - if (numOfRows <= pBlockInfo->capacity) { +/* + * NOTE: the type of the input column may be TSDB_DATA_TYPE_NULL, which is used to denote + * the all NULL value in this column. It is an internal representation of all NULL value column, and no visible to + * any users. The length of TSDB_DATA_TYPE_NULL is 0, and it is an special case. + */ +static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows, + bool clearPayload) { + if (numOfRows <= 0 || numOfRows <= pBlockInfo->capacity) { return TSDB_CODE_SUCCESS; } - // todo temp disable it - // ASSERT(pColumn->info.bytes != 0); - int32_t existedRows = pBlockInfo->rows; if (IS_VAR_DATA_TYPE(pColumn->info.type)) { @@ -1196,9 +1192,12 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* int32_t oldLen = BitmapLen(existedRows); pColumn->nullbitmap = tmp; memset(&pColumn->nullbitmap[oldLen], 0, BitmapLen(numOfRows) - oldLen); - ASSERT(pColumn->info.bytes); + if (pColumn->info.bytes == 0) { + return TSDB_CODE_FAILED; + } - // make sure the allocated memory is MALLOC_ALIGN_BYTES aligned + // here we employ the aligned malloc function, to make sure that the address of allocated memory is aligned + // to MALLOC_ALIGN_BYTES tmp = taosMemoryMallocAlign(MALLOC_ALIGN_BYTES, numOfRows * pColumn->info.bytes); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -1212,9 +1211,11 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pColumn->pData = tmp; - // todo remove it soon + // check if the allocated memory is aligned to the requried bytes. #if defined LINUX - ASSERT((((uint64_t)pColumn->pData) & (MALLOC_ALIGN_BYTES - 1)) == 0x0); + if ((((uint64_t)pColumn->pData) & (MALLOC_ALIGN_BYTES - 1)) != 0x0) { + return TSDB_CODE_FAILED; + } #endif if (clearPayload) { @@ -1225,7 +1226,7 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* return TSDB_CODE_SUCCESS; } -void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) { +void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) { pColumn->hasNull = false; if (IS_VAR_DATA_TYPE(pColumn->info.type)) { @@ -1251,25 +1252,6 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { return TSDB_CODE_SUCCESS; } - size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - code = doEnsureCapacity(p, &pDataBlock->info, numOfRows, true); - if (code) { - return code; - } - } - - pDataBlock->info.capacity = numOfRows; - return TSDB_CODE_SUCCESS; -} - -int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRows) { - int32_t code = 0; - if (numOfRows == 0 || numOfRows <= pDataBlock->info.capacity) { - return TSDB_CODE_SUCCESS; - } - size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); @@ -1293,7 +1275,6 @@ void blockDataFreeRes(SSDataBlock* pBlock) { taosArrayDestroy(pBlock->pDataBlock); pBlock->pDataBlock = NULL; taosMemoryFreeClear(pBlock->pBlockAgg); - taosMemoryFree(pBlock->info.pTag); memset(&pBlock->info, 0, sizeof(SDataBlockInfo)); } @@ -1308,8 +1289,6 @@ void* blockDataDestroy(SSDataBlock* pBlock) { } int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) { - ASSERT(src != NULL); - dst->info = src->info; dst->info.rows = 0; dst->info.capacity = 0; @@ -1344,8 +1323,6 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) { } int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src) { - ASSERT(src != NULL && dst != NULL); - blockDataCleanup(dst); int32_t code = blockDataEnsureCapacity(dst, src->info.rows); if (code != TSDB_CODE_SUCCESS) { @@ -1501,7 +1478,6 @@ SSDataBlock* createDataBlock() { } int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData) { - ASSERT(pBlock != NULL && pColInfoData != NULL); if (pBlock->pDataBlock == NULL) { pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); if (pBlock->pDataBlock == NULL) { @@ -1536,7 +1512,6 @@ SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId) } SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index) { - ASSERT(pBlock != NULL); if (index >= taosArrayGetSize(pBlock->pDataBlock)) { return NULL; } @@ -1875,7 +1850,7 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) { char pBuf[128] = {0}; int32_t sz = taosArrayGetSize(dataBlocks); for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGetP(dataBlocks, i); + SSDataBlock* pDataBlock = taosArrayGet(dataBlocks, i); size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); int32_t rows = pDataBlock->info.rows; @@ -1887,21 +1862,37 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) { for (int32_t k = 0; k < numOfCols; k++) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + if (k == 0) { + printf("cols:%d |", (int32_t)numOfCols); + } if (colDataIsNull(pColInfoData, rows, j, NULL)) { printf(" %15s |", "NULL"); continue; } + switch (pColInfoData->info.type) { case TSDB_DATA_TYPE_TIMESTAMP: formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); printf(" %25s |", pBuf); break; case TSDB_DATA_TYPE_BOOL: - printf(" %15d |", *(int32_t*)var); + printf(" %15" PRIi8 " |", *(int8_t*)var); + break; + case TSDB_DATA_TYPE_TINYINT: + printf(" %15" PRIi8 " |", *(int8_t*)var); + break; + case TSDB_DATA_TYPE_SMALLINT: + printf(" %15" PRIi16 " |", *(int16_t*)var); break; case TSDB_DATA_TYPE_INT: printf(" %15d |", *(int32_t*)var); break; + case TSDB_DATA_TYPE_UTINYINT: + printf(" %15" PRIu8 " |", *(uint8_t*)var); + break; + case TSDB_DATA_TYPE_USMALLINT: + printf(" %15" PRIu16 " |", *(uint16_t*)var); + break; case TSDB_DATA_TYPE_UINT: printf(" %15u |", *(uint32_t*)var); break; @@ -1951,9 +1942,10 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) int32_t len = 0; len += snprintf(dumpBuf + len, size - len, "===stream===%s|block type %d|child id %d|group id:%" PRIu64 "|uid:%" PRId64 - "|rows:%d|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "\n", + "|rows:%d|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "|tbl:%s\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.id.groupId, - pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version, pDataBlock->info.calWin.skey, pDataBlock->info.calWin.ekey); + pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version, + pDataBlock->info.calWin.skey, pDataBlock->info.calWin.ekey, pDataBlock->info.parTbName); if (len >= size - 1) return dumpBuf; for (int32_t j = 0; j < rows; j++) { @@ -2055,6 +2047,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) * @param suid * */ +#if 0 int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataBlock, STSchema* pTSchema, int32_t vgId, tb_uid_t suid) { int32_t bufSize = sizeof(SSubmitReq); @@ -2142,7 +2135,6 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_MEDIUMBLOB: uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); - ASSERT(0); break; default: if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { @@ -2176,7 +2168,6 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB } } else { uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); - ASSERT(0); } break; } @@ -2220,32 +2211,189 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB return TSDB_CODE_SUCCESS; } +#endif + +int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDataBlock, const STSchema* pTSchema, + int64_t uid, int32_t vgId, tb_uid_t suid) { + SSubmitReq2* pReq = *ppReq; + SArray* pVals = NULL; + int32_t numOfBlks = 0; + int32_t sz = 1; + + terrno = TSDB_CODE_SUCCESS; + + if (NULL == pReq) { + if (!(pReq = taosMemoryMalloc(sizeof(SSubmitReq2)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { + goto _end; + } + } + + for (int32_t i = 0; i < sz; ++i) { + int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); + int32_t rows = pDataBlock->info.rows; + + if (colNum <= 1) { // invalid if only with TS col + continue; + } + + // the rsma result should has the same column number with schema. + ASSERT(colNum == pTSchema->numOfCols); + + SSubmitTbData tbData = {0}; + + if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow*)))) { + goto _end; + } + tbData.suid = suid; + tbData.uid = uid; + tbData.sver = pTSchema->version; + + if (!pVals && !(pVals = taosArrayInit(colNum, sizeof(SColVal)))) { + taosArrayDestroy(tbData.aRowP); + goto _end; + } + + for (int32_t j = 0; j < rows; ++j) { // iterate by row + + taosArrayClear(pVals); + + bool isStartKey = false; + int32_t offset = 0; + for (int32_t k = 0; k < colNum; ++k) { // iterate by column + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + const STColumn* pCol = &pTSchema->columns[k]; + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + ASSERT(pColInfoData->info.type == pCol->type); + if (!isStartKey) { + isStartKey = true; + ASSERT(PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId); + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, (SValue){.val = *(TSKEY*)var}); + taosArrayPush(pVals, &cv); + } else if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, (SValue){.val = *(int64_t*)var}); + taosArrayPush(pVals, &cv); + } + break; + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY + ASSERT(pColInfoData->info.type == pCol->type); + if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + void* data = colDataGetVarData(pColInfoData, j); + SValue sv = (SValue){.nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } + break; + } + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_MEDIUMBLOB: + uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); + ASSERT(0); + break; + default: + if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { + if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); // should use pCol->type + taosArrayPush(pVals, &cv); + } else { + SValue sv; + if (pCol->type == pColInfoData->info.type) { + memcpy(&sv.val, var, tDataTypes[pCol->type].bytes); + } else { + /** + * 1. sum/avg would convert to int64_t/uint64_t/double during aggregation + * 2. below conversion may lead to overflow or loss, the app should select the right data type. + */ + char tv[8] = {0}; + if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) { + float v = 0; + GET_TYPED_DATA(v, float, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) { + double v = 0; + GET_TYPED_DATA(v, double, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) { + int64_t v = 0; + GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else { + uint64_t v = 0; + GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } + memcpy(&sv.val, tv, tDataTypes[pCol->type].bytes); + } + SColVal cv = COL_VAL_VALUE(pCol->colId, pColInfoData->info.type, sv); + taosArrayPush(pVals, &cv); + } + } else { + uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); + ASSERT(0); + } + break; + } + } + SRow* pRow = NULL; + if ((terrno = tRowBuild(pVals, pTSchema, &pRow)) < 0) { + tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + goto _end; + } + ASSERT(pRow); + taosArrayPush(tbData.aRowP, &pRow); + } + + taosArrayPush(pReq->aSubmitTbData, &tbData); + } +_end: + taosArrayDestroy(pVals); + if (terrno != 0) { + *ppReq = NULL; + if (pReq) tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + return TSDB_CODE_FAILED; + } + *ppReq = pReq; + return TSDB_CODE_SUCCESS; +} char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { - ASSERT(stbFullName[0] != 0); - SArray* tags = taosArrayInit(0, sizeof(void*)); - if (tags == NULL) { + if (stbFullName[0] == 0) { return NULL; } - SSmlKv* pTag = taosMemoryCalloc(1, sizeof(SSmlKv)); - if (pTag == NULL) { - taosArrayDestroy(tags); + SArray* tags = taosArrayInit(0, sizeof(SSmlKv)); + if (tags == NULL) { return NULL; } void* cname = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1); if (cname == NULL) { taosArrayDestroy(tags); - taosMemoryFree(pTag); return NULL; } - pTag->key = "group_id"; - pTag->keyLen = strlen(pTag->key); - pTag->type = TSDB_DATA_TYPE_UBIGINT; - pTag->u = groupId; - pTag->length = sizeof(uint64_t); + SSmlKv pTag = {.key = "group_id", + .keyLen = sizeof("group_id") - 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .u = groupId, + .length = sizeof(uint64_t)}; taosArrayPush(tags, &pTag); RandTableName rname = { @@ -2257,10 +2405,11 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { buildChildTableName(&rname); - taosMemoryFree(pTag); taosArrayDestroy(tags); - ASSERT(rname.ctbShortName && rname.ctbShortName[0]); + if ((rname.ctbShortName && rname.ctbShortName[0]) == 0) { + return NULL; + } return rname.ctbShortName; } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index fc49c033794c946a9881b6ae1efeabcb101e99a7..5e001a96872306d9369d388f8823a2baecdbf04f 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -20,6 +20,8 @@ #include "tdatablock.h" #include "tlog.h" +static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData); + // SBuffer ================================ void tBufferDestroy(SBuffer *pBuffer) { tFree(pBuffer->pBuf); @@ -61,9 +63,9 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); #define KV_FLG_MID ((uint8_t)0x20) #define KV_FLG_BIG ((uint8_t)0x30) -#define ROW_BIT_NONE ((uint8_t)0x0) -#define ROW_BIT_NULL ((uint8_t)0x1) -#define ROW_BIT_VALUE ((uint8_t)0x2) +#define BIT_FLG_NONE ((uint8_t)0x0) +#define BIT_FLG_NULL ((uint8_t)0x1) +#define BIT_FLG_VALUE ((uint8_t)0x2) #pragma pack(push, 1) typedef struct { @@ -95,28 +97,29 @@ typedef struct { } \ } while (0) -int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { +int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) { int32_t code = 0; - ASSERT(taosArrayGetSize(aColVal) > 0); + ASSERT(TARRAY_SIZE(aColVal) > 0); ASSERT(((SColVal *)aColVal->pData)[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); ASSERT(((SColVal *)aColVal->pData)[0].type == TSDB_DATA_TYPE_TIMESTAMP); // scan --------------- - uint8_t flag = 0; - int32_t iColVal = 1; - const int32_t nColVal = taosArrayGetSize(aColVal); - SColVal *pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - int32_t iTColumn = 1; - STColumn *pTColumn = pTSchema->columns + iTColumn; - int32_t ntp = 0; - int32_t nkv = 0; - int32_t maxIdx = 0; - int32_t nIdx = 0; + SRow *pRow = NULL; + SColVal *colVals = (SColVal *)TARRAY_DATA(aColVal); + uint8_t flag = 0; + int32_t iColVal = 1; + const int32_t nColVal = TARRAY_SIZE(aColVal); + SColVal *pColVal = (iColVal < nColVal) ? &colVals[iColVal] : NULL; + int32_t iTColumn = 1; + const STColumn *pTColumn = pTSchema->columns + iTColumn; + int32_t ntp = 0; + int32_t nkv = 0; + int32_t maxIdx = 0; + int32_t nIdx = 0; while (pTColumn) { if (pColVal) { if (pColVal->cid == pTColumn->colId) { - ntp += TYPE_BYTES[pTColumn->type]; if (COL_VAL_IS_VALUE(pColVal)) { // VALUE flag |= HAS_VALUE; maxIdx = nkv; @@ -139,17 +142,15 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } else if (pColVal->cid > pTColumn->colId) { // NONE flag |= HAS_NONE; - ntp += TYPE_BYTES[pTColumn->type]; pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } } else { // NONE flag |= HAS_NONE; - ntp += TYPE_BYTES[pTColumn->type]; pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } } @@ -161,17 +162,17 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { ntp = sizeof(SRow); break; case HAS_VALUE: - ntp = sizeof(SRow) + ntp; + ntp = sizeof(SRow) + pTSchema->flen + ntp; break; case (HAS_NULL | HAS_NONE): ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1); break; case (HAS_VALUE | HAS_NONE): case (HAS_VALUE | HAS_NULL): - ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1) + ntp; + ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntp; break; case (HAS_VALUE | HAS_NULL | HAS_NONE): - ntp = sizeof(SRow) + BIT2_SIZE(pTSchema->numOfCols - 1) + ntp; + ntp = sizeof(SRow) + BIT2_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntp; break; default: ASSERT(0); @@ -196,12 +197,14 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } // alloc -------------- - SRow *pRow = NULL; - code = tBufferReserve(pBuffer, nRow, (void **)&pRow); - if (code) return code; + pRow = taosMemoryMalloc(nRow); + if (NULL == pRow) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } // build -------------- - pColVal = (SColVal *)taosArrayGet(aColVal, 0); + pColVal = &colVals[0]; pRow->flag = flag; pRow->rsv = 0; @@ -214,10 +217,10 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } iColVal = 1; - pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (iColVal < nColVal) ? &colVals[iColVal] : NULL; iTColumn = 1; pTColumn = pTSchema->columns + iTColumn; - if (flag & 0xf0) { // KV + if (flag >> 4) { // KV SKVIdx *pIdx = (SKVIdx *)pRow->data; int32_t iIdx = 0; int32_t nv = 0; @@ -266,11 +269,11 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } else if (pColVal->cid > pTColumn->colId) { // NONE pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } } else { // NONE pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; @@ -306,12 +309,20 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { break; } + if (pb) { + if (flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) { + memset(pb, 0, BIT2_SIZE(pTSchema->numOfCols - 1)); + } else { + memset(pb, 0, BIT1_SIZE(pTSchema->numOfCols - 1)); + } + } + // build impl while (pTColumn) { if (pColVal) { if (pColVal->cid == pTColumn->colId) { if (COL_VAL_IS_VALUE(pColVal)) { // VALUE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_VALUE); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_VALUE); if (IS_VAR_DATA_TYPE(pTColumn->type)) { *(int32_t *)(pf + pTColumn->offset) = nv; @@ -324,24 +335,24 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { memcpy(pf + pTColumn->offset, &pColVal->value.val, TYPE_BYTES[pTColumn->type]); } } else if (COL_VAL_IS_NONE(pColVal)) { // NONE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NONE); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); } else { // NULL - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NULL); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NULL); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); } pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } else if (pColVal->cid > pTColumn->colId) { // NONE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NONE); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } } else { // NONE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NONE); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } @@ -349,6 +360,12 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } _exit: + if (code) { + *ppRow = NULL; + tRowDestroy(pRow); + } else { + *ppRow = pRow; + } return code; } @@ -376,7 +393,7 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { return; } - if (pRow->flag & 0xf0) { // KV Row + if (pRow->flag >> 4) { // KV Row SKVIdx *pIdx = (SKVIdx *)pRow->data; uint8_t *pv = NULL; if (pRow->flag & KV_FLG_LIT) { @@ -432,64 +449,190 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); } else { // Tuple Row - uint8_t *pf = NULL; - uint8_t *pv = NULL; - uint8_t bv = ROW_BIT_VALUE; + if (pRow->flag == HAS_VALUE) { + pColVal->cid = pTColumn->colId; + pColVal->type = pTColumn->type; + pColVal->flag = CV_FLAG_VALUE; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + uint8_t *pData = pRow->data + pTSchema->flen + *(int32_t *)(pRow->data + pTColumn->offset); + pData += tGetU32v(pData, &pColVal->value.nData); + if (pColVal->value.nData) { + pColVal->value.pData = pData; + } else { + pColVal->value.pData = NULL; + } + } else { + memcpy(&pColVal->value.val, pRow->data + pTColumn->offset, TYPE_BYTES[pTColumn->type]); + } + } else { + uint8_t *pf; + uint8_t *pv; + uint8_t bv = BIT_FLG_VALUE; - switch (pRow->flag) { - case HAS_VALUE: - pf = pRow->data; - pv = pf + pTSchema->flen; - break; - case (HAS_NULL | HAS_NONE): - bv = GET_BIT1(pRow->data, iCol - 1); - break; - case (HAS_VALUE | HAS_NONE): - bv = GET_BIT1(pRow->data, iCol - 1); - if (bv) bv++; - pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; - break; - case (HAS_VALUE | HAS_NULL): - bv = GET_BIT1(pRow->data, iCol - 1); - bv++; - pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; - break; - case (HAS_VALUE | HAS_NULL | HAS_NONE): - bv = GET_BIT2(pRow->data, iCol - 1); - pf = pRow->data + BIT2_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; - break; - default: - break; - } + switch (pRow->flag) { + case (HAS_NULL | HAS_NONE): + bv = GET_BIT1(pRow->data, iCol - 1); + break; + case (HAS_VALUE | HAS_NONE): + bv = GET_BIT1(pRow->data, iCol - 1); + if (bv) bv++; + pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL): + bv = GET_BIT1(pRow->data, iCol - 1); + bv++; + pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + bv = GET_BIT2(pRow->data, iCol - 1); + pf = pRow->data + BIT2_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + default: + ASSERT(0); + break; + } - if (bv == ROW_BIT_NONE) { - *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); - return; - } else if (bv == ROW_BIT_NULL) { - *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); - return; - } + if (bv == BIT_FLG_NONE) { + *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + return; + } else if (bv == BIT_FLG_NULL) { + *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); + return; + } - pColVal->cid = pTColumn->colId; - pColVal->type = pTColumn->type; - pColVal->flag = CV_FLAG_VALUE; - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset); - pData += tGetU32v(pData, &pColVal->value.nData); - if (pColVal->value.nData) { - pColVal->value.pData = pData; + pColVal->cid = pTColumn->colId; + pColVal->type = pTColumn->type; + pColVal->flag = CV_FLAG_VALUE; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset); + pData += tGetU32v(pData, &pColVal->value.nData); + if (pColVal->value.nData) { + pColVal->value.pData = pData; + } else { + pColVal->value.pData = NULL; + } } else { - pColVal->value.pData = NULL; + memcpy(&pColVal->value.val, pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]); } - } else { - memcpy(&pColVal->value.val, pv + pTColumn->offset, pTColumn->bytes); } } } +void tRowDestroy(SRow *pRow) { + if (pRow) taosMemoryFree(pRow); +} + +static int32_t tRowPCmprFn(const void *p1, const void *p2) { + if ((*(SRow **)p1)->ts < (*(SRow **)p2)->ts) { + return -1; + } else if ((*(SRow **)p1)->ts > (*(SRow **)p2)->ts) { + return 1; + } + + return 0; +} +static void tRowPDestroy(SRow **ppRow) { tRowDestroy(*ppRow); } +static int32_t tRowMergeImpl(SArray *aRowP, STSchema *pTSchema, int32_t iStart, int32_t iEnd, int8_t flag) { + int32_t code = 0; + + int32_t nRow = iEnd - iStart; + SRowIter **aIter = NULL; + SArray *aColVal = NULL; + SRow *pRow = NULL; + + aIter = taosMemoryCalloc(nRow, sizeof(SRowIter *)); + if (aIter == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t i = 0; i < nRow; i++) { + SRow *pRowT = taosArrayGetP(aRowP, iStart + i); + + code = tRowIterOpen(pRowT, pTSchema, &aIter[i]); + if (code) goto _exit; + } + + // merge + aColVal = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)); + if (aColVal == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t iCol = 0; iCol < pTSchema->numOfCols; iCol++) { + SColVal *pColVal = NULL; + for (int32_t iRow = 0; iRow < nRow; iRow++) { + SColVal *pColValT = tRowIterNext(aIter[iRow]); + + // todo: take strategy according to the flag + if (COL_VAL_IS_VALUE(pColValT)) { + pColVal = pColValT; + } else if (COL_VAL_IS_NULL(pColValT)) { + if (pColVal == NULL) { + pColVal = pColValT; + } + } + } + + if (pColVal) taosArrayPush(aColVal, pColVal); + } + + // build + code = tRowBuild(aColVal, pTSchema, &pRow); + if (code) goto _exit; + + taosArrayRemoveBatch(aRowP, iStart, nRow, (FDelete)tRowPDestroy); + taosArrayInsert(aRowP, iStart, &pRow); + +_exit: + if (aIter) { + for (int32_t i = 0; i < nRow; i++) { + tRowIterClose(&aIter[i]); + } + taosMemoryFree(aIter); + } + if (aColVal) taosArrayDestroy(aColVal); + if (code) tRowDestroy(pRow); + return code; +} + +void tRowSort(SArray *aRowP) { + if (TARRAY_SIZE(aRowP) <= 1) return; + taosArraySort(aRowP, tRowPCmprFn); +} + +int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag) { + int32_t code = 0; + + int32_t iStart = 0; + while (iStart < aRowP->size) { + SRow *pRow = (SRow *)taosArrayGetP(aRowP, iStart); + + int32_t iEnd = iStart + 1; + while (iEnd < aRowP->size) { + SRow *pRowT = (SRow *)taosArrayGetP(aRowP, iEnd); + + if (pRow->ts != pRowT->ts) break; + + iEnd++; + } + + if (iEnd - iStart > 1) { + code = tRowMergeImpl(aRowP, pTSchema, iStart, iEnd, flag); + if (code) return code; + } + + // the array is also changing, so the iStart just ++ instead of iEnd + iStart++; + } + + return code; +} + // SRowIter ======================================== struct SRowIter { SRow *pRow; @@ -505,10 +648,9 @@ struct SRowIter { uint8_t *pb; uint8_t *pf; }; - uint8_t *pv; }; - - SColVal cv; + uint8_t *pv; + SColVal cv; }; int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { @@ -528,15 +670,15 @@ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { if (pRow->flag == HAS_NONE || pRow->flag == HAS_NULL) goto _exit; - if (pRow->flag & 0xf0) { + if (pRow->flag >> 4) { pIter->iCol = 0; pIter->pIdx = (SKVIdx *)pRow->data; if (pRow->flag & KV_FLG_LIT) { pIter->pv = pIter->pIdx->idx + pIter->pIdx->nCol; } else if (pRow->flag & KV_FLG_MID) { - pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 1); + pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 1); // * sizeof(uint16_t) } else { - pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 2); + pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 2); // * sizeof(uint32_t) } } else { switch (pRow->flag) { @@ -608,7 +750,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { goto _exit; } - if (pIter->pRow->flag & 0xf0) { // KV + if (pIter->pRow->flag >> 4) { // KV if (pIter->iCol < pIter->pIdx->nCol) { uint8_t *pData; @@ -656,7 +798,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { goto _exit; } } else { // Tuple - uint8_t bv = ROW_BIT_VALUE; + uint8_t bv = BIT_FLG_VALUE; if (pIter->pb) { switch (pIter->pRow->flag) { case (HAS_NULL | HAS_NONE): @@ -677,10 +819,10 @@ SColVal *tRowIterNext(SRowIter *pIter) { break; } - if (bv == ROW_BIT_NONE) { + if (bv == BIT_FLG_NONE) { pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); goto _exit; - } else if (bv == ROW_BIT_NULL) { + } else if (bv == BIT_FLG_NULL) { pIter->cv = COL_VAL_NULL(pTColumn->colId, pTColumn->type); goto _exit; } @@ -698,7 +840,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { pIter->cv.value.pData = NULL; } } else { - memcpy(&pIter->cv.value.val, pIter->pv + pTColumn->offset, pTColumn->bytes); + memcpy(&pIter->cv.value.val, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]); } goto _exit; } @@ -708,7 +850,266 @@ _exit: return &pIter->cv; } -// STSchema ======================================== +static int32_t tRowAppendNoneToColData(SColData *aColData, int32_t nColData) { + int32_t code = 0; + + for (int32_t iColData = 0; iColData < nColData; iColData++) { + SColData *pColData = &aColData[iColData]; + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + } + +_exit: + return code; +} +static int32_t tRowAppendNullToColData(SColData *aColData, int32_t nColData, STSchema *pSchema) { + int32_t code = 0; + + int32_t iColData = 0; + SColData *pColData = &aColData[iColData]; + int32_t iTColumn = 1; + STColumn *pTColumn = &pSchema->columns[iTColumn]; + + while (pColData) { + if (pTColumn) { + if (pTColumn->colId == pColData->cid) { // NULL + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + pTColumn = (++iTColumn < pSchema->numOfCols) ? &pSchema->columns[iTColumn] : NULL; + } else if (pTColumn->colId > pColData->cid) { // NONE + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } else { + pTColumn = (++iTColumn < pSchema->numOfCols) ? &pSchema->columns[iTColumn] : NULL; + } + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } + } + +_exit: + return code; +} +static int32_t tRowAppendTupleToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData) { + int32_t code = 0; + + int32_t iColData = 0; + SColData *pColData = &aColData[iColData]; + int32_t iTColumn = 1; + STColumn *pTColumn = &pTSchema->columns[iTColumn]; + + uint8_t *pb = NULL, *pf = NULL, *pv = NULL; + + switch (pRow->flag) { + case HAS_VALUE: + pf = pRow->data; + pv = pf + pTSchema->flen; + break; + case (HAS_NULL | HAS_NONE): + pb = pRow->data; + break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + pb = pRow->data; + pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + pb = pRow->data; + pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + default: + ASSERT(0); + break; + } + + while (pColData) { + if (pTColumn) { + if (pTColumn->colId == pColData->cid) { + ASSERT(pTColumn->type == pColData->type); + if (pb) { + uint8_t bv; + switch (pRow->flag) { + case (HAS_NULL | HAS_NONE): + bv = GET_BIT1(pb, iTColumn - 1); + break; + case (HAS_VALUE | HAS_NONE): + bv = GET_BIT1(pb, iTColumn - 1); + if (bv) bv++; + break; + case (HAS_VALUE | HAS_NULL): + bv = GET_BIT1(pb, iTColumn - 1) + 1; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + bv = GET_BIT2(pb, iTColumn - 1); + break; + default: + ASSERT(0); + break; + } + + if (bv == BIT_FLG_NONE) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + goto _continue; + } else if (bv == BIT_FLG_NULL) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + goto _continue; + } + } + + if (IS_VAR_DATA_TYPE(pColData->type)) { + uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset); + uint32_t nData; + pData += tGetU32v(pData, &nData); + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pf + pTColumn->offset, + TYPE_BYTES[pColData->type]); + if (code) goto _exit; + } + + _continue: + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + } else if (pTColumn->colId > pColData->cid) { // NONE + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } else { + pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + } + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } + } + +_exit: + return code; +} +static int32_t tRowAppendKVToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData) { + int32_t code = 0; + + SKVIdx *pKVIdx = (SKVIdx *)pRow->data; + uint8_t *pv = NULL; + int32_t iColData = 0; + SColData *pColData = &aColData[iColData]; + int32_t iTColumn = 1; + STColumn *pTColumn = &pTSchema->columns[iTColumn]; + int32_t iCol = 0; + + if (pRow->flag & KV_FLG_LIT) { + pv = pKVIdx->idx + pKVIdx->nCol; + } else if (pRow->flag & KV_FLG_MID) { + pv = pKVIdx->idx + (pKVIdx->nCol << 1); + } else if (pRow->flag & KV_FLG_BIG) { + pv = pKVIdx->idx + (pKVIdx->nCol << 2); + } else { + ASSERT(0); + } + + while (pColData) { + if (pTColumn) { + if (pTColumn->colId == pColData->cid) { + while (iCol < pKVIdx->nCol) { + uint8_t *pData; + if (pRow->flag & KV_FLG_LIT) { + pData = pv + ((uint8_t *)pKVIdx->idx)[iCol]; + } else if (pRow->flag & KV_FLG_MID) { + pData = pv + ((uint16_t *)pKVIdx->idx)[iCol]; + } else if (pRow->flag & KV_FLG_BIG) { + pData = pv + ((uint32_t *)pKVIdx->idx)[iCol]; + } else { + ASSERT(0); + } + + int16_t cid; + pData += tGetI16v(pData, &cid); + + if (TABS(cid) == pTColumn->colId) { + if (cid < 0) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } else { + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + uint32_t nData; + pData += tGetU32v(pData, &nData); + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, 0); + if (code) goto _exit; + } + } + iCol++; + goto _continue; + } else if (TABS(cid) > pTColumn->colId) { // NONE + break; + } else { + iCol++; + } + } + + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + + _continue: + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + } else if (pTColumn->colId > pColData->cid) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } else { + pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + } + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } + } + +_exit: + return code; +} +int32_t tRowAppendToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData) { + ASSERT(pRow->sver == pTSchema->version); + ASSERT(nColData > 0); + + int32_t code = 0; + + if (pRow->flag == HAS_NONE) { + code = tRowAppendNoneToColData(aColData, nColData); + goto _exit; + } else if (pRow->flag == HAS_NULL) { + code = tRowAppendNullToColData(aColData, nColData, pTSchema); + goto _exit; + } + + if (pRow->flag >> 4) { // KV row + code = tRowAppendKVToColData(pRow, pTSchema, aColData, nColData); + if (code) goto _exit; + } else { + code = tRowAppendTupleToColData(pRow, pTSchema, aColData, nColData); + if (code) goto _exit; + } + +_exit: + return code; +} // STag ======================================== static int tTagValCmprFn(const void *p1, const void *p2) { @@ -1086,120 +1487,47 @@ void tTagSetCid(const STag *pTag, int16_t iTag, int16_t cid) { tPutI16v(p + offset, cid); } -#if 1 // =================================================================================================================== -int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) { - if (pBuilder == NULL) return -1; +// STSchema ======================================== +STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version) { + STSchema *pTSchema = taosMemoryCalloc(1, sizeof(STSchema) + sizeof(STColumn) * numOfCols); + if (pTSchema == NULL) return NULL; - pBuilder->tCols = 256; - pBuilder->columns = (STColumn *)taosMemoryMalloc(sizeof(STColumn) * pBuilder->tCols); - if (pBuilder->columns == NULL) return -1; + pTSchema->numOfCols = numOfCols; + pTSchema->version = version; - tdResetTSchemaBuilder(pBuilder, version); - return 0; -} + // timestamp column + ASSERT(aSchema[0].type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(aSchema[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID); + pTSchema->columns[0].colId = aSchema[0].colId; + pTSchema->columns[0].type = aSchema[0].type; + pTSchema->columns[0].flags = aSchema[0].flags; + pTSchema->columns[0].bytes = TYPE_BYTES[aSchema[0].type]; + pTSchema->columns[0].offset = -1; -void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder) { - if (pBuilder) { - taosMemoryFreeClear(pBuilder->columns); - } -} + // other columns + for (int32_t iCol = 1; iCol < numOfCols; iCol++) { + SSchema *pSchema = &aSchema[iCol]; + STColumn *pTColumn = &pTSchema->columns[iCol]; -void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) { - pBuilder->nCols = 0; - pBuilder->tlen = 0; - pBuilder->flen = 0; - pBuilder->version = version; -} + pTColumn->colId = pSchema->colId; + pTColumn->type = pSchema->type; + pTColumn->flags = pSchema->flags; + pTColumn->offset = pTSchema->flen; -int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes) { - if (!isValidDataType(type)) return -1; + if (IS_VAR_DATA_TYPE(pSchema->type)) { + pTColumn->bytes = pSchema->bytes; + pTSchema->tlen += (TYPE_BYTES[pSchema->type] + pSchema->bytes); // todo: remove + } else { + pTColumn->bytes = TYPE_BYTES[pSchema->type]; + pTSchema->tlen += TYPE_BYTES[pSchema->type]; // todo: remove + } - if (pBuilder->nCols >= pBuilder->tCols) { - pBuilder->tCols *= 2; - STColumn *columns = (STColumn *)taosMemoryRealloc(pBuilder->columns, sizeof(STColumn) * pBuilder->tCols); - if (columns == NULL) return -1; - pBuilder->columns = columns; + pTSchema->flen += TYPE_BYTES[pTColumn->type]; } - STColumn *pCol = &(pBuilder->columns[pBuilder->nCols]); - pCol->type = type; - pCol->colId = colId; - pCol->flags = flags; - if (pBuilder->nCols == 0) { - pCol->offset = -1; - } else { - pCol->offset = pBuilder->flen; - pBuilder->flen += TYPE_BYTES[type]; - } - - if (IS_VAR_DATA_TYPE(type)) { - pCol->bytes = bytes; - pBuilder->tlen += (TYPE_BYTES[type] + bytes); - } else { - pCol->bytes = TYPE_BYTES[type]; - pBuilder->tlen += TYPE_BYTES[type]; - } - - pBuilder->nCols++; - - ASSERT(pCol->offset < pBuilder->flen); - - return 0; -} - -STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { - if (pBuilder->nCols <= 0) return NULL; - - int tlen = sizeof(STSchema) + sizeof(STColumn) * pBuilder->nCols; - - STSchema *pSchema = (STSchema *)taosMemoryMalloc(tlen); - if (pSchema == NULL) return NULL; - - pSchema->version = pBuilder->version; - pSchema->numOfCols = pBuilder->nCols; - pSchema->tlen = pBuilder->tlen; - pSchema->flen = pBuilder->flen; - -#ifdef TD_SUPPORT_BITMAP - pSchema->tlen += (int)TD_BITMAP_BYTES(pSchema->numOfCols); -#endif - - memcpy(&pSchema->columns[0], pBuilder->columns, sizeof(STColumn) * pBuilder->nCols); - - return pSchema; -} - -#endif - -STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version) { - STSchema *pTSchema = taosMemoryCalloc(1, sizeof(STSchema) + sizeof(STColumn) * numOfCols); - if (pTSchema == NULL) return NULL; - - pTSchema->numOfCols = numOfCols; - pTSchema->version = version; - - // timestamp column - ASSERT(aSchema[0].type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(aSchema[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID); - pTSchema->columns[0].colId = aSchema[0].colId; - pTSchema->columns[0].type = aSchema[0].type; - pTSchema->columns[0].flags = aSchema[0].flags; - pTSchema->columns[0].bytes = aSchema[0].bytes; - pTSchema->columns[0].offset = -1; - - // other columns - for (int32_t iCol = 1; iCol < numOfCols; iCol++) { - SSchema *pSchema = &aSchema[iCol]; - STColumn *pTColumn = &pTSchema->columns[iCol]; - - pTColumn->colId = pSchema->colId; - pTColumn->type = pSchema->type; - pTColumn->flags = pSchema->flags; - pTColumn->bytes = pSchema->bytes; - pTColumn->offset = pTSchema->flen; - - pTSchema->flen += TYPE_BYTES[pTColumn->type]; - } +#if 1 // todo : remove this + pTSchema->tlen += (int32_t)TD_BITMAP_BYTES(numOfCols); +#endif return pTSchema; } @@ -1213,7 +1541,7 @@ void tColDataDestroy(void *ph) { SColData *pColData = (SColData *)ph; tFree(pColData->pBitMap); - tFree((uint8_t *)pColData->aOffset); + tFree(pColData->aOffset); tFree(pColData->pData); } @@ -1230,25 +1558,37 @@ void tColDataClear(SColData *pColData) { pColData->nData = 0; } -static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVal) { +void tColDataDeepClear(SColData *pColData) { + pColData->pBitMap = NULL; + pColData->aOffset = NULL; + pColData->pData = NULL; + + tColDataClear(pColData); +} + +static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; if (IS_VAR_DATA_TYPE(pColData->type)) { - code = tRealloc((uint8_t **)(&pColData->aOffset), sizeof(int32_t) * (pColData->nVal + 1)); + code = tRealloc((uint8_t **)(&pColData->aOffset), (pColData->nVal + 1) << 2); if (code) goto _exit; pColData->aOffset[pColData->nVal] = pColData->nData; - if (pColVal->value.nData) { - code = tRealloc(&pColData->pData, pColData->nData + pColVal->value.nData); + if (nData) { + code = tRealloc(&pColData->pData, pColData->nData + nData); if (code) goto _exit; - memcpy(pColData->pData + pColData->nData, pColVal->value.pData, pColVal->value.nData); - pColData->nData += pColVal->value.nData; + memcpy(pColData->pData + pColData->nData, pData, nData); + pColData->nData += nData; } } else { ASSERT(pColData->nData == tDataTypes[pColData->type].bytes * pColData->nVal); code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes); if (code) goto _exit; - memcpy(pColData->pData + pColData->nData, &pColVal->value.val, tDataTypes[pColData->type].bytes); + if (pData) { + memcpy(pColData->pData + pColData->nData, pData, TYPE_BYTES[pColData->type]); + } else { + memset(pColData->pData + pColData->nData, 0, TYPE_BYTES[pColData->type]); + } pColData->nData += tDataTypes[pColData->type].bytes; } pColData->nVal++; @@ -1256,21 +1596,21 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVa _exit: return code; } -static FORCE_INLINE int32_t tColDataAppendValue00(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue00(SColData *pColData, uint8_t *pData, uint32_t nData) { pColData->flag = HAS_VALUE; - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue01(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue01(SColData *pColData, uint8_t *pData, uint32_t nData) { pColData->flag = HAS_NONE; pColData->nVal++; return 0; } -static FORCE_INLINE int32_t tColDataAppendValue02(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue02(SColData *pColData, uint8_t *pData, uint32_t nData) { pColData->flag = HAS_NULL; pColData->nVal++; return 0; } -static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; int32_t nBit = BIT1_SIZE(pColData->nVal + 1); @@ -1278,7 +1618,7 @@ static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 0, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); pColData->flag |= HAS_VALUE; @@ -1296,13 +1636,13 @@ static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *p } } - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue11(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue11(SColData *pColData, uint8_t *pData, uint32_t nData) { pColData->nVal++; return 0; } -static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; int32_t nBit = BIT1_SIZE(pColData->nVal + 1); @@ -1310,14 +1650,14 @@ static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 0, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); pColData->flag |= HAS_NULL; pColData->nVal++; return code; } -static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; int32_t nBit = BIT1_SIZE(pColData->nVal + 1); @@ -1325,7 +1665,7 @@ static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 0, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); pColData->flag |= HAS_VALUE; @@ -1343,9 +1683,9 @@ static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, SColVal *p } } - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; int32_t nBit = BIT1_SIZE(pColData->nVal + 1); @@ -1353,18 +1693,18 @@ static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 255, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); pColData->flag |= HAS_NONE; pColData->nVal++; return code; } -static FORCE_INLINE int32_t tColDataAppendValue22(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue22(SColData *pColData, uint8_t *pData, uint32_t nData) { pColData->nVal++; return 0; } -static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; pColData->flag |= HAS_VALUE; @@ -1374,9 +1714,9 @@ static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, SColVal *p if (code) return code; for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal)); + SET_BIT2_EX(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal)); } - SET_BIT2(pBitMap, pColData->nVal, 2); + SET_BIT2_EX(pBitMap, pColData->nVal, 2); tFree(pColData->pBitMap); pColData->pBitMap = pBitMap; @@ -1395,32 +1735,32 @@ static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, SColVal *p } } - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue31(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue31(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); pColData->nVal++; return code; } -static FORCE_INLINE int32_t tColDataAppendValue32(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue32(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); pColData->nVal++; return code; } #define tColDataAppendValue40 tColDataPutValue -static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; pColData->flag |= HAS_NONE; @@ -1430,11 +1770,11 @@ static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 255, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; pColData->flag |= HAS_NULL; @@ -1444,31 +1784,31 @@ static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 255, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; pColData->flag |= HAS_NULL; @@ -1478,25 +1818,25 @@ static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, SColVal *p if (code) return code; for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0); + SET_BIT2_EX(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0); } - SET_BIT2(pBitMap, pColData->nVal, 1); + SET_BIT2_EX(pBitMap, pColData->nVal, 1); tFree(pColData->pBitMap); pColData->pBitMap = pBitMap; - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; pColData->flag |= HAS_NONE; @@ -1506,52 +1846,52 @@ static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, SColVal *p if (code) return code; for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1); + SET_BIT2_EX(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1); } - SET_BIT2(pBitMap, pColData->nVal, 0); + SET_BIT2_EX(pBitMap, pColData->nVal, 0); tFree(pColData->pBitMap); pColData->pBitMap = pBitMap; - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT2(pColData->pBitMap, pColData->nVal, 2); + SET_BIT2_EX(pColData->pBitMap, pColData->nVal, 2); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT2(pColData->pBitMap, pColData->nVal, 0); + SET_BIT2_EX(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT2(pColData->pBitMap, pColData->nVal, 1); + SET_BIT2_EX(pColData->pBitMap, pColData->nVal, 1); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, SColVal *pColVal) = { +static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData) = { {tColDataAppendValue00, tColDataAppendValue01, tColDataAppendValue02}, // 0 {tColDataAppendValue10, tColDataAppendValue11, tColDataAppendValue12}, // HAS_NONE {tColDataAppendValue20, tColDataAppendValue21, tColDataAppendValue22}, // HAS_NULL @@ -1563,7 +1903,9 @@ static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, SColVal *pCo }; int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) { ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type); - return tColDataAppendValueImpl[pColData->flag][pColVal->flag](pColData, pColVal); + return tColDataAppendValueImpl[pColData->flag][pColVal->flag]( + pColData, IS_VAR_DATA_TYPE(pColData->type) ? pColVal->value.pData : (uint8_t *)&pColVal->value.val, + pColVal->value.nData); } static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NONE @@ -1572,7 +1914,8 @@ static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SCo static FORCE_INLINE void tColDataGetValue2(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NULL *pColVal = COL_VAL_NULL(pColData->cid, pColData->type); } -static FORCE_INLINE void tColDataGetValue3(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NULL|HAS_NONE +static FORCE_INLINE void tColDataGetValue3(SColData *pColData, int32_t iVal, + SColVal *pColVal) { // HAS_NULL|HAS_NONE switch (GET_BIT1(pColData->pBitMap, iVal)) { case 0: *pColVal = COL_VAL_NONE(pColData->cid, pColData->type); @@ -1687,46 +2030,776 @@ uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal) { return v; } -int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { +int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMalloc, void *arg) { int32_t code = 0; - int32_t size; - - ASSERT(pColDataSrc->nVal > 0); - ASSERT(pColDataDest->cid == pColDataSrc->cid); - ASSERT(pColDataDest->type == pColDataSrc->type); - pColDataDest->smaOn = pColDataSrc->smaOn; - pColDataDest->nVal = pColDataSrc->nVal; - pColDataDest->flag = pColDataSrc->flag; + *pColData = *pColDataFrom; // bitmap - if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) { - size = BIT2_SIZE(pColDataSrc->nVal); - code = tRealloc(&pColDataDest->pBitMap, size); - if (code) goto _exit; - memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size); + switch (pColData->flag) { + case (HAS_NULL | HAS_NONE): + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + pColData->pBitMap = xMalloc(arg, BIT1_SIZE(pColData->nVal)); + if (pColData->pBitMap == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT1_SIZE(pColData->nVal)); + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + pColData->pBitMap = xMalloc(arg, BIT2_SIZE(pColData->nVal)); + if (pColData->pBitMap == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT2_SIZE(pColData->nVal)); + break; + default: + pColData->pBitMap = NULL; + break; } // offset - if (IS_VAR_DATA_TYPE(pColDataDest->type)) { - size = sizeof(int32_t) * pColDataSrc->nVal; + if (IS_VAR_DATA_TYPE(pColData->type) && (pColData->flag & HAS_VALUE)) { + pColData->aOffset = xMalloc(arg, pColData->nVal << 2); + if (pColData->aOffset == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(pColData->aOffset, pColDataFrom->aOffset, pColData->nVal << 2); + } else { + pColData->aOffset = NULL; + } - code = tRealloc((uint8_t **)&pColDataDest->aOffset, size); - if (code) goto _exit; + // value + if (pColData->nData) { + pColData->pData = xMalloc(arg, pColData->nData); + if (pColData->pData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } - memcpy(pColDataDest->aOffset, pColDataSrc->aOffset, size); + memcpy(pColData->pData, pColDataFrom->pData, pColData->nData); + } else { + pColData->pData = NULL; } - // value - pColDataDest->nData = pColDataSrc->nData; - code = tRealloc(&pColDataDest->pData, pColDataSrc->nData); - if (code) goto _exit; - memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData); +_exit: + return code; +} + +int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t bytes, int32_t nRows, char *lengthOrbitmap, + char *data) { + int32_t code = 0; + + if (IS_VAR_DATA_TYPE(type)) { // var-length data type + for (int32_t i = 0; i < nRows; ++i) { + int32_t offset = *((int32_t *)lengthOrbitmap + i); + if (offset == -1) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, (uint8_t *)varDataVal(data + offset), + varDataLen(data + offset)); + } + } + } else { // fixed-length data type + bool allValue = true; + bool allNull = true; + for (int32_t i = 0; i < nRows; ++i) { + if (!colDataIsNull_f(lengthOrbitmap, i)) { + allNull = false; + } else { + allValue = false; + } + } + + if (allValue) { + // optimize (todo) + for (int32_t i = 0; i < nRows; ++i) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, (uint8_t *)data + bytes * i, bytes); + } + } else if (allNull) { + // optimize (todo) + for (int32_t i = 0; i < nRows; ++i) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } + } else { + for (int32_t i = 0; i < nRows; ++i) { + if (colDataIsNull_f(lengthOrbitmap, i)) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, (uint8_t *)data + bytes * i, bytes); + } + } + } + } _exit: return code; } +int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) { + int32_t code = 0; + + ASSERT(pColData->type == pBind->buffer_type); + + if (IS_VAR_DATA_TYPE(pBind->buffer_type)) { // var-length data type + for (int32_t i = 0; i < pBind->num; ++i) { + if (pBind->is_null && pBind->is_null[i]) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE]( + pColData, (uint8_t *)pBind->buffer + pBind->buffer_length * i, pBind->length[i]); + } + } + } else { // fixed-length data type + bool allValue; + bool allNull; + if (pBind->is_null) { + bool same = (memcmp(pBind->is_null, pBind->is_null + 1, pBind->num - 1) == 0); + allNull = (same && pBind->is_null[0] != 0); + allValue = (same && pBind->is_null[0] == 0); + } else { + allNull = false; + allValue = true; + } + + if (allValue) { + // optimize (todo) + for (int32_t i = 0; i < pBind->num; ++i) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE]( + pColData, (uint8_t *)pBind->buffer + TYPE_BYTES[pColData->type] * i, pBind->buffer_length); + } + } else if (allNull) { + // optimize (todo) + for (int32_t i = 0; i < pBind->num; ++i) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } + } else { + for (int32_t i = 0; i < pBind->num; ++i) { + if (pBind->is_null[i]) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE]( + pColData, (uint8_t *)pBind->buffer + TYPE_BYTES[pColData->type] * i, pBind->buffer_length); + } + } + } + } + +_exit: + return code; +} + +static int32_t tColDataSwapValue(SColData *pColData, int32_t i, int32_t j) { + int32_t code = 0; + + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nData1 = pColData->aOffset[i + 1] - pColData->aOffset[i]; + int32_t nData2 = (j < pColData->nVal - 1) ? pColData->aOffset[j + 1] - pColData->aOffset[j] + : pColData->nData - pColData->aOffset[j]; + uint8_t *pData = taosMemoryMalloc(TMAX(nData1, nData2)); + if (pData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + if (nData1 > nData2) { + memcpy(pData, pColData->pData + pColData->aOffset[i], nData1); + memcpy(pColData->pData + pColData->aOffset[i], pColData->pData + pColData->aOffset[j], nData2); + // memmove(pColData->pData + pColData->aOffset[i] + nData2, pColData->pData + pColData->aOffset[i] + nData1, + // pColData->aOffset[j] - pColData->aOffset[i + 1]); + memmove(pColData->pData + pColData->aOffset[i] + nData2, pColData->pData + pColData->aOffset[i + 1], + pColData->aOffset[j] - pColData->aOffset[i + 1]); + memcpy(pColData->pData + pColData->aOffset[j] + nData2 - nData1, pData, nData1); + } else { + memcpy(pData, pColData->pData + pColData->aOffset[j], nData2); + memcpy(pColData->pData + pColData->aOffset[j] + nData2 - nData1, pColData->pData + pColData->aOffset[i], nData1); + // memmove(pColData->pData + pColData->aOffset[j] + nData2 - nData1, pColData->pData + pColData->aOffset[i] + + // nData1, + // pColData->aOffset[j] - pColData->aOffset[i + 1]); + memmove(pColData->pData + pColData->aOffset[i] + nData2, pColData->pData + pColData->aOffset[i + 1], + pColData->aOffset[j] - pColData->aOffset[i + 1]); + memcpy(pColData->pData + pColData->aOffset[i], pData, nData2); + } + for (int32_t k = i + 1; k <= j; ++k) { + pColData->aOffset[k] = pColData->aOffset[k] + nData2 - nData1; + } + + taosMemoryFree(pData); + } else { + uint64_t val; + memcpy(&val, &pColData->pData[TYPE_BYTES[pColData->type] * i], TYPE_BYTES[pColData->type]); + memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * i], &pColData->pData[TYPE_BYTES[pColData->type] * j], + TYPE_BYTES[pColData->type]); + memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * j], &val, TYPE_BYTES[pColData->type]); + } + +_exit: + return code; +} + +static void tColDataSwap(SColData *pColData, int32_t i, int32_t j) { + ASSERT(i < j); + ASSERT(j < pColData->nVal); + + switch (pColData->flag) { + case HAS_NONE: + case HAS_NULL: + break; + case (HAS_NULL | HAS_NONE): { + uint8_t bv = GET_BIT1(pColData->pBitMap, i); + SET_BIT1(pColData->pBitMap, i, GET_BIT1(pColData->pBitMap, j)); + SET_BIT1(pColData->pBitMap, j, bv); + } break; + case HAS_VALUE: { + tColDataSwapValue(pColData, i, j); + } break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): { + uint8_t bv = GET_BIT1(pColData->pBitMap, i); + SET_BIT1(pColData->pBitMap, i, GET_BIT1(pColData->pBitMap, j)); + SET_BIT1(pColData->pBitMap, j, bv); + tColDataSwapValue(pColData, i, j); + } break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): { + uint8_t bv = GET_BIT2(pColData->pBitMap, i); + SET_BIT2(pColData->pBitMap, i, GET_BIT2(pColData->pBitMap, j)); + SET_BIT2(pColData->pBitMap, j, bv); + tColDataSwapValue(pColData, i, j); + } break; + default: + ASSERT(0); + break; + } +} + +static int32_t tColDataCopyRowCell(SColData *pFromColData, int32_t iFromRow, SColData *pToColData, int32_t iToRow) { + int32_t code = TSDB_CODE_SUCCESS; + + if (IS_VAR_DATA_TYPE(pToColData->type)) { + int32_t nData = (iFromRow < pFromColData->nVal - 1) + ? pFromColData->aOffset[iFromRow + 1] - pFromColData->aOffset[iFromRow] + : pFromColData->nData - pFromColData->aOffset[iFromRow]; + if (iToRow == 0) { + pToColData->aOffset[iToRow] = 0; + } + + if (iToRow < pToColData->nVal - 1) { + pToColData->aOffset[iToRow + 1] = pToColData->aOffset[iToRow] + nData; + } + + memcpy(pToColData->pData + pToColData->aOffset[iToRow], pFromColData->pData + pFromColData->aOffset[iFromRow], + nData); + } else { + memcpy(&pToColData->pData[TYPE_BYTES[pToColData->type] * iToRow], + &pFromColData->pData[TYPE_BYTES[pToColData->type] * iFromRow], TYPE_BYTES[pToColData->type]); + } + return code; +} + +static int32_t tColDataCopyRowSingleCol(SColData *pFromColData, int32_t iFromRow, SColData *pToColData, + int32_t iToRow) { + int32_t code = TSDB_CODE_SUCCESS; + + switch (pFromColData->flag) { + case HAS_NONE: + case HAS_NULL: + break; + case (HAS_NULL | HAS_NONE): { + SET_BIT1(pToColData->pBitMap, iToRow, GET_BIT1(pFromColData->pBitMap, iFromRow)); + } break; + case HAS_VALUE: { + tColDataCopyRowCell(pFromColData, iFromRow, pToColData, iToRow); + } break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): { + SET_BIT1(pToColData->pBitMap, iToRow, GET_BIT1(pFromColData->pBitMap, iFromRow)); + tColDataCopyRowCell(pFromColData, iFromRow, pToColData, iToRow); + } break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): { + SET_BIT2(pToColData->pBitMap, iToRow, GET_BIT2(pFromColData->pBitMap, iFromRow)); + tColDataCopyRowCell(pFromColData, iFromRow, pToColData, iToRow); + } break; + default: + return -1; + } + + return code; +} + +static int32_t tColDataCopyRow(SColData *aFromColData, int32_t iFromRow, SColData *aToColData, int32_t iToRow, + int32_t nColData) { + int32_t code = TSDB_CODE_SUCCESS; + + for (int32_t i = 0; i < nColData; i++) { + code = tColDataCopyRowSingleCol(&aFromColData[i], iFromRow, &aToColData[i], iToRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + return code; +} + +static int32_t tColDataCopyRowAppend(SColData *aFromColData, int32_t iFromRow, SColData *aToColData, int32_t nColData) { + int32_t code = TSDB_CODE_SUCCESS; + + for (int32_t i = 0; i < nColData; i++) { + SColVal cv = {0}; + tColDataGetValue(&aFromColData[i], iFromRow, &cv); + code = tColDataAppendValue(&aToColData[i], &cv); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + return code; +} + +static int32_t tColDataMergeSortMerge(SColData *aColData, int32_t start, int32_t mid, int32_t end, int32_t nColData) { + SColData *aDstColData = NULL; + TSKEY *aKey = (TSKEY *)aColData[0].pData; + + int32_t i = start, j = mid + 1, k = 0; + + if (end > start) { + aDstColData = taosMemoryCalloc(1, sizeof(SColData) * nColData); + for (int c = 0; c < nColData; ++c) { + tColDataInit(&aDstColData[c], aColData[c].cid, aColData[c].type, aColData[c].smaOn); + } + if (aDstColData == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + /* + for (int32_t i = 0; i < nColData; i++) { + tColDataCopy(&aColData[i], &aDstColData[i], tColDataDefaultMalloc, NULL); + } + */ + } + + while (i <= mid && j <= end) { + if (aKey[i] <= aKey[j]) { + // tColDataCopyRow(aColData, i++, aDstColData, k++); + tColDataCopyRowAppend(aColData, i++, aDstColData, nColData); + } else { + // tColDataCopyRow(aColData, j++, aDstColData, k++); + tColDataCopyRowAppend(aColData, j++, aDstColData, nColData); + } + } + + while (i <= mid) { + // tColDataCopyRow(aColData, i++, aDstColData, k++); + tColDataCopyRowAppend(aColData, i++, aDstColData, nColData); + } + + while (j <= end) { + // tColDataCopyRow(aColData, j++, aDstColData, k++); + tColDataCopyRowAppend(aColData, j++, aDstColData, nColData); + } + + for (i = start, k = 0; i <= end; ++i, ++k) { + tColDataCopyRow(aDstColData, k, aColData, i, nColData); + } + + if (aDstColData) { + for (int32_t i = 0; i < nColData; i++) { + tColDataDestroy(&aDstColData[i]); + } + taosMemoryFree(aDstColData); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t tColDataMergeSort(SColData *aColData, int32_t start, int32_t end, int32_t nColData) { + int32_t ret = TSDB_CODE_SUCCESS; + int32_t mid; + + if (start >= end) { + return TSDB_CODE_SUCCESS; + } + + mid = (start + end) / 2; + + ret = tColDataMergeSort(aColData, start, mid, nColData); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + ret = tColDataMergeSort(aColData, mid + 1, end, nColData); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + return tColDataMergeSortMerge(aColData, start, mid, end, nColData); +} + +static int32_t tColDataSort(SColData *aColData, int32_t nColData) { + int32_t nVal = aColData[0].nVal; + + if (nVal < 2) return TSDB_CODE_SUCCESS; + + return tColDataMergeSort(aColData, 0, nVal - 1, nColData); +} +static void tColDataMergeImpl(SColData *pColData, int32_t iStart, int32_t iEnd /* not included */) { + switch (pColData->flag) { + case HAS_NONE: + case HAS_NULL: { + pColData->nVal -= (iEnd - iStart - 1); + } break; + case (HAS_NULL | HAS_NONE): { + if (GET_BIT1(pColData->pBitMap, iStart) == 0) { + for (int32_t i = iStart + 1; i < iEnd; ++i) { + if (GET_BIT1(pColData->pBitMap, i) == 1) { + SET_BIT1(pColData->pBitMap, iStart, 1); + break; + } + } + } + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + SET_BIT1(pColData->pBitMap, j, GET_BIT1(pColData->pBitMap, i)); + } + + pColData->nVal -= (iEnd - iStart - 1); + + uint8_t flag = 0; + for (int32_t i = 0; i < pColData->nVal; ++i) { + uint8_t bv = GET_BIT1(pColData->pBitMap, i); + if (bv == BIT_FLG_NONE) { + flag |= HAS_NONE; + } else if (bv == BIT_FLG_NULL) { + flag |= HAS_NULL; + } else { + ASSERT(0); + } + + if (flag == pColData->flag) break; + } + pColData->flag = flag; + } break; + case HAS_VALUE: { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nDiff = pColData->aOffset[iEnd - 1] - pColData->aOffset[iStart]; + + memmove(pColData->pData + pColData->aOffset[iStart], pColData->pData + pColData->aOffset[iEnd - 1], + pColData->nData - pColData->aOffset[iEnd - 1]); + pColData->nData -= nDiff; + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + pColData->aOffset[j] = pColData->aOffset[i] - nDiff; + } + } else { + memmove(pColData->pData + TYPE_BYTES[pColData->type] * iStart, + pColData->pData + TYPE_BYTES[pColData->type] * (iEnd - 1), + TYPE_BYTES[pColData->type] * (pColData->nVal - iEnd + 1)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + pColData->nVal -= (iEnd - iStart - 1); + } break; + case (HAS_VALUE | HAS_NONE): { + uint8_t bv; + int32_t iv; + for (int32_t i = iEnd - 1; i >= iStart; --i) { + bv = GET_BIT1(pColData->pBitMap, i); + if (bv) { + iv = i; + break; + } + } + + if (bv) { // has a value + if (IS_VAR_DATA_TYPE(pColData->type)) { + if (iv != iStart) { + memmove(&pColData->pData[pColData->aOffset[iStart]], &pColData->pData[pColData->aOffset[iv]], + iv < (pColData->nVal - 1) ? pColData->aOffset[iv + 1] - pColData->aOffset[iv] + : pColData->nData - pColData->aOffset[iv]); + } + // TODO + ASSERT(0); + } else { + if (iv != iStart) { + memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * iStart], + &pColData->pData[TYPE_BYTES[pColData->type] * iv], TYPE_BYTES[pColData->type]); + } + memmove(&pColData->pData[TYPE_BYTES[pColData->type] * (iStart + 1)], + &pColData->pData[TYPE_BYTES[pColData->type] * iEnd], + TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + SET_BIT1(pColData->pBitMap, iStart, 1); + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + SET_BIT1(pColData->pBitMap, j, GET_BIT1(pColData->pBitMap, i)); + } + + uint8_t flag = HAS_VALUE; + for (int32_t i = 0; i < pColData->nVal - (iEnd - iStart - 1); ++i) { + if (GET_BIT1(pColData->pBitMap, i) == 0) { + flag |= HAS_NONE; + } + + if (flag == pColData->flag) break; + } + pColData->flag = flag; + } else { // all NONE + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nDiff = pColData->aOffset[iEnd - 1] - pColData->aOffset[iStart]; + + memmove(&pColData->pData[pColData->aOffset[iStart]], &pColData->pData[pColData->aOffset[iEnd - 1]], + pColData->nData - pColData->aOffset[iEnd - 1]); + pColData->nData -= nDiff; + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + pColData->aOffset[j] = pColData->aOffset[i] - nDiff; + } + } else { + memmove(pColData->pData + TYPE_BYTES[pColData->type] * (iStart + 1), + pColData->pData + TYPE_BYTES[pColData->type] * iEnd, + TYPE_BYTES[pColData->type] * (pColData->nVal - iEnd + 1)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + SET_BIT1(pColData->pBitMap, j, GET_BIT1(pColData->pBitMap, i)); + } + } + pColData->nVal -= (iEnd - iStart - 1); + } break; + case (HAS_VALUE | HAS_NULL): { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nDiff = pColData->aOffset[iEnd - 1] - pColData->aOffset[iStart]; + + memmove(pColData->pData + pColData->aOffset[iStart], pColData->pData + pColData->aOffset[iEnd - 1], + pColData->nData - pColData->aOffset[iEnd - 1]); + pColData->nData -= nDiff; + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + pColData->aOffset[j] = pColData->aOffset[i] - nDiff; + } + } else { + memmove(pColData->pData + TYPE_BYTES[pColData->type] * iStart, + pColData->pData + TYPE_BYTES[pColData->type] * (iEnd - 1), + TYPE_BYTES[pColData->type] * (pColData->nVal - iEnd + 1)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + for (int32_t i = iEnd - 1, j = iStart; i < pColData->nVal; ++i, ++j) { + SET_BIT1(pColData->pBitMap, j, GET_BIT1(pColData->pBitMap, i)); + } + + pColData->nVal -= (iEnd - iStart - 1); + + uint8_t flag = 0; + for (int32_t i = 0; i < pColData->nVal; ++i) { + if (GET_BIT1(pColData->pBitMap, i)) { + flag |= HAS_VALUE; + } else { + flag |= HAS_NULL; + } + + if (flag == pColData->flag) break; + } + pColData->flag = flag; + } break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): { + uint8_t bv; + int32_t iv; + for (int32_t i = iEnd - 1; i >= iStart; --i) { + bv = GET_BIT2(pColData->pBitMap, i); + if (bv) { + iv = i; + break; + } + } + + if (bv) { + // TODO + ASSERT(0); + } else { // ALL NONE + if (IS_VAR_DATA_TYPE(pColData->type)) { + // TODO + ASSERT(0); + } else { + memmove(pColData->pData + TYPE_BYTES[pColData->type] * (iStart + 1), + pColData->pData + TYPE_BYTES[pColData->type] * iEnd, + TYPE_BYTES[pColData->type] * (pColData->nVal - iEnd)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + SET_BIT2(pColData->pBitMap, j, GET_BIT2(pColData->pBitMap, i)); + } + } + pColData->nVal -= (iEnd - iStart - 1); + } break; + default: + ASSERT(0); + break; + } +} +static void tColDataMerge(SColData *aColData, int32_t nColData) { + int32_t iStart = 0; + for (;;) { + if (iStart >= aColData[0].nVal - 1) break; + + int32_t iEnd = iStart + 1; + while (iEnd < aColData[0].nVal) { + if (((TSKEY *)aColData[0].pData)[iEnd] != ((TSKEY *)aColData[0].pData)[iStart]) break; + + iEnd++; + } + + if (iEnd - iStart > 1) { + for (int32_t i = 0; i < nColData; i++) { + tColDataMergeImpl(&aColData[i], iStart, iEnd); + } + } + + iStart++; + } +} +void tColDataSortMerge(SArray *colDataArr) { + int32_t nColData = TARRAY_SIZE(colDataArr); + SColData *aColData = (SColData *)TARRAY_DATA(colDataArr); + + if (aColData[0].nVal <= 1) goto _exit; + + ASSERT(aColData[0].type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(aColData[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); + ASSERT(aColData[0].flag == HAS_VALUE); + + int8_t doSort = 0; + int8_t doMerge = 0; + // scan ------- + TSKEY *aKey = (TSKEY *)aColData[0].pData; + for (int32_t iVal = 1; iVal < aColData[0].nVal; ++iVal) { + if (aKey[iVal] > aKey[iVal - 1]) { + continue; + } else if (aKey[iVal] < aKey[iVal - 1]) { + doSort = 1; + break; + } else { + doMerge = 1; + } + } + + // sort ------- + if (doSort) { + tColDataSort(aColData, nColData); + } + + if (doMerge != 1) { + for (int32_t iVal = 1; iVal < aColData[0].nVal; ++iVal) { + if (aKey[iVal] == aKey[iVal - 1]) { + doMerge = 1; + break; + } + } + } + + // merge ------- + if (doMerge) { + tColDataMerge(aColData, nColData); + } + +_exit: + return; +} + +int32_t tPutColData(uint8_t *pBuf, SColData *pColData) { + int32_t n = 0; + + n += tPutI16v(pBuf ? pBuf + n : NULL, pColData->cid); + n += tPutI8(pBuf ? pBuf + n : NULL, pColData->type); + n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nVal); + n += tPutI8(pBuf ? pBuf + n : NULL, pColData->flag); + + // bitmap + switch (pColData->flag) { + case (HAS_NULL | HAS_NONE): + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT1_SIZE(pColData->nVal)); + n += BIT1_SIZE(pColData->nVal); + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT2_SIZE(pColData->nVal)); + n += BIT2_SIZE(pColData->nVal); + break; + default: + break; + } + + // value + if (pColData->flag & HAS_VALUE) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + if (pBuf) memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2); + n += (pColData->nVal << 2); + + n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nData); + if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); + n += pColData->nData; + } else { + if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); + n += pColData->nData; + } + } + + return n; +} + +int32_t tGetColData(uint8_t *pBuf, SColData *pColData) { + int32_t n = 0; + + n += tGetI16v(pBuf + n, &pColData->cid); + n += tGetI8(pBuf + n, &pColData->type); + n += tGetI32v(pBuf + n, &pColData->nVal); + n += tGetI8(pBuf + n, &pColData->flag); + + // bitmap + switch (pColData->flag) { + case (HAS_NULL | HAS_NONE): + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + pColData->pBitMap = pBuf + n; + n += BIT1_SIZE(pColData->nVal); + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + pColData->pBitMap = pBuf + n; + n += BIT2_SIZE(pColData->nVal); + break; + default: + break; + } + + // value + if (pColData->flag & HAS_VALUE) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + pColData->aOffset = (int32_t *)(pBuf + n); + n += (pColData->nVal << 2); + + n += tGetI32v(pBuf + n, &pColData->nData); + pColData->pData = pBuf + n; + n += pColData->nData; + } else { + pColData->pData = pBuf + n; + pColData->nData = TYPE_BYTES[pColData->type] * pColData->nVal; + n += pColData->nData; + } + } + + return n; +} + #define CALC_SUM_MAX_MIN(SUM, MAX, MIN, VAL) \ do { \ (SUM) += (VAL); \ diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 9caf0cc33e3e97243b0fd00b7ccfb7f2c3e81110..037c8a45419353864c4e6b962beb641daf3b35c1 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -47,7 +47,7 @@ int32_t tsNumOfMnodeQueryThreads = 4; int32_t tsNumOfMnodeFetchThreads = 1; int32_t tsNumOfMnodeReadThreads = 1; int32_t tsNumOfVnodeQueryThreads = 4; -int32_t tsNumOfVnodeStreamThreads = 2; +float tsRatioOfVnodeStreamThreads = 1.0; int32_t tsNumOfVnodeFetchThreads = 4; int32_t tsNumOfVnodeRsmaThreads = 2; int32_t tsNumOfQnodeQueryThreads = 4; @@ -60,6 +60,9 @@ int32_t tsElectInterval = 25 * 1000; int32_t tsHeartbeatInterval = 1000; int32_t tsHeartbeatTimeout = 20 * 1000; +// vnode +int64_t tsVndCommitMaxIntervalMs = 60 * 1000; + // monitor bool tsEnableMonitor = true; int32_t tsMonitorInterval = 30; @@ -73,6 +76,11 @@ bool tsEnableTelem = true; int32_t tsTelemInterval = 43200; char tsTelemServer[TSDB_FQDN_LEN] = "telemetry.taosdata.com"; uint16_t tsTelemPort = 80; +char* tsTelemUri = "/report"; + +bool tsEnableCrashReport = true; +char* tsClientCrashReportUri = "/ccrashreport"; +char* tsSvrCrashReportUri = "/dcrashreport"; // schemaless char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null"; @@ -202,7 +210,9 @@ int32_t taosSetTfsCfg(SConfig *pCfg) { int32_t taosSetTfsCfg(SConfig *pCfg); #endif -struct SConfig *taosGetCfg() { return tsCfg; } +struct SConfig *taosGetCfg() { + return tsCfg; +} static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, char *apolloUrl) { @@ -314,6 +324,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "maxMemUsedByInsert", tsMaxMemUsedByInsert, 1, INT32_MAX, true) != 0) return -1; if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, 0) != 0) return -1; if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, true) != 0) return -1; + if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, true) != 0) return -1; tsNumOfTaskQueueThreads = tsNumOfCores / 2; tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 4); @@ -377,7 +388,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, 0) != 0) return -1; tsNumOfRpcThreads = tsNumOfCores / 2; - tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 1, 4); + tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 1, TSDB_MAX_RPC_THREADS); if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, 0) != 0) return -1; tsNumOfCommitThreads = tsNumOfCores / 2; @@ -392,9 +403,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 4); if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 4, 1024, 0) != 0) return -1; - tsNumOfVnodeStreamThreads = tsNumOfCores / 4; - tsNumOfVnodeStreamThreads = TMAX(tsNumOfVnodeStreamThreads, 4); - if (cfgAddInt32(pCfg, "numOfVnodeStreamThreads", tsNumOfVnodeStreamThreads, 4, 1024, 0) != 0) return -1; + if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 100, 0) != 0) return -1; tsNumOfVnodeFetchThreads = tsNumOfCores / 4; tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4); @@ -429,6 +438,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, 0) != 0) return -1; if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, 0) != 0) return -1; + if (cfgAddInt64(pCfg, "vndCommitMaxInterval", tsVndCommitMaxIntervalMs, 1000, 1000 * 60 * 60, 0) != 0) return -1; + if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1; if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, 0) != 0) return -1; if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, 0) != 0) return -1; @@ -436,6 +447,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, 0) != 0) return -1; if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, 0) != 0) return -1; + if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, 0) != 0) return -1; if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, 0) != 0) return -1; if (cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, 0) != 0) return -1; if (cfgAddString(pCfg, "telemetryServer", tsTelemServer, 0) != 0) return -1; @@ -513,11 +525,9 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { pItem->stype = stype; } - pItem = cfgGetItem(tsCfg, "numOfVnodeStreamThreads"); + pItem = cfgGetItem(tsCfg, "ratioOfVnodeStreamThreads"); if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { - tsNumOfVnodeStreamThreads = numOfCores / 4; - tsNumOfVnodeStreamThreads = TMAX(tsNumOfVnodeStreamThreads, 4); - pItem->i32 = tsNumOfVnodeStreamThreads; + pItem->fval = tsRatioOfVnodeStreamThreads; pItem->stype = stype; } @@ -669,6 +679,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsQueryUseNodeAllocator = cfgGetItem(pCfg, "queryUseNodeAllocator")->bval; tsKeepColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; tsUseAdapter = cfgGetItem(pCfg, "useAdapter")->bval; + tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval; tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; return 0; @@ -710,7 +721,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32; tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32; tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32; - tsNumOfVnodeStreamThreads = cfgGetItem(pCfg, "numOfVnodeStreamThreads")->i32; + tsRatioOfVnodeStreamThreads = cfgGetItem(pCfg, "ratioOfVnodeStreamThreads")->fval; tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32; tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32; tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32; @@ -719,6 +730,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsNumOfSnodeWriteThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32; tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64; + tsSIMDBuiltins = (bool)cfgGetItem(pCfg, "SIMD-builtins")->bval; + tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval; tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32; tstrncpy(tsMonitorFqdn, cfgGetItem(pCfg, "monitorFqdn")->str, TSDB_FQDN_LEN); @@ -728,6 +741,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsQueryRspPolicy = cfgGetItem(pCfg, "queryRspPolicy")->i32; tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; + tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval; tsTelemInterval = cfgGetItem(pCfg, "telemetryInterval")->i32; tstrncpy(tsTelemServer, cfgGetItem(pCfg, "telemetryServer")->str, TSDB_FQDN_LEN); tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32; @@ -745,6 +759,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsHeartbeatInterval = cfgGetItem(pCfg, "syncHeartbeatInterval")->i32; tsHeartbeatTimeout = cfgGetItem(pCfg, "syncHeartbeatTimeout")->i32; + tsVndCommitMaxIntervalMs = cfgGetItem(pCfg, "vndCommitMaxInterval")->i64; + tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; tstrncpy(tsUdfdResFuncs, cfgGetItem(pCfg, "udfdResFuncs")->str, sizeof(tsUdfdResFuncs)); tstrncpy(tsUdfdLdLibPath, cfgGetItem(pCfg, "udfdLdLibPath")->str, sizeof(tsUdfdLdLibPath)); @@ -797,6 +813,8 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32; } else if (strcasecmp("cDebugFlag", name) == 0) { cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; + } else if (strcasecmp("crashReporting", name) == 0) { + tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval; } break; } @@ -1316,12 +1334,17 @@ void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal, if (pItem != NULL && (rewrite || pItem->i32 == 0)) { pItem->i32 = flagVal; } - *pFlagPtr = flagVal; + if (pFlagPtr != NULL) { + *pFlagPtr = flagVal; + } } void taosSetAllDebugFlag(int32_t flag, bool rewrite) { if (flag <= 0) return; + taosSetDebugFlag(NULL, "debugFlag", flag, rewrite); + taosSetDebugFlag(NULL, "simDebugFlag", flag, rewrite); + taosSetDebugFlag(NULL, "tmrDebugFlag", flag, rewrite); taosSetDebugFlag(&uDebugFlag, "uDebugFlag", flag, rewrite); taosSetDebugFlag(&rpcDebugFlag, "rpcDebugFlag", flag, rewrite); taosSetDebugFlag(&jniDebugFlag, "jniDebugFlag", flag, rewrite); @@ -1340,6 +1363,5 @@ void taosSetAllDebugFlag(int32_t flag, bool rewrite) { taosSetDebugFlag(&idxDebugFlag, "idxDebugFlag", flag, rewrite); taosSetDebugFlag(&tdbDebugFlag, "tdbDebugFlag", flag, rewrite); taosSetDebugFlag(&metaDebugFlag, "metaDebugFlag", flag, rewrite); - taosSetDebugFlag(&metaDebugFlag, "tmrDebugFlag", flag, rewrite); uInfo("all debug flag are set to %d", flag); } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 95625e8d9360946fbcf1432f804ec7841e006dfc..38655fb027648af69c3ce097850f8de08e3a66a2 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3191,6 +3191,7 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq if (tEncodeI64(&encoder, pReq->showId) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->filterTb) < 0) return -1; if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; tEndEncode(&encoder); @@ -3207,6 +3208,7 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR if (tDecodeI64(&decoder, &pReq->showId) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->filterTb) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; tEndDecode(&decoder); @@ -5424,6 +5426,8 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS if (tEncodeI32(&encoder, pField->bytes) < 0) return -1; if (tEncodeCStr(&encoder, pField->name) < 0) return -1; } + if (tEncodeI8(&encoder, pReq->createStb) < 0) return -1; + if (tEncodeU64(&encoder, pReq->targetStbUid) < 0) return -1; tEndEncode(&encoder); @@ -5484,6 +5488,8 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea } } } + if (tDecodeI8(&decoder, &pReq->createStb) < 0) return -1; + if (tDecodeU64(&decoder, &pReq->targetStbUid) < 0) return -1; tEndDecode(&decoder); @@ -5627,30 +5633,6 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) { return 0; } -STSchema *tdGetSTSChemaFromSSChema(SSchema *pSchema, int32_t nCols, int32_t sver) { - STSchemaBuilder schemaBuilder = {0}; - if (tdInitTSchemaBuilder(&schemaBuilder, sver) < 0) { - return NULL; - } - - for (int i = 0; i < nCols; i++) { - SSchema *schema = pSchema + i; - if (tdAddColToSchema(&schemaBuilder, schema->type, schema->flags, schema->colId, schema->bytes) < 0) { - tdDestroyTSchemaBuilder(&schemaBuilder); - return NULL; - } - } - - STSchema *pNSchema = tdGetSchemaFromBuilder(&schemaBuilder); - if (pNSchema == NULL) { - tdDestroyTSchemaBuilder(&schemaBuilder); - return NULL; - } - - tdDestroyTSchemaBuilder(&schemaBuilder); - return pNSchema; -} - int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) { if (tStartEncode(pCoder) < 0) return -1; @@ -5728,6 +5710,25 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) { return 0; } +void tDestroySVCreateTbReq(SVCreateTbReq *pReq, int32_t flags) { + if (pReq == NULL) return; + + if (flags & TSDB_MSG_FLG_ENCODE) { + // TODO + } else if (flags & TSDB_MSG_FLG_DECODE) { + if (pReq->comment) { + pReq->comment = NULL; + taosMemoryFree(pReq->comment); + } + + if (pReq->type == TSDB_CHILD_TABLE) { + if (pReq->ctb.tagName) taosArrayDestroy(pReq->ctb.tagName); + } else if (pReq->type == TSDB_NORMAL_TABLE) { + if (pReq->ntb.schemaRow.pSchema) taosMemoryFree(pReq->ntb.schemaRow.pSchema); + } + } +} + int tEncodeSVCreateTbBatchReq(SEncoder *pCoder, const SVCreateTbBatchReq *pReq) { int32_t nReq = taosArrayGetSize(pReq->pArray); @@ -6719,3 +6720,328 @@ int32_t tDecodeSBatchDeleteReq(SDecoder *pDecoder, SBatchDeleteReq *pReq) { } return 0; } + +static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubmitTbData) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32v(pCoder, pSubmitTbData->flags) < 0) return -1; + + // auto create table + if (pSubmitTbData->flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { + ASSERT(pSubmitTbData->pCreateTbReq); + if (tEncodeSVCreateTbReq(pCoder, pSubmitTbData->pCreateTbReq) < 0) return -1; + } + + // submit data + if (tEncodeI64(pCoder, pSubmitTbData->suid) < 0) return -1; + if (tEncodeI64(pCoder, pSubmitTbData->uid) < 0) return -1; + if (tEncodeI32v(pCoder, pSubmitTbData->sver) < 0) return -1; + + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + uint64_t nColData = TARRAY_SIZE(pSubmitTbData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pSubmitTbData->aCol); + + if (tEncodeU64v(pCoder, nColData) < 0) return -1; + + for (uint64_t i = 0; i < nColData; i++) { + pCoder->pos += tPutColData(pCoder->data ? pCoder->data + pCoder->pos : NULL, &aColData[i]); + } + } else { + if (tEncodeU64v(pCoder, TARRAY_SIZE(pSubmitTbData->aRowP)) < 0) return -1; + + SRow **rows = (SRow **)TARRAY_DATA(pSubmitTbData->aRowP); + for (int32_t iRow = 0; iRow < TARRAY_SIZE(pSubmitTbData->aRowP); ++iRow) { + if (pCoder->data) memcpy(pCoder->data + pCoder->pos, rows[iRow], rows[iRow]->len); + pCoder->pos += rows[iRow]->len; + } + } + + tEndEncode(pCoder); + return 0; +} + +static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbData) { + int32_t code = 0; + + if (tStartDecode(pCoder) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (tDecodeI32v(pCoder, &pSubmitTbData->flags) < 0) return -1; + + if (pSubmitTbData->flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { + pSubmitTbData->pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (pSubmitTbData->pCreateTbReq == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + if (tDecodeSVCreateTbReq(pCoder, pSubmitTbData->pCreateTbReq) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + } + + // submit data + if (tDecodeI64(pCoder, &pSubmitTbData->suid) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + if (tDecodeI64(pCoder, &pSubmitTbData->uid) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + if (tDecodeI32v(pCoder, &pSubmitTbData->sver) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + uint64_t nColData; + + if (tDecodeU64v(pCoder, &nColData) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + pSubmitTbData->aCol = taosArrayInit(nColData, sizeof(SColData)); + if (pSubmitTbData->aCol == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t i = 0; i < nColData; ++i) { + pCoder->pos += tGetColData(pCoder->data + pCoder->pos, taosArrayReserve(pSubmitTbData->aCol, 1)); + } + } else { + uint64_t nRow; + if (tDecodeU64v(pCoder, &nRow) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + pSubmitTbData->aRowP = taosArrayInit(nRow, sizeof(SRow *)); + if (pSubmitTbData->aRowP == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t iRow = 0; iRow < nRow; ++iRow) { + SRow **ppRow = taosArrayReserve(pSubmitTbData->aRowP, 1); + + *ppRow = (SRow *)(pCoder->data + pCoder->pos); + pCoder->pos += (*ppRow)->len; + } + } + + tEndDecode(pCoder); + +_exit: + if (code) { + // TODO: clear + } + return 0; +} + +int32_t tEncodeSSubmitReq2(SEncoder *pCoder, const SSubmitReq2 *pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeU64v(pCoder, taosArrayGetSize(pReq->aSubmitTbData)) < 0) return -1; + for (uint64_t i = 0; i < taosArrayGetSize(pReq->aSubmitTbData); i++) { + if (tEncodeSSubmitTbData(pCoder, taosArrayGet(pReq->aSubmitTbData, i)) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSSubmitReq2(SDecoder *pCoder, SSubmitReq2 *pReq) { + int32_t code = 0; + + memset(pReq, 0, sizeof(*pReq)); + + // decode + if (tStartDecode(pCoder) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + uint64_t nSubmitTbData; + if (tDecodeU64v(pCoder, &nSubmitTbData) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + pReq->aSubmitTbData = taosArrayInit(nSubmitTbData, sizeof(SSubmitTbData)); + if (pReq->aSubmitTbData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (uint64_t i = 0; i < nSubmitTbData; i++) { + if (tDecodeSSubmitTbData(pCoder, taosArrayReserve(pReq->aSubmitTbData, 1)) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + } + + tEndDecode(pCoder); + +_exit: + if (code) { + if (pReq->aSubmitTbData) { + // todo + taosArrayDestroy(pReq->aSubmitTbData); + pReq->aSubmitTbData = NULL; + } + } + return code; +} + +void tDestroySSubmitTbData(SSubmitTbData *pTbData, int32_t flag) { + if (NULL == pTbData) { + return; + } + + if (flag == TSDB_MSG_FLG_ENCODE) { + if (pTbData->pCreateTbReq) { + tdDestroySVCreateTbReq(pTbData->pCreateTbReq); + taosMemoryFree(pTbData->pCreateTbReq); + } + + if (pTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + int32_t nColData = TARRAY_SIZE(pTbData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pTbData->aCol); + + for (int32_t i = 0; i < nColData; ++i) { + tColDataDestroy(&aColData[i]); + } + taosArrayDestroy(pTbData->aCol); + } else { + int32_t nRow = TARRAY_SIZE(pTbData->aRowP); + SRow **rows = (SRow **)TARRAY_DATA(pTbData->aRowP); + + for (int32_t i = 0; i < nRow; ++i) { + tRowDestroy(rows[i]); + } + taosArrayDestroy(pTbData->aRowP); + } + } else if (flag == TSDB_MSG_FLG_DECODE) { + if (pTbData->pCreateTbReq) { + tDestroySVCreateTbReq(pTbData->pCreateTbReq, TSDB_MSG_FLG_DECODE); + taosMemoryFree(pTbData->pCreateTbReq); + } + + if (pTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + taosArrayDestroy(pTbData->aCol); + } else { + taosArrayDestroy(pTbData->aRowP); + } + } +} + +void tDestroySSubmitReq2(SSubmitReq2 *pReq, int32_t flag) { + if (pReq->aSubmitTbData == NULL) return; + + int32_t nSubmitTbData = TARRAY_SIZE(pReq->aSubmitTbData); + SSubmitTbData *aSubmitTbData = (SSubmitTbData *)TARRAY_DATA(pReq->aSubmitTbData); + + for (int32_t i = 0; i < nSubmitTbData; i++) { + tDestroySSubmitTbData(&aSubmitTbData[i], flag); + } + taosArrayDestroy(pReq->aSubmitTbData); +} + +int32_t tEncodeSSubmitRsp2(SEncoder *pCoder, const SSubmitRsp2 *pRsp) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32v(pCoder, pRsp->affectedRows) < 0) return -1; + + if (tEncodeU64v(pCoder, taosArrayGetSize(pRsp->aCreateTbRsp)) < 0) return -1; + for (int32_t i = 0; i < taosArrayGetSize(pRsp->aCreateTbRsp); ++i) { + if (tEncodeSVCreateTbRsp(pCoder, taosArrayGet(pRsp->aCreateTbRsp, i)) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSSubmitRsp2(SDecoder *pCoder, SSubmitRsp2 *pRsp) { + int32_t code = 0; + + memset(pRsp, 0, sizeof(SSubmitRsp2)); + + // decode + if (tStartDecode(pCoder) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (tDecodeI32v(pCoder, &pRsp->affectedRows) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + uint64_t nCreateTbRsp; + if (tDecodeU64v(pCoder, &nCreateTbRsp) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (nCreateTbRsp) { + pRsp->aCreateTbRsp = taosArrayInit(nCreateTbRsp, sizeof(SVCreateTbRsp)); + if (pRsp->aCreateTbRsp == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t i = 0; i < nCreateTbRsp; ++i) { + SVCreateTbRsp *pCreateTbRsp = taosArrayReserve(pRsp->aCreateTbRsp, 1); + if (tDecodeSVCreateTbRsp(pCoder, pCreateTbRsp) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + } + } + + tEndDecode(pCoder); + +_exit: + if (code) { + if (pRsp->aCreateTbRsp) { + taosArrayDestroyEx(pRsp->aCreateTbRsp, NULL /* todo */); + } + } + return code; +} + +void tDestroySSubmitRsp2(SSubmitRsp2 *pRsp, int32_t flag) { + if (NULL == pRsp) { + return; + } + + if (flag & TSDB_MSG_FLG_ENCODE) { + if (pRsp->aCreateTbRsp) { + int32_t nCreateTbRsp = TARRAY_SIZE(pRsp->aCreateTbRsp); + SVCreateTbRsp *aCreateTbRsp = TARRAY_DATA(pRsp->aCreateTbRsp); + for (int32_t i = 0; i < nCreateTbRsp; ++i) { + if (aCreateTbRsp[i].pMeta) { + taosMemoryFree(aCreateTbRsp[i].pMeta); + } + } + taosArrayDestroy(pRsp->aCreateTbRsp); + } + } else if (flag & TSDB_MSG_FLG_DECODE) { + if (pRsp->aCreateTbRsp) { + int32_t nCreateTbRsp = TARRAY_SIZE(pRsp->aCreateTbRsp); + SVCreateTbRsp *aCreateTbRsp = TARRAY_DATA(pRsp->aCreateTbRsp); + for (int32_t i = 0; i < nCreateTbRsp; ++i) { + if (aCreateTbRsp[i].pMeta) { + taosMemoryFree(aCreateTbRsp[i].pMeta); + } + } + taosArrayDestroy(pRsp->aCreateTbRsp); + } + } +} diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 0d47ef1e7f70c791a32feaa6b4075bd9c625d69f..e5ed7a3728cebe78ae9cc24304e6e8ce2e94c7ef 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -90,16 +90,12 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName) { pName->type = TSDB_TABLE_NAME_T; pName->acctId = acctId; - memset(pName->dbname, 0, TSDB_DB_NAME_LEN); - strncpy(pName->dbname, pDbName, TSDB_DB_NAME_LEN - 1); - memset(pName->tname, 0, TSDB_TABLE_NAME_LEN); - strncpy(pName->tname, pTableName, TSDB_TABLE_NAME_LEN - 1); + snprintf(pName->dbname, sizeof(pName->dbname), "%s", pDbName); + snprintf(pName->tname, sizeof(pName->tname), "%s", pTableName); return pName; } int32_t tNameExtractFullName(const SName* name, char* dst) { - assert(name != NULL && dst != NULL); - // invalid full name format, abort if (!tNameIsValid(name)) { return -1; @@ -109,7 +105,7 @@ int32_t tNameExtractFullName(const SName* name, char* dst) { size_t tnameLen = strlen(name->tname); if (tnameLen > 0) { - /*assert(name->type == TSDB_TABLE_NAME_T);*/ + /*ASSERT(name->type == TSDB_TABLE_NAME_T);*/ dst[len] = TS_PATH_DELIMITER[0]; memcpy(dst + len + 1, name->tname, tnameLen); @@ -120,25 +116,21 @@ int32_t tNameExtractFullName(const SName* name, char* dst) { } int32_t tNameLen(const SName* name) { - assert(name != NULL); - char tmp[12] = {0}; int32_t len = sprintf(tmp, "%d", name->acctId); int32_t len1 = (int32_t)strlen(name->dbname); int32_t len2 = (int32_t)strlen(name->tname); if (name->type == TSDB_DB_NAME_T) { - assert(len2 == 0); + ASSERT(len2 == 0); return len + len1 + TSDB_NAME_DELIMITER_LEN; } else { - assert(len2 > 0); + ASSERT(len2 > 0); return len + len1 + len2 + TSDB_NAME_DELIMITER_LEN * 2; } } bool tNameIsValid(const SName* name) { - assert(name != NULL); - if (!VALID_NAME_TYPE(name->type)) { return false; } @@ -151,15 +143,12 @@ bool tNameIsValid(const SName* name) { } SName* tNameDup(const SName* name) { - assert(name != NULL); - SName* p = taosMemoryMalloc(sizeof(SName)); memcpy(p, name, sizeof(SName)); return p; } int32_t tNameGetDbName(const SName* name, char* dst) { - assert(name != NULL && dst != NULL); strncpy(dst, name->dbname, tListLen(name->dbname)); return 0; } @@ -167,28 +156,22 @@ int32_t tNameGetDbName(const SName* name, char* dst) { const char* tNameGetDbNameP(const SName* name) { return &name->dbname[0]; } int32_t tNameGetFullDbName(const SName* name, char* dst) { - assert(name != NULL && dst != NULL); snprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname); return 0; } -bool tNameIsEmpty(const SName* name) { - assert(name != NULL); - return name->type == 0 || name->acctId == 0; -} +bool tNameIsEmpty(const SName* name) { return name->type == 0 || name->acctId == 0; } const char* tNameGetTableName(const SName* name) { - assert(name != NULL && name->type == TSDB_TABLE_NAME_T); + ASSERT(name != NULL && name->type == TSDB_TABLE_NAME_T); return &name->tname[0]; } void tNameAssign(SName* dst, const SName* src) { memcpy(dst, src, sizeof(SName)); } int32_t tNameSetDbName(SName* dst, int32_t acct, const char* dbName, size_t nameLen) { - assert(dst != NULL && dbName != NULL && nameLen > 0); - // too long account id or too long db name - if (nameLen >= tListLen(dst->dbname)) { + if (nameLen <= 0 || nameLen >= tListLen(dst->dbname)) { return -1; } @@ -199,8 +182,6 @@ int32_t tNameSetDbName(SName* dst, int32_t acct, const char* dbName, size_t name } int32_t tNameAddTbName(SName* dst, const char* tbName, size_t nameLen) { - assert(dst != NULL && tbName != NULL && nameLen > 0); - // too long account id or too long db name if (nameLen >= tListLen(dst->tname) || nameLen <= 0) { return -1; @@ -212,7 +193,6 @@ int32_t tNameAddTbName(SName* dst, const char* tbName, size_t nameLen) { } int32_t tNameSetAcctId(SName* dst, int32_t acctId) { - assert(dst != NULL); dst->acctId = acctId; return 0; } @@ -247,7 +227,9 @@ bool tNameTbNameEqual(SName* left, SName* right) { } int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { - assert(dst != NULL && str != NULL && strlen(str) > 0); + if (strlen(str) == 0) { + return -1; + } char* p = NULL; if ((type & T_NAME_ACCT) == T_NAME_ACCT) { @@ -298,8 +280,8 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { } static int compareKv(const void* p1, const void* p2) { - SSmlKv* kv1 = *(SSmlKv**)p1; - SSmlKv* kv2 = *(SSmlKv**)p2; + SSmlKv* kv1 = (SSmlKv*)p1; + SSmlKv* kv2 = (SSmlKv*)p2; int32_t kvLen1 = kv1->keyLen; int32_t kvLen2 = kv2->keyLen; int32_t res = strncasecmp(kv1->key, kv2->key, TMIN(kvLen1, kvLen2)); @@ -316,11 +298,11 @@ static int compareKv(const void* p1, const void* p2) { void buildChildTableName(RandTableName* rName) { SStringBuilder sb = {0}; taosStringBuilderAppendStringLen(&sb, rName->stbFullName, rName->stbFullNameLen); - if(sb.buf == NULL) return; + if (sb.buf == NULL) return; taosArraySort(rName->tags, compareKv); for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) { taosStringBuilderAppendChar(&sb, ','); - SSmlKv* tagKv = taosArrayGetP(rName->tags, j); + SSmlKv* tagKv = taosArrayGet(rName->tags, j); taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); taosStringBuilderAppendChar(&sb, '='); if (IS_VAR_DATA_TYPE(tagKv->type)) { @@ -344,5 +326,4 @@ void buildChildTableName(RandTableName* rName) { strcat(rName->ctbShortName, temp); } taosStringBuilderDestroy(&sb); - rName->uid = *(uint64_t*)(context.digest); } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index ca2c0567439f22bcc77e23ae69065f065a275764..9d381ce15fa762b74cecf23eeec1fa6667a10b99 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -76,7 +76,6 @@ void tdSCellValPrint(SCellVal *pVal, int8_t colType) { return; } if (!pVal->val) { - ASSERT(0); printf("BadVal "); return; } @@ -342,7 +341,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { } if (iColumn == 0) { - ASSERT(pColVal->cid == pTColumn->colId); + ASSERT(pColVal && pColVal->cid == pTColumn->colId); ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); ASSERT(pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID); } else { @@ -490,7 +489,6 @@ bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCell int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { if (!pBitmap || colIdx < 0) { - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -512,7 +510,6 @@ int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pVa *pValType = ((*pDestByte) & 0x03); break; default: - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -521,7 +518,6 @@ int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pVa int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { if (!pBitmap || colIdx < 0) { - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -555,7 +551,6 @@ int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pVal *pValType = ((*pDestByte) & 0x01); break; default: - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -564,7 +559,6 @@ int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pVal int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { if (!pBitmap || colIdx < 0) { - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -607,7 +601,6 @@ int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { // *pDestByte |= (valType); break; default: - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -630,7 +623,6 @@ int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_ output->val = POINTER_SHIFT(pRow, offset); } #else - ASSERT(0); if (offset < 0) { terrno = TSDB_CODE_INVALID_PARA; output->valType = TD_VTYPE_NONE; @@ -680,7 +672,6 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp return terrno; } #else - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; #endif @@ -707,8 +698,8 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp if (!pBuilder->hasNone) pBuilder->hasNone = true; return TSDB_CODE_SUCCESS; default: - ASSERT(0); - break; + terrno = TSDB_CODE_INVALID_PARA; + return terrno; } if (TD_IS_TP_ROW(pRow)) { @@ -722,7 +713,6 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId) { if ((offset < (int32_t)sizeof(SKvRowIdx)) || (colIdx < 1)) { - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -810,7 +800,6 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou pBuilder->nCols = nCols; pBuilder->nBoundCols = nBoundCols; if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -832,7 +821,6 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { pBuilder->pBuf = (STSRow *)pBuf; if (!pBuilder->pBuf) { - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -869,7 +857,6 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols); break; default: - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -880,7 +867,6 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { pBuilder->pBuf = (STSRow *)pBuf; if (!pBuilder->pBuf) { - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -900,7 +886,6 @@ int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { #endif break; default: - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -920,7 +905,6 @@ int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen) { pBuilder->flen = flen; pBuilder->nCols = nCols; if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -939,7 +923,6 @@ int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, pBuilder->nCols = nCols; pBuilder->nBoundCols = nBoundCols; if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -968,7 +951,6 @@ int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValT tdGetBitmapValTypeI(pBitmap, colIdx, pValType); break; default: - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return TSDB_CODE_FAILED; } @@ -987,7 +969,6 @@ bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode) int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType) { if (!pBitmap || colIdx < 0) { - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -1014,7 +995,6 @@ int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType) { // *pDestByte |= (valType); break; default: - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -1031,7 +1011,6 @@ int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int tdSetBitmapValTypeI(pBitmap, colIdx, valType); break; default: - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return TSDB_CODE_FAILED; } diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index ec05ef4c449c3ad3cb917a11feb727030d8ee06d..559ffd2aaf24e3628110a6f5418cd7d8094ed397 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -168,12 +168,13 @@ int64_t parseFraction(char* str, char** end, int32_t timePrec) { i = MICRO_SEC_FRACTION_LEN; } times = MICRO_SEC_FRACTION_LEN - i; - } else { - assert(timePrec == TSDB_TIME_PRECISION_NANO); + } else if (timePrec == TSDB_TIME_PRECISION_NANO) { if (i >= NANO_SEC_FRACTION_LEN) { i = NANO_SEC_FRACTION_LEN; } times = NANO_SEC_FRACTION_LEN - i; + } else { + return -1; } fraction = strnatoi(str, i) * factor[times]; @@ -510,8 +511,11 @@ int64_t convertTimePrecision(int64_t utime, int32_t fromPrecision, int32_t toPre // !!!!notice: double lose precison if time is too large, for example: 1626006833631000000*1.0 = double = // 1626006833631000064 int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit) { - assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || - fromPrecision == TSDB_TIME_PRECISION_NANO); + if (fromPrecision != TSDB_TIME_PRECISION_MILLI && fromPrecision != TSDB_TIME_PRECISION_MICRO && + fromPrecision != TSDB_TIME_PRECISION_NANO) { + return -1; + } + int64_t factors[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; double tmp = time; switch (toUnit) { @@ -761,8 +765,7 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char } int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision) { - if (pInterval->sliding == 0) { - assert(pInterval->interval == 0); + if (pInterval->sliding == 0 && pInterval->interval == 0) { return t; } @@ -931,7 +934,7 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) default: fractionLen = 0; - assert(false); + ASSERT(false); } taosLocalTime(", &ptm); diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index 3a6bc097c1ae97c7cfadac1df81b6cfb0897dab3..0faa6eb4c85a318a25e6324063cf8ede237c88f6 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -17,6 +17,7 @@ #include "ttszip.h" #include "taoserror.h" #include "tcompression.h" +#include "tlog.h" static int32_t getDataStartOffset(); static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo); @@ -202,14 +203,14 @@ void* tsBufDestroy(STSBuf* pTSBuf) { static STSGroupBlockInfoEx* tsBufGetLastGroupInfo(STSBuf* pTSBuf) { int32_t last = pTSBuf->numOfGroups - 1; - assert(last >= 0); + ASSERT(last >= 0); return &pTSBuf->pData[last]; } static STSGroupBlockInfoEx* addOneGroupInfo(STSBuf* pTSBuf, int32_t id) { if (pTSBuf->numOfAlloc <= pTSBuf->numOfGroups) { uint32_t newSize = (uint32_t)(pTSBuf->numOfAlloc * 1.5); - assert((int32_t)newSize > pTSBuf->numOfAlloc); + ASSERT((int32_t)newSize > pTSBuf->numOfAlloc); STSGroupBlockInfoEx* tmp = (STSGroupBlockInfoEx*)taosMemoryRealloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); @@ -233,7 +234,7 @@ static STSGroupBlockInfoEx* addOneGroupInfo(STSBuf* pTSBuf, int32_t id) { STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[pTSBuf->numOfGroups].info; pBlockInfo->id = id; pBlockInfo->offset = pTSBuf->fileSize; - assert(pBlockInfo->offset >= getDataStartOffset()); + ASSERT(pBlockInfo->offset >= getDataStartOffset()); // update vnode info in file TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups, pBlockInfo); @@ -282,7 +283,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) { pTsData->allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); int64_t r = taosLSeekFile(pTSBuf->pFile, pTSBuf->fileSize, SEEK_SET); - assert(r == 0); + ASSERT(r == 0); /* * format for output data: @@ -316,7 +317,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) { taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen)); metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen)); - assert(metaLen == getTagAreaLength(&pBlock->tag)); + ASSERT(metaLen == getTagAreaLength(&pBlock->tag)); int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; pTSBuf->fileSize += blockSize; @@ -379,7 +380,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { size_t sz = 0; if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { char* tp = taosMemoryRealloc(pBlock->tag.pz, pBlock->tag.nLen + 1); - assert(tp != NULL); + ASSERT(tp != NULL); memset(tp, 0, pBlock->tag.nLen + 1); pBlock->tag.pz = tp; @@ -410,14 +411,14 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { // read the comp length at the length of comp block sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding)); - assert(pBlock->padding == pBlock->compLen); + ASSERT(pBlock->padding == pBlock->compLen); int32_t n = 0; sz = taosReadFile(pTSBuf->pFile, &n, sizeof(pBlock->tag.nLen)); if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) { - assert(n == 0); + ASSERT(n == 0); } else { - assert(n == pBlock->tag.nLen); + ASSERT(n == pBlock->tag.nLen); } UNUSED(sz); @@ -477,7 +478,7 @@ void tsBufAppend(STSBuf* pTSBuf, int32_t id, SVariant* tag, const char* pData, i pBlockInfo = tsBufGetLastGroupInfo(pTSBuf); } - assert(pBlockInfo->info.id == id); + ASSERT(pBlockInfo->info.id == id); if ((taosVariantCompare(&pTSBuf->block.tag, tag) != 0) && ptsData->len > 0) { // new arrived data with different tags value, save current value into disk first @@ -596,7 +597,7 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex) { STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[groupIndex].info; if (pBlockInfo->numOfBlocks <= blockIndex) { - assert(false); + ASSERT(false); } STSCursor* pCur = &pTSBuf->cur; @@ -613,7 +614,7 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex } } else { if (tsBufFindBlock(pTSBuf, pBlockInfo, blockIndex) == -1) { - assert(false); + ASSERT(false); } } @@ -633,7 +634,7 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf, pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); - assert((pTSBuf->tsData.len / TSDB_KEYSIZE == pBlock->numOfElem) && (pTSBuf->tsData.allocSize >= pTSBuf->tsData.len)); + ASSERT((pTSBuf->tsData.len / TSDB_KEYSIZE == pBlock->numOfElem) && (pTSBuf->tsData.allocSize >= pTSBuf->tsData.len)); pCur->vgroupIndex = groupIndex; pCur->blockIndex = blockIndex; @@ -668,7 +669,9 @@ int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { return -1; } - assert(pHeader->tsOrder == TSDB_ORDER_ASC || pHeader->tsOrder == TSDB_ORDER_DESC); + if (pHeader->tsOrder != TSDB_ORDER_ASC && pHeader->tsOrder != TSDB_ORDER_DESC) { + return -1; + } int32_t r = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET); if (r != 0) { @@ -705,7 +708,7 @@ bool tsBufNextPos(STSBuf* pTSBuf) { } } else { // get the last timestamp record in the last block of the last vnode - assert(pTSBuf->numOfGroups > 0); + ASSERT(pTSBuf->numOfGroups > 0); int32_t groupIndex = pTSBuf->numOfGroups - 1; pCur->vgroupIndex = groupIndex; @@ -729,7 +732,7 @@ bool tsBufNextPos(STSBuf* pTSBuf) { int32_t step = pCur->order == TSDB_ORDER_ASC ? 1 : -1; while (1) { - assert(pTSBuf->tsData.len == pTSBuf->block.numOfElem * TSDB_KEYSIZE); + ASSERT(pTSBuf->tsData.len == pTSBuf->block.numOfElem * TSDB_KEYSIZE); if ((pCur->order == TSDB_ORDER_ASC && pCur->tsIndex >= pTSBuf->block.numOfElem - 1) || (pCur->order == TSDB_ORDER_DESC && pCur->tsIndex <= 0)) { @@ -810,7 +813,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { } // src can only have one vnode index - assert(pSrcBuf->numOfGroups == 1); + ASSERT(pSrcBuf->numOfGroups == 1); // there are data in buffer, flush to disk first tsBufFlush(pDestBuf); @@ -853,7 +856,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { } int32_t r = taosLSeekFile(pDestBuf->pFile, 0, SEEK_END); - assert(r == 0); + ASSERT(r == 0); int64_t offset = getDataStartOffset(); int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset; @@ -881,7 +884,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { } pDestBuf->fileSize = (uint32_t)file_size; - assert(pDestBuf->fileSize == oldSize + size); + ASSERT(pDestBuf->fileSize == oldSize + size); return 0; } @@ -913,7 +916,10 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ pTSBuf->fileSize += len; pTSBuf->tsOrder = order; - assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); + if (order != TSDB_ORDER_ASC && order != TSDB_ORDER_DESC) { + tsBufDestroy(pTSBuf); + return NULL; + } STSBufFileHeader header = { .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; @@ -1095,7 +1101,7 @@ void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id) { } int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, int32_t* len, int32_t* numOfBlocks) { - assert(groupIndex >= 0 && groupIndex < pTSBuf->numOfGroups); + ASSERT(groupIndex >= 0 && groupIndex < pTSBuf->numOfGroups); STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[groupIndex].info; *len = 0; diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 7b5d0a8805ed77ac580d5c4c2f02d699338012f7..c83bdc0e3293e97c01a2eb7aed24652e9fd9775d 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -131,13 +131,16 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) { varDataCopy(val, src); break; default: { - memcpy(val, src, len); + if (len > 0) { + memcpy(val, src, len); + } + break; } } } -void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { +int32_t operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { if (optr == OP_TYPE_ADD) { switch (type) { case TSDB_DATA_TYPE_TINYINT: @@ -174,11 +177,12 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { SET_DOUBLE_VAL(dst, GET_DOUBLE_VAL(s1) + GET_DOUBLE_VAL(s2)); break; default: { - assert(0); - break; + return -1; } } } else { - assert(0); + return -1; } + + return 0; } diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 65e9a767f5843d7ce025ccfbe2f789475d7105f6..de225581a679db43a4c5b7aaccc12854d75d0495 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -168,7 +168,7 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { pSrc->nType == TSDB_DATA_TYPE_JSON) { int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE; char *p = taosMemoryRealloc(pDst->pz, len); - assert(p); + ASSERT(p); memset(p, 0, len); pDst->pz = p; @@ -192,7 +192,7 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { size_t num = taosArrayGetSize(pSrc->arr); pDst->arr = taosArrayInit(num, sizeof(int64_t)); pDst->nLen = pSrc->nLen; - assert(pSrc->nLen == num); + ASSERT(pSrc->nLen == num); for (size_t i = 0; i < num; i++) { int64_t *p = taosArrayGet(pSrc->arr, i); taosArrayPush(pDst->arr, p); diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index b05ae602f8ee5299ed108756f73f648867d1f81a..4f8652d02cc0602583edff7f59354d9d093275fc 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -117,7 +117,7 @@ STSchema *genSTSchema(int16_t nCols) { } STSchema *pResult = NULL; - pResult = tdGetSTSChemaFromSSChema(pSchema, nCols, 1); + pResult = tBuildTSchema(pSchema, nCols, 1); taosMemoryFree(pSchema); return pResult; diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index a8103351b429d2676c2cbaa37bddb2aa17741241..4910b0ac3faab414b4a3197cd2412a7e4badd21e 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -44,6 +44,7 @@ static struct { char apolloUrl[PATH_MAX]; const char **envCmd; SArray *pArgs; // SConfigPair + int64_t startTime; } global = {0}; static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetAllDebugFlag(143, true); } @@ -67,23 +68,71 @@ static void dmStopDnode(int signum, void *sigInfo, void *context) { dmStop(); } +void dmLogCrash(int signum, void *sigInfo, void *context) { + taosIgnSignal(SIGTERM); + taosIgnSignal(SIGHUP); + taosIgnSignal(SIGINT); + taosIgnSignal(SIGBREAK); + +#ifndef WINDOWS + taosIgnSignal(SIGBUS); +#endif + taosIgnSignal(SIGABRT); + taosIgnSignal(SIGFPE); + taosIgnSignal(SIGSEGV); + + char *pMsg = NULL; + const char *flags = "UTL FATAL "; + ELogLevel level = DEBUG_FATAL; + int32_t dflag = 255; + int64_t msgLen= -1; + + if (tsEnableCrashReport) { + if (taosGenCrashJsonMsg(signum, &pMsg, dmGetClusterId(), global.startTime)) { + taosPrintLog(flags, level, dflag, "failed to generate crash json msg"); + goto _return; + } else { + msgLen = strlen(pMsg); + } + } + +_return: + + taosLogCrashInfo("taosd", pMsg, msgLen, signum, sigInfo); + +#ifdef _TD_DARWIN_64 + exit(signum); +#elif defined(WINDOWS) + exit(signum); +#endif +} + static void dmSetSignalHandle() { taosSetSignal(SIGUSR1, dmSetDebugFlag); taosSetSignal(SIGUSR2, dmSetAssert); taosSetSignal(SIGTERM, dmStopDnode); taosSetSignal(SIGHUP, dmStopDnode); taosSetSignal(SIGINT, dmStopDnode); - taosSetSignal(SIGABRT, dmStopDnode); taosSetSignal(SIGBREAK, dmStopDnode); #ifndef WINDOWS taosSetSignal(SIGTSTP, dmStopDnode); taosSetSignal(SIGQUIT, dmStopDnode); #endif + +#ifndef WINDOWS + taosSetSignal(SIGBUS, dmLogCrash); +#endif + taosSetSignal(SIGABRT, dmLogCrash); + taosSetSignal(SIGFPE, dmLogCrash); + taosSetSignal(SIGSEGV, dmLogCrash); } static int32_t dmParseArgs(int32_t argc, char const *argv[]) { + global.startTime = taosGetTimestampMs(); + int32_t cmdEnvIndex = 0; if (argc < 2) return 0; + global.envCmd = taosMemoryMalloc((argc - 1) * sizeof(char *)); memset(global.envCmd, 0, (argc - 1) * sizeof(char *)); for (int32_t i = 1; i < argc; ++i) { @@ -178,7 +227,7 @@ static int32_t dmInitLog() { } static void taosCleanupArgs() { - if (global.envCmd != NULL) taosMemoryFree(global.envCmd); + if (global.envCmd != NULL) taosMemoryFreeClear(global.envCmd); } int main(int argc, char const *argv[]) { @@ -268,6 +317,10 @@ int mainWindows(int argc, char **argv) { if (dmInit() != 0) { dError("failed to init dnode since %s", terrstr()); + + taosCleanupCfg(); + taosCloseLog(); + taosConvDestroy(); return -1; } diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index c776beb3f099d40173bc69490f7dc65bef67ed71..ff32cbcb08b2c743644df3b077408dfd4878e999 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -29,6 +29,7 @@ typedef struct SDnodeMgmt { const char *name; TdThread statusThread; TdThread monitorThread; + TdThread crashReportThread; SSingleWorker mgmtWorker; ProcessCreateNodeFp processCreateNodeFp; ProcessDropNodeFp processDropNodeFp; @@ -55,6 +56,8 @@ int32_t dmStartStatusThread(SDnodeMgmt *pMgmt); void dmStopStatusThread(SDnodeMgmt *pMgmt); int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt); void dmStopMonitorThread(SDnodeMgmt *pMgmt); +int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt); +void dmStopCrashReportThread(SDnodeMgmt *pMgmt); int32_t dmStartWorker(SDnodeMgmt *pMgmt); void dmStopWorker(SDnodeMgmt *pMgmt); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index d2db1a4a62fd157b2df235133c85bb6e38ac680d..51df293ba70bc37be14c763ecb4a5ea296077bd4 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -23,6 +23,9 @@ static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) { if (dmStartMonitorThread(pMgmt) != 0) { return -1; } + if (dmStartCrashReportThread(pMgmt) != 0) { + return -1; + } return 0; } @@ -30,6 +33,7 @@ static void dmStopMgmt(SDnodeMgmt *pMgmt) { pMgmt->pData->stopped = true; dmStopMonitorThread(pMgmt); dmStopStatusThread(pMgmt); + dmStopCrashReportThread(pMgmt); } static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 80c040a5e8f84a8d5d7ea887b8a633f945576e17..76c8e09b70a31c8e9168809dea69bc1a1a3e5478 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "dmInt.h" +#include "thttp.h" static void *dmStatusThreadFp(void *param) { SDnodeMgmt *pMgmt = param; @@ -63,6 +64,63 @@ static void *dmMonitorThreadFp(void *param) { return NULL; } +static void *dmCrashReportThreadFp(void *param) { + SDnodeMgmt *pMgmt = param; + int64_t lastTime = taosGetTimestampMs(); + setThreadName("dnode-crashReport"); + char filepath[PATH_MAX] = {0}; + snprintf(filepath, sizeof(filepath), "%s%s.taosdCrashLog", tsLogDir, TD_DIRSEP); + char *pMsg = NULL; + int64_t msgLen = 0; + TdFilePtr pFile = NULL; + bool truncateFile = false; + int32_t sleepTime = 200; + int32_t reportPeriodNum = 3600 * 1000 / sleepTime;; + int32_t loopTimes = reportPeriodNum; + + while (1) { + if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; + if (loopTimes++ < reportPeriodNum) { + taosMsleep(sleepTime); + continue; + } + + taosReadCrashInfo(filepath, &pMsg, &msgLen, &pFile); + if (pMsg && msgLen > 0) { + if (taosSendHttpReport(tsTelemServer, tsSvrCrashReportUri, tsTelemPort, pMsg, msgLen, HTTP_FLAT) != 0) { + dError("failed to send crash report"); + if (pFile) { + taosReleaseCrashLogFile(pFile, false); + continue; + } + } else { + dInfo("succeed to send crash report"); + truncateFile = true; + } + } else { + dDebug("no crash info"); + } + + taosMemoryFree(pMsg); + + if (pMsg && msgLen > 0) { + pMsg = NULL; + continue; + } + + if (pFile) { + taosReleaseCrashLogFile(pFile, truncateFile); + truncateFile = false; + } + + taosMsleep(sleepTime); + loopTimes = 0; + } + + return NULL; +} + + int32_t dmStartStatusThread(SDnodeMgmt *pMgmt) { TdThreadAttr thAttr; taosThreadAttrInit(&thAttr); @@ -105,6 +163,36 @@ void dmStopMonitorThread(SDnodeMgmt *pMgmt) { } } +int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt) { + if (!tsEnableCrashReport) { + return 0; + } + + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + if (taosThreadCreate(&pMgmt->crashReportThread, &thAttr, dmCrashReportThreadFp, pMgmt) != 0) { + dError("failed to create crashReport thread since %s", strerror(errno)); + return -1; + } + + taosThreadAttrDestroy(&thAttr); + tmsgReportStartup("dnode-crashReport", "initialized"); + return 0; +} + +void dmStopCrashReportThread(SDnodeMgmt *pMgmt) { + if (!tsEnableCrashReport) { + return; + } + + if (taosCheckPthreadValid(pMgmt->crashReportThread)) { + taosThreadJoin(pMgmt->crashReportThread, NULL); + taosThreadClear(&pMgmt->crashReportThread); + } +} + + static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SDnodeMgmt *pMgmt = pInfo->ahandle; int32_t code = -1; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index c5ddb9f021c65a2ac7f616a9236bced7954a2aa0..f06669a610142a38b6b2937c95bd150c324df568 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -15,171 +15,166 @@ #define _DEFAULT_SOURCE #include "mmInt.h" +#include "tjson.h" + +static int32_t mmDecodeOption(SJson *pJson, SMnodeOpt *pOption) { + int32_t code = 0; + + tjsonGetInt32ValueFromDouble(pJson, "deployed", pOption->deploy, code); + if (code < 0) return -1; + tjsonGetInt32ValueFromDouble(pJson, "selfIndex", pOption->selfIndex, code); + if (code < 0) return 0; + + SJson *replicas = tjsonGetObjectItem(pJson, "replicas"); + if (replicas == NULL) return 0; + pOption->numOfReplicas = tjsonGetArraySize(replicas); + + for (int32_t i = 0; i < pOption->numOfReplicas; ++i) { + SJson *replica = tjsonGetArrayItem(replicas, i); + if (replica == NULL) return -1; + + SReplica *pReplica = pOption->replicas + i; + tjsonGetInt32ValueFromDouble(replica, "id", pReplica->id, code); + if (code < 0) return -1; + code = tjsonGetStringValue(replica, "fqdn", pReplica->fqdn); + if (code < 0) return -1; + tjsonGetUInt16ValueFromDouble(replica, "port", pReplica->port, code); + if (code < 0) return -1; + } + + return 0; +} int32_t mmReadFile(const char *path, SMnodeOpt *pOption) { - int32_t code = TSDB_CODE_INVALID_JSON_FORMAT; - int32_t len = 0; - int32_t maxLen = 4096; - char *content = taosMemoryCalloc(1, maxLen + 1); - cJSON *root = NULL; - char file[PATH_MAX] = {0}; + int32_t code = -1; TdFilePtr pFile = NULL; - + char *pData = NULL; + SJson *pJson = NULL; + char file[PATH_MAX] = {0}; snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP); + + if (taosStatFile(file, NULL, NULL) < 0) { + dInfo("mnode file:%s not exist", file); + return 0; + } + pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - code = 0; + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to open mnode file:%s since %s", file, terrstr()); goto _OVER; } - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", file); + int64_t size = 0; + if (taosFStatFile(pFile, &size, NULL) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to fstat mnode file:%s since %s", file, terrstr()); goto _OVER; } - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); + pData = taosMemoryMalloc(size + 1); + if (pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } - cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); - if (!deployed || deployed->type != cJSON_Number) { - dError("failed to read %s since deployed not found", file); + if (taosReadFile(pFile, pData, size) != size) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to read mnode file:%s since %s", file, terrstr()); goto _OVER; } - pOption->deploy = deployed->valueint; - cJSON *selfIndex = cJSON_GetObjectItem(root, "selfIndex"); - if (selfIndex) { - if (selfIndex->type != cJSON_Number) { - dError("failed to read %s since selfIndex not found", file); - goto _OVER; - } - pOption->selfIndex = selfIndex->valueint; - } + pData[size] = '\0'; - cJSON *replicas = cJSON_GetObjectItem(root, "replicas"); - if (replicas) { - if (replicas->type != cJSON_Array) { - dError("failed to read %s since replicas not found", file); - goto _OVER; - } + pJson = tjsonParse(pData); + if (pJson == NULL) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; + goto _OVER; + } - int32_t numOfReplicas = cJSON_GetArraySize(replicas); - if (numOfReplicas <= 0) { - dError("failed to read %s since numOfReplicas:%d invalid", file, numOfReplicas); - goto _OVER; - } - pOption->numOfReplicas = numOfReplicas; - - for (int32_t i = 0; i < numOfReplicas; ++i) { - SReplica *pReplica = pOption->replicas + i; - - cJSON *replica = cJSON_GetArrayItem(replicas, i); - if (replica == NULL) break; - - cJSON *id = cJSON_GetObjectItem(replica, "id"); - if (id) { - if (id->type != cJSON_Number) { - dError("failed to read %s since id not found", file); - goto _OVER; - } - if (pReplica) { - pReplica->id = id->valueint; - } - } - - cJSON *fqdn = cJSON_GetObjectItem(replica, "fqdn"); - if (fqdn) { - if (fqdn->type != cJSON_String || fqdn->valuestring == NULL) { - dError("failed to read %s since fqdn not found", file); - goto _OVER; - } - if (pReplica) { - tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN); - } - } - - cJSON *port = cJSON_GetObjectItem(replica, "port"); - if (port) { - if (port->type != cJSON_Number) { - dError("failed to read %s since port not found", file); - goto _OVER; - } - if (pReplica) { - pReplica->port = (uint16_t)port->valueint; - } - } - } + if (mmDecodeOption(pJson, pOption) < 0) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; + goto _OVER; } code = 0; + dInfo("succceed to read mnode file %s", file); _OVER: - if (content != NULL) taosMemoryFree(content); - if (root != NULL) cJSON_Delete(root); + if (pData != NULL) taosMemoryFree(pData); + if (pJson != NULL) cJSON_Delete(pJson); if (pFile != NULL) taosCloseFile(&pFile); - if (code == 0) { - dDebug("succcessed to read file %s, deployed:%d", file, pOption->deploy); - } - terrno = code; + if (code != 0) { + dError("failed to read mnode file:%s since %s", file, terrstr()); + } return code; } -int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) { - char file[PATH_MAX] = {0}; - char realfile[PATH_MAX] = {0}; - snprintf(file, sizeof(file), "%s%smnode.json.bak", path, TD_DIRSEP); - snprintf(realfile, sizeof(realfile), "%s%smnode.json", path, TD_DIRSEP); - - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to write %s since %s", file, terrstr()); - return -1; - } - - int32_t len = 0; - int32_t maxLen = 4096; - char *content = taosMemoryCalloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); +static int32_t mmEncodeOption(SJson *pJson, const SMnodeOpt *pOption) { if (pOption->deploy && pOption->numOfReplicas > 0) { - len += snprintf(content + len, maxLen - len, " \"selfIndex\": %d,\n", pOption->selfIndex); - len += snprintf(content + len, maxLen - len, " \"replicas\": [{\n"); + if (tjsonAddDoubleToObject(pJson, "selfIndex", pOption->selfIndex) < 0) return -1; + + SJson *replicas = tjsonCreateArray(); + if (replicas == NULL) return -1; + if (tjsonAddItemToObject(pJson, "replicas", replicas) < 0) return -1; for (int32_t i = 0; i < pOption->numOfReplicas; ++i) { + SJson *replica = tjsonCreateObject(); + if (replica == NULL) return -1; + const SReplica *pReplica = pOption->replicas + i; - if (pReplica != NULL && pReplica->id > 0) { - len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); - len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); - len += snprintf(content + len, maxLen - len, " \"port\": %u\n", pReplica->port); - } - if (i < pOption->numOfReplicas - 1) { - len += snprintf(content + len, maxLen - len, " },{\n"); - } else { - len += snprintf(content + len, maxLen - len, " }],\n"); - } + if (tjsonAddDoubleToObject(replica, "id", pReplica->id) < 0) return -1; + if (tjsonAddStringToObject(replica, "fqdn", pReplica->fqdn) < 0) return -1; + if (tjsonAddDoubleToObject(replica, "port", pReplica->port) < 0) return -1; + if (tjsonAddItemToArray(replicas, replica) < 0) return -1; } } - len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", pOption->deploy); - len += snprintf(content + len, maxLen - len, "}\n"); - taosWriteFile(pFile, content, len); - taosFsyncFile(pFile); + if (tjsonAddDoubleToObject(pJson, "deployed", pOption->deploy) < 0) return -1; + + return 0; +} + +int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) { + int32_t code = -1; + char *buffer = NULL; + SJson *pJson = NULL; + TdFilePtr pFile = NULL; + char file[PATH_MAX] = {0}; + char realfile[PATH_MAX] = {0}; + snprintf(file, sizeof(file), "%s%smnode.json.bak", path, TD_DIRSEP); + snprintf(realfile, sizeof(realfile), "%s%smnode.json", path, TD_DIRSEP); + + terrno = TSDB_CODE_OUT_OF_MEMORY; + pJson = tjsonCreateObject(); + if (pJson == NULL) goto _OVER; + if (mmEncodeOption(pJson, pOption) != 0) goto _OVER; + buffer = tjsonToString(pJson); + if (buffer == NULL) goto _OVER; + terrno = 0; + + pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) goto _OVER; + + int32_t len = strlen(buffer); + if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER; + if (taosFsyncFile(pFile) < 0) goto _OVER; + taosCloseFile(&pFile); - taosMemoryFree(content); + if (taosRenameFile(file, realfile) != 0) goto _OVER; - if (taosRenameFile(file, realfile) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to rename %s since %s", file, terrstr()); - return -1; - } + code = 0; + dInfo("succeed to write mnode file:%s, deloyed:%d", realfile, pOption->deploy); - dDebug("successed to write %s, deployed:%d", realfile, pOption->deploy); - return 0; +_OVER: + if (pJson != NULL) tjsonDelete(pJson); + if (buffer != NULL) taosMemoryFree(buffer); + if (pFile != NULL) taosCloseFile(&pFile); + + if (code != 0) { + if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to write mnode file:%s since %s, deloyed:%d", realfile, terrstr(), pOption->deploy); + } + return code; } diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index 9bd5be520196caea28c13aa9c860c05d28ac7248..e8402eb7c02ff8be9cd5d83b838aa46f6f4f4779 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -58,11 +58,7 @@ static void smProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { dTrace("msg:%p, get from snode-stream queue", pMsg); int32_t code = sndProcessStreamMsg(pMgmt->pSnode, pMsg); if (code < 0) { - if (pMsg) { - dGError("snd, msg:%p failed to process stream msg %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr(code)); - } else { - dGError("snd, msg:%p failed to process stream empty msg since %s", pMsg, terrstr(code)); - } + dGError("snd, msg:%p failed to process stream msg %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr(code)); smSendRsp(pMsg, terrno); } @@ -161,8 +157,10 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { smPutNodeMsgToWriteQueue(pMgmt, pMsg); break; default: - ASSERTS(0, "msg:%p failed to put into snode queue since %s, type:%s qtype:%d", pMsg, terrstr(), - TMSG_INFO(pMsg->msgType), qtype); + terrno = TSDB_CODE_INVALID_PARA; + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); + return -1; } return 0; } diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index b5c554e0caf83724e7e2bb8b157793765527a844..6e724f4d4322c3aee7a143b1cffaa4cae155d3e7 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -26,20 +26,20 @@ extern "C" { #endif typedef struct SVnodeMgmt { - SDnodeData *pData; - SMsgCb msgCb; - const char *path; - const char *name; - SQWorkerPool queryPool; - SQWorkerPool streamPool; - SWWorkerPool fetchPool; - SSingleWorker mgmtWorker; - SHashObj *hash; - TdThreadRwlock lock; - SVnodesStat state; - STfs *pTfs; - TdThread thread; - bool stop; + SDnodeData *pData; + SMsgCb msgCb; + const char *path; + const char *name; + SQWorkerPool queryPool; + SAutoQWorkerPool streamPool; + SWWorkerPool fetchPool; + SSingleWorker mgmtWorker; + SHashObj *hash; + TdThreadRwlock lock; + SVnodesStat state; + STfs *pTfs; + TdThread thread; + bool stop; } SVnodeMgmt; typedef struct { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index a49e855e39ec1f635e48dd9c47b9a4f942864655..bf176ebb40574b5806fe67f5a0b6fb2519886d05 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -14,9 +14,10 @@ */ #define _DEFAULT_SOURCE +#include "tjson.h" #include "vmInt.h" -#define MAX_CONTENT_LEN 1024 * 1024 +#define MAX_CONTENT_LEN 2 * 1024 * 1024 SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes) { taosThreadRwlockRdlock(&pMgmt->lock); @@ -45,162 +46,171 @@ SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes) { return pVnodes; } +static int32_t vmDecodeVnodeList(SJson *pJson, SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) { + int32_t code = -1; + SWrapperCfg *pCfgs = NULL; + *ppCfgs = NULL; + + SJson *vnodes = tjsonGetObjectItem(pJson, "vnodes"); + if (vnodes == NULL) return -1; + + int32_t vnodesNum = cJSON_GetArraySize(vnodes); + if (vnodesNum > 0) { + pCfgs = taosMemoryCalloc(vnodesNum, sizeof(SWrapperCfg)); + if (pCfgs == NULL) return -1; + } + + for (int32_t i = 0; i < vnodesNum; ++i) { + SJson *vnode = tjsonGetArrayItem(vnodes, i); + if (vnode == NULL) goto _OVER; + + SWrapperCfg *pCfg = &pCfgs[i]; + tjsonGetInt32ValueFromDouble(vnode, "vgId", pCfg->vgId, code); + if (code < 0) goto _OVER; + tjsonGetInt32ValueFromDouble(vnode, "dropped", pCfg->dropped, code); + if (code < 0) goto _OVER; + tjsonGetInt32ValueFromDouble(vnode, "vgVersion", pCfg->vgVersion, code); + if (code < 0) goto _OVER; + + snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCfg->vgId); + } + + code = 0; + *ppCfgs = pCfgs; + *numOfVnodes = vnodesNum; + +_OVER: + if (*ppCfgs == NULL) taosMemoryFree(pCfgs); + return code; +} + int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) { - int32_t code = TSDB_CODE_INVALID_JSON_FORMAT; - int32_t len = 0; - int32_t maxLen = MAX_CONTENT_LEN; - char *content = taosMemoryCalloc(1, maxLen + 1); - cJSON *root = NULL; - FILE *fp = NULL; + int32_t code = -1; + TdFilePtr pFile = NULL; + char *pData = NULL; + SJson *pJson = NULL; char file[PATH_MAX] = {0}; SWrapperCfg *pCfgs = NULL; - TdFilePtr pFile = NULL; - snprintf(file, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP); + if (taosStatFile(file, NULL, NULL) < 0) { + dInfo("vnode file:%s not exist", file); + return 0; + } + pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - dDebug("file %s not exist", file); - code = 0; + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to open vnode file:%s since %s", file, terrstr()); goto _OVER; } - if (content == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", file); + int64_t size = 0; + if (taosFStatFile(pFile, &size, NULL) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to fstat mnode file:%s since %s", file, terrstr()); goto _OVER; } - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); + pData = taosMemoryMalloc(size + 1); + if (pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } - cJSON *vnodes = cJSON_GetObjectItem(root, "vnodes"); - if (!vnodes || vnodes->type != cJSON_Array) { - dError("failed to read %s since vnodes not found", file); + if (taosReadFile(pFile, pData, size) != size) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to read vnode file:%s since %s", file, terrstr()); goto _OVER; } - int32_t vnodesNum = cJSON_GetArraySize(vnodes); - if (vnodesNum > 0) { - pCfgs = taosMemoryCalloc(vnodesNum, sizeof(SWrapperCfg)); - if (pCfgs == NULL) { - dError("failed to read %s since out of memory", file); - code = TSDB_CODE_OUT_OF_MEMORY; - goto _OVER; - } - - for (int32_t i = 0; i < vnodesNum; ++i) { - cJSON *vnode = cJSON_GetArrayItem(vnodes, i); - SWrapperCfg *pCfg = &pCfgs[i]; - - cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId"); - if (!vgId || vgId->type != cJSON_Number) { - dError("failed to read %s since vgId not found", file); - taosMemoryFree(pCfgs); - goto _OVER; - } - pCfg->vgId = vgId->valueint; - snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCfg->vgId); - - cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", file); - taosMemoryFree(pCfgs); - goto _OVER; - } - pCfg->dropped = dropped->valueint; + pData[size] = '\0'; - cJSON *vgVersion = cJSON_GetObjectItem(vnode, "vgVersion"); - if (!vgVersion || vgVersion->type != cJSON_Number) { - dError("failed to read %s since vgVersion not found", file); - taosMemoryFree(pCfgs); - goto _OVER; - } - pCfg->vgVersion = vgVersion->valueint; - } + pJson = tjsonParse(pData); + if (pJson == NULL) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; + goto _OVER; + } - *ppCfgs = pCfgs; + if (vmDecodeVnodeList(pJson, pMgmt, ppCfgs, numOfVnodes) < 0) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; + goto _OVER; } - *numOfVnodes = vnodesNum; code = 0; - dDebug("succcessed to read file %s, numOfVnodes:%d", file, vnodesNum); + dInfo("succceed to read vnode file %s", file); _OVER: - if (content != NULL) taosMemoryFree(content); - if (root != NULL) cJSON_Delete(root); + if (pData != NULL) taosMemoryFree(pData); + if (pJson != NULL) cJSON_Delete(pJson); if (pFile != NULL) taosCloseFile(&pFile); - terrno = code; + if (code != 0) { + dError("failed to read vnode file:%s since %s", file, terrstr()); + } return code; } -int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) { - int32_t code = 0; - char file[PATH_MAX] = {0}; - char realfile[PATH_MAX] = {0}; - snprintf(file, sizeof(file), "%s%svnodes.json.bak", pMgmt->path, TD_DIRSEP); - snprintf(realfile, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP); - - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to write %s since %s", file, terrstr()); - return -1; - } - - int32_t numOfVnodes = 0; - SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); - if (ppVnodes == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = -1; - goto _OVER; - } - - int32_t len = 0; - int32_t maxLen = MAX_CONTENT_LEN; - char *content = taosMemoryCalloc(1, maxLen + 1); - if (content == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = -1; - goto _OVER; - } +static int32_t vmEncodeVnodeList(SJson *pJson, SVnodeObj **ppVnodes, int32_t numOfVnodes) { + SJson *vnodes = tjsonCreateArray(); + if (vnodes == NULL) return -1; + if (tjsonAddItemToObject(pJson, "vnodes", vnodes) < 0) return -1; - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"vnodes\": [\n"); for (int32_t i = 0; i < numOfVnodes; ++i) { SVnodeObj *pVnode = ppVnodes[i]; if (pVnode == NULL) continue; - len += snprintf(content + len, maxLen - len, " {\n"); - len += snprintf(content + len, maxLen - len, " \"vgId\": %d,\n", pVnode->vgId); - len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pVnode->dropped); - len += snprintf(content + len, maxLen - len, " \"vgVersion\": %d\n", pVnode->vgVersion); - if (i < numOfVnodes - 1) { - len += snprintf(content + len, maxLen - len, " },\n"); - } else { - len += snprintf(content + len, maxLen - len, " }\n"); - } + SJson *vnode = tjsonCreateObject(); + if (vnode == NULL) return -1; + if (tjsonAddDoubleToObject(vnode, "vgId", pVnode->vgId) < 0) return -1; + if (tjsonAddDoubleToObject(vnode, "dropped", pVnode->dropped) < 0) return -1; + if (tjsonAddDoubleToObject(vnode, "vgVersion", pVnode->vgVersion) < 0) return -1; + if (tjsonAddItemToArray(vnodes, vnode) < 0) return -1; } - len += snprintf(content + len, maxLen - len, " ]\n"); - len += snprintf(content + len, maxLen - len, "}\n"); + + return 0; +} + +int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) { + int32_t code = -1; + char *buffer = NULL; + SJson *pJson = NULL; + TdFilePtr pFile = NULL; + SVnodeObj **ppVnodes = NULL; + char file[PATH_MAX] = {0}; + char realfile[PATH_MAX] = {0}; + snprintf(file, sizeof(file), "%s%svnodes.json.bak", pMgmt->path, TD_DIRSEP); + snprintf(realfile, sizeof(realfile), "%s%svnodes.json", pMgmt->path, TD_DIRSEP); + + int32_t numOfVnodes = 0; + ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); + if (ppVnodes == NULL) goto _OVER; + + terrno = TSDB_CODE_OUT_OF_MEMORY; + pJson = tjsonCreateObject(); + if (pJson == NULL) goto _OVER; + if (vmEncodeVnodeList(pJson, ppVnodes, numOfVnodes) != 0) goto _OVER; + buffer = tjsonToString(pJson); + if (buffer == NULL) goto _OVER; terrno = 0; -_OVER: - taosWriteFile(pFile, content, len); - taosFsyncFile(pFile); + pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) goto _OVER; + + int32_t len = strlen(buffer); + if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER; + if (taosFsyncFile(pFile) < 0) goto _OVER; + taosCloseFile(&pFile); - taosMemoryFree(content); + if (taosRenameFile(file, realfile) != 0) goto _OVER; + code = 0; + dInfo("succeed to write vnodes file:%s, vnodes:%d", realfile, numOfVnodes); + +_OVER: + if (pJson != NULL) tjsonDelete(pJson); + if (buffer != NULL) taosMemoryFree(buffer); + if (pFile != NULL) taosCloseFile(&pFile); if (ppVnodes != NULL) { for (int32_t i = 0; i < numOfVnodes; ++i) { SVnodeObj *pVnode = ppVnodes[i]; @@ -211,8 +221,9 @@ _OVER: taosMemoryFree(ppVnodes); } - if (code != 0) return -1; - - dDebug("successed to write %s, numOfVnodes:%d", realfile, numOfVnodes); - return taosRenameFile(file, realfile); + if (code != 0) { + if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to write vnodes file:%s since %s, vnodes:%d", realfile, terrstr(), numOfVnodes); + } + return code; } \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index bc4677285808e3d8fd76d9f8d65da7e3ab7249f6..220e55f7f38862b0f787a1ee4e13faee0fae44c2 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -132,10 +132,12 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->syncCfg.myIndex = pCreate->selfIndex; pCfg->syncCfg.replicaNum = pCreate->replica; memset(&pCfg->syncCfg.nodeInfo, 0, sizeof(pCfg->syncCfg.nodeInfo)); - for (int i = 0; i < pCreate->replica; ++i) { + for (int32_t i = 0; i < pCreate->replica; ++i) { SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i]; + pNode->nodeId = pCreate->replicas[i].id; pNode->nodePort = pCreate->replicas[i].port; - tstrncpy(pNode->nodeFqdn, pCreate->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); + tstrncpy(pNode->nodeFqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN); + tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); } } @@ -188,8 +190,8 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix, req.replica, req.selfIndex, req.strict); for (int32_t i = 0; i < req.replica; ++i) { - dInfo("vgId:%d, replica:%d id:%d fqdn:%s port:%u", req.vgId, i, req.replicas[i].id, req.replicas[i].fqdn, - req.replicas[i].port); + dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", req.vgId, i, req.replicas[i].fqdn, req.replicas[i].port, + req.replicas[i].id); } SReplica *pReplica = &req.replicas[req.selfIndex]; @@ -213,7 +215,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId); if (pVnode != NULL) { - dDebug("vgId:%d, already exist", req.vgId); + dInfo("vgId:%d, already exist", req.vgId); tFreeSCreateVnodeReq(&req); vmReleaseVnode(pMgmt, pVnode); terrno = TSDB_CODE_VND_ALREADY_EXIST; @@ -286,7 +288,8 @@ int32_t vmProcessAlterVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { dInfo("vgId:%d, start to alter vnode, replica:%d selfIndex:%d strict:%d", alterReq.vgId, alterReq.replica, alterReq.selfIndex, alterReq.strict); for (int32_t i = 0; i < alterReq.replica; ++i) { - dInfo("vgId:%d, replica:%d ep:%s:%u", alterReq.vgId, i, alterReq.replicas[i].fqdn, alterReq.replicas[i].port); + SReplica *pReplica = &alterReq.replicas[i]; + dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", alterReq.vgId, i, pReplica->fqdn, pReplica->port, pReplica->port); } if (alterReq.replica <= 0 || alterReq.selfIndex < 0 || alterReq.selfIndex >= alterReq.replica) { @@ -358,7 +361,7 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t vgId = dropReq.vgId; - dDebug("vgId:%d, start to drop vnode", vgId); + dInfo("vgId:%d, start to drop vnode", vgId); if (dropReq.dnodeId != pMgmt->pData->dnodeId) { terrno = TSDB_CODE_INVALID_MSG; @@ -368,7 +371,7 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); if (pVnode == NULL) { - dDebug("vgId:%d, failed to drop since %s", vgId, terrstr()); + dInfo("vgId:%d, failed to drop since %s", vgId, terrstr()); terrno = TSDB_CODE_VND_NOT_EXIST; return -1; } @@ -383,6 +386,7 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { vmCloseVnode(pMgmt, pVnode); vmWriteVnodeListToFile(pMgmt); + dInfo("vgId:%d, is dropped", vgId); return 0; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 313a88fc5c60f3ac2ae84b1ca6cf811033f5779c..99ba9b9b3bcc1e0efe9d1a1bb987e01c89786374 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -79,6 +79,8 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { char path[TSDB_FILENAME_LEN] = {0}; + vnodeProposeCommitOnNeed(pVnode->pImpl); + taosThreadRwlockWrlock(&pMgmt->lock); taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); taosThreadRwlockUnlock(&pMgmt->lock); @@ -118,6 +120,9 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { dInfo("vgId:%d, all vnode queues is empty", pVnode->vgId); + dInfo("vgId:%d, post close", pVnode->vgId); + vnodePostClose(pVnode->pImpl); + vmFreeQueue(pMgmt, pVnode); vnodeClose(pVnode->pImpl); pVnode->pImpl = NULL; @@ -138,7 +143,7 @@ static void *vmOpenVnodeInThread(void *param) { SVnodeMgmt *pMgmt = pThread->pMgmt; char path[TSDB_FILENAME_LEN]; - dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); + dInfo("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); setThreadName("open-vnodes"); for (int32_t v = 0; v < pThread->vnodeNum; ++v) { @@ -156,14 +161,14 @@ static void *vmOpenVnodeInThread(void *param) { pThread->failed++; } else { vmOpenVnode(pMgmt, pCfg, pImpl); - dDebug("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex); + dInfo("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex); pThread->opened++; atomic_add_fetch_32(&pMgmt->state.openVnodes, 1); } } - dDebug("thread:%d, numOfVnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, - pThread->failed); + dInfo("thread:%d, numOfVnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, + pThread->failed); return NULL; } @@ -338,13 +343,12 @@ static void vmCheckSyncTimeout(SVnodeMgmt *pMgmt) { int32_t numOfVnodes = 0; SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); - for (int32_t i = 0; i < numOfVnodes; ++i) { - SVnodeObj *pVnode = ppVnodes[i]; - vnodeSyncCheckTimeout(pVnode->pImpl); - vmReleaseVnode(pMgmt, pVnode); - } - if (ppVnodes != NULL) { + for (int32_t i = 0; i < numOfVnodes; ++i) { + SVnodeObj *pVnode = ppVnodes[i]; + vnodeSyncCheckTimeout(pVnode->pImpl); + vmReleaseVnode(pMgmt, pVnode); + } taosMemoryFree(ppVnodes); } } @@ -496,7 +500,7 @@ static void *vmRestoreVnodeInThread(void *param) { dError("vgId:%d, failed to restore vnode by thread:%d", pVnode->vgId, pThread->threadIndex); pThread->failed++; } else { - dDebug("vgId:%d, is restored by thread:%d", pVnode->vgId, pThread->threadIndex); + dInfo("vgId:%d, is restored by thread:%d", pVnode->vgId, pThread->threadIndex); pThread->opened++; atomic_add_fetch_32(&pMgmt->state.openVnodes, 1); } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 7e3915f3d1dcdfc713746aad6af1bd609231d001..cd29b115508ccfcd51d9c130c4847b2ef265dba5 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -86,12 +86,8 @@ static void vmProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); if (code != 0) { if (terrno != 0) code = terrno; - if (pMsg) { - dGError("vgId:%d, msg:%p failed to process stream msg %s since %s", pVnode->vgId, pMsg, TMSG_INFO(pMsg->msgType), - terrstr(code)); - } else { - dGError("vgId:%d, msg:%p failed to process stream empty msg since %s", pVnode->vgId, pMsg, terrstr(code)); - } + dGError("vgId:%d, msg:%p failed to process stream msg %s since %s", pVnode->vgId, pMsg, TMSG_INFO(pMsg->msgType), + terrstr(code)); vmSendRsp(pMsg, code); } @@ -138,19 +134,34 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf } } +static void vmSendResponse(SRpcMsg *pMsg) { + if (pMsg->info.handle) { + SRpcMsg rsp = {.info = pMsg->info, .code = terrno}; + rpcSendResponse(&rsp); + } +} + static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) { const STraceId *trace = &pMsg->info.traceId; - SMsgHead *pHead = pMsg->pCont; - int32_t code = 0; + if (pMsg->contLen < sizeof(SMsgHead)) { + dGError("invalid rpc msg with no msg head at pCont. pMsg:%p, type:%s, contLen:%d", pMsg, TMSG_INFO(pMsg->msgType), + pMsg->contLen); + return -1; + } + + SMsgHead *pHead = pMsg->pCont; + int32_t code = 0; pHead->contLen = ntohl(pHead->contLen); pHead->vgId = ntohl(pHead->vgId); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) { - dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg, terrstr(), - TMSG_INFO(pMsg->msgType), qtype, pHead->contLen); - return terrno != 0 ? terrno : -1; + dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg, + terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen); + terrno = (terrno != 0) ? terrno : -1; + vmSendResponse(pMsg); + return terrno; } switch (qtype) { @@ -233,6 +244,14 @@ int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { + if (pRpc->contLen < sizeof(SMsgHead)) { + dError("invalid rpc msg with no msg head at pCont. pRpc:%p, type:%s, len:%d", pRpc, TMSG_INFO(pRpc->msgType), + pRpc->contLen); + rpcFreeCont(pRpc->pCont); + pRpc->pCont = NULL; + return -1; + } + SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen); if (pMsg == NULL) { rpcFreeCont(pRpc->pCont); @@ -304,7 +323,7 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { (void)tMultiWorkerInit(&pVnode->pApplyW, &acfg); pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue); - pVnode->pStreamQ = tQWorkerAllocQueue(&pMgmt->streamPool, pVnode, (FItem)vmProcessStreamQueue); + pVnode->pStreamQ = tAutoQWorkerAllocQueue(&pMgmt->streamPool, pVnode, (FItem)vmProcessStreamQueue); pVnode->pFetchQ = tWWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItems)vmProcessFetchQueue); if (pVnode->pWriteW.queue == NULL || pVnode->pSyncW.queue == NULL || pVnode->pSyncCtrlW.queue == NULL || @@ -330,7 +349,7 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ); - tQWorkerFreeQueue(&pMgmt->streamPool, pVnode->pStreamQ); + tAutoQWorkerFreeQueue(&pMgmt->streamPool, pVnode->pStreamQ); tWWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); pVnode->pQueryQ = NULL; pVnode->pStreamQ = NULL; @@ -345,11 +364,10 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) { pQPool->max = tsNumOfVnodeQueryThreads; if (tQWorkerInit(pQPool) != 0) return -1; - SQWorkerPool *pStreamPool = &pMgmt->streamPool; + SAutoQWorkerPool *pStreamPool = &pMgmt->streamPool; pStreamPool->name = "vnode-stream"; - pStreamPool->min = tsNumOfVnodeStreamThreads; - pStreamPool->max = tsNumOfVnodeStreamThreads; - if (tQWorkerInit(pStreamPool) != 0) return -1; + pStreamPool->ratio = tsRatioOfVnodeStreamThreads; + if (tAutoQWorkerInit(pStreamPool) != 0) return -1; SWWorkerPool *pFPool = &pMgmt->fetchPool; pFPool->name = "vnode-fetch"; @@ -371,7 +389,7 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) { void vmStopWorker(SVnodeMgmt *pMgmt) { tQWorkerCleanup(&pMgmt->queryPool); - tQWorkerCleanup(&pMgmt->streamPool); + tAutoQWorkerCleanup(&pMgmt->streamPool); tWWorkerCleanup(&pMgmt->fetchPool); dDebug("vnode workers are closed"); } diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h index 7e85e6b72239260d13aba44306309cd2528c3364..02cd6784331b0f7c9f49238f537d70bf36bf0a12 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h @@ -85,6 +85,7 @@ typedef struct SDnode { // dmEnv.c SDnode *dmInstance(); void dmReportStartup(const char *pName, const char *pDesc); +int64_t dmGetClusterId(); // dmMgmt.c int32_t dmInitDnode(SDnode *pDnode); diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index e3bda5a3f01355f33bdf758e0a14824162c50a2b..acf96ad397c405c35fe7089aa4d618dcd8eb13e5 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -16,9 +16,9 @@ #define _DEFAULT_SOURCE #include "dmMgmt.h" -static SDnode global = {0}; +static SDnode globalDnode = {0}; -SDnode *dmInstance() { return &global; } +SDnode *dmInstance() { return &globalDnode; } static int32_t dmCheckRepeatInit(SDnode *pDnode) { if (atomic_val_compare_exchange_8(&pDnode->once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { @@ -268,3 +268,8 @@ void dmReportStartup(const char *pName, const char *pDesc) { tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); dDebug("step:%s, %s", pStartup->name, pStartup->desc); } + +int64_t dmGetClusterId() { + return globalDnode.data.clusterId; +} + diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index 981797834ae268e2eeab923a61fa98837a1d4de0..16931ab6dfa1a67f37792b19946c337461108062 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -109,8 +109,8 @@ static int32_t dmStartNodes(SDnode *pDnode) { } } - dInfo("TDengine initialized successfully"); - dmReportStartup("TDengine", "initialized successfully"); + dInfo("The daemon initialized successfully"); + dmReportStartup("The daemon", "initialized successfully"); return 0; } @@ -142,7 +142,7 @@ int32_t dmRunDnode(SDnode *pDnode) { while (1) { if (pDnode->stop) { - dInfo("TDengine is about to stop"); + dInfo("The daemon is about to stop"); dmSetStatus(pDnode, DND_STAT_STOPPED); dmStopNodes(pDnode); dmCloseNodes(pDnode); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index f11378f84ce216af5c6e8a5f8e73bc4dc46290c0..dcb63f65246cfc103790c2e46b2ca4c8bd4beafe 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -111,12 +111,12 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { dGError("msg:%p, type:%s pCont is NULL", pRpc, TMSG_INFO(pRpc->msgType)); terrno = TSDB_CODE_INVALID_MSG_LEN; goto _OVER; - } /* else if ((pRpc->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pRpc->code == TSDB_CODE_RPC_BROKEN_LINK) && - (!IsReq(pRpc)) && (pRpc->pCont == NULL)) { - dGError("msg:%p, type:%s pCont is NULL, err: %s", pRpc, TMSG_INFO(pRpc->msgType), tstrerror(pRpc->code)); - terrno = pRpc->code; - goto _OVER; - }*/ + } else if ((pRpc->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pRpc->code == TSDB_CODE_RPC_BROKEN_LINK) && + (!IsReq(pRpc)) && (pRpc->pCont == NULL)) { + dGError("msg:%p, type:%s pCont is NULL, err: %s", pRpc, TMSG_INFO(pRpc->msgType), tstrerror(pRpc->code)); + terrno = pRpc->code; + goto _OVER; + } if (pHandle->defaultNtype == NODE_END) { dGError("msg:%p, type:%s not processed since no handle", pRpc, TMSG_INFO(pRpc->msgType)); @@ -248,8 +248,9 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { rpcRe static bool rpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_MNODE_NOT_FOUND || - code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_VND_STOPPED || - code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) { + code == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED || code == TSDB_CODE_SYN_NOT_LEADER || + code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_VND_STOPPED || code == TSDB_CODE_APP_IS_STARTING || + code == TSDB_CODE_APP_IS_STOPPING) { if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) { return false; @@ -344,6 +345,8 @@ SMsgCb dmGetMsgcb(SDnode *pDnode) { .registerBrokenLinkArgFp = dmRegisterBrokenLinkArg, .releaseHandleFp = dmReleaseHandle, .reportStartupFp = dmReportStartup, + .updateDnodeInfoFp = dmUpdateDnodeInfo, + .data = &pDnode->data, }; return msgCb; } diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 2124b387ec926a3add0c5efe3a51f43bdf9ae47c..8ecce20f537d429775b3f827e08019f965c4ba39 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -167,6 +167,7 @@ void dmUpdateEps(SDnodeData *pData, SArray *pDnodeEps); void dmGetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet); void dmGetMnodeEpSetForRedirect(SDnodeData *pData, SRpcMsg *pMsg, SEpSet *pEpSet); void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet); +void dmUpdateDnodeInfo(void *pData, int32_t *dnodeId, int64_t *clusterId, char *fqdn, uint16_t *port); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/node_util/src/dmEps.c b/source/dnode/mgmt/node_util/src/dmEps.c index 2ced9a350d8d95f1d30371317b30795fdcdec4c1..9640c0b5c95710e5479e7842769a1d6bc57eb5a9 100644 --- a/source/dnode/mgmt/node_util/src/dmEps.c +++ b/source/dnode/mgmt/node_util/src/dmEps.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "dmUtil.h" +#include "tjson.h" #include "tmisce.h" static void dmPrintEps(SDnodeData *pData); @@ -40,14 +41,49 @@ static void dmGetDnodeEp(SDnodeData *pData, int32_t dnodeId, char *pEp, char *pF taosThreadRwlockUnlock(&pData->lock); } +static int32_t dmDecodeEps(SJson *pJson, SDnodeData *pData) { + int32_t code = 0; + + tjsonGetInt32ValueFromDouble(pJson, "dnodeId", pData->dnodeId, code); + if (code < 0) return -1; + tjsonGetNumberValue(pJson, "dnodeVer", pData->dnodeVer, code); + if (code < 0) return -1; + tjsonGetNumberValue(pJson, "clusterId", pData->clusterId, code); + if (code < 0) return -1; + tjsonGetInt32ValueFromDouble(pJson, "dropped", pData->dropped, code); + if (code < 0) return -1; + + SJson *dnodes = tjsonGetObjectItem(pJson, "dnodes"); + if (dnodes == NULL) return 0; + int32_t numOfDnodes = tjsonGetArraySize(dnodes); + + for (int32_t i = 0; i < numOfDnodes; ++i) { + SJson *dnode = tjsonGetArrayItem(dnodes, i); + if (dnode == NULL) return -1; + + SDnodeEp dnodeEp = {0}; + tjsonGetInt32ValueFromDouble(dnode, "id", dnodeEp.id, code); + if (code < 0) return -1; + code = tjsonGetStringValue(dnode, "fqdn", dnodeEp.ep.fqdn); + if (code < 0) return -1; + tjsonGetUInt16ValueFromDouble(dnode, "port", dnodeEp.ep.port, code); + if (code < 0) return -1; + tjsonGetInt8ValueFromDouble(dnode, "isMnode", dnodeEp.isMnode, code); + if (code < 0) return -1; + + if (taosArrayPush(pData->dnodeEps, &dnodeEp) == NULL) return -1; + } + + return 0; +} + int32_t dmReadEps(SDnodeData *pData) { - int32_t code = TSDB_CODE_INVALID_JSON_FORMAT; - int32_t len = 0; - int32_t maxLen = 256 * 1024; - char *content = taosMemoryCalloc(1, maxLen + 1); - cJSON *root = NULL; - char file[PATH_MAX] = {0}; + int32_t code = -1; TdFilePtr pFile = NULL; + char *content = NULL; + SJson *pJson = NULL; + char file[PATH_MAX] = {0}; + snprintf(file, sizeof(file), "%s%sdnode%sdnode.json", tsDataDir, TD_DIRSEP, TD_DIRSEP); pData->dnodeEps = taosArrayInit(1, sizeof(SDnodeEp)); if (pData->dnodeEps == NULL) { @@ -55,113 +91,63 @@ int32_t dmReadEps(SDnodeData *pData) { goto _OVER; } - snprintf(file, sizeof(file), "%s%sdnode%sdnode.json", tsDataDir, TD_DIRSEP, TD_DIRSEP); - pFile = taosOpenFile(file, TD_FILE_READ); - if (pFile == NULL) { + if (taosStatFile(file, NULL, NULL) < 0) { + dInfo("dnode file:%s not exist", file); code = 0; goto _OVER; } - len = (int32_t)taosReadFile(pFile, content, maxLen); - if (len <= 0) { - dError("failed to read %s since content is null", file); - goto _OVER; - } - - content[len] = 0; - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); + pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to open dnode file:%s since %s", file, terrstr()); goto _OVER; } - cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId"); - if (!dnodeId || dnodeId->type != cJSON_Number) { - dError("failed to read %s since dnodeId not found", file); + int64_t size = 0; + if (taosFStatFile(pFile, &size, NULL) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to fstat dnode file:%s since %s", file, terrstr()); goto _OVER; } - pData->dnodeId = dnodeId->valueint; - cJSON *dnodeVer = cJSON_GetObjectItem(root, "dnodeVer"); - if (!dnodeVer || dnodeVer->type != cJSON_String) { - dError("failed to read %s since dnodeVer not found", file); + content = taosMemoryMalloc(size + 1); + if (content == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } - pData->dnodeVer = atoll(dnodeVer->valuestring); - cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId"); - if (!clusterId || clusterId->type != cJSON_String) { - dError("failed to read %s since clusterId not found", file); + if (taosReadFile(pFile, content, size) != size) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to read dnode file:%s since %s", file, terrstr()); goto _OVER; } - pData->clusterId = atoll(clusterId->valuestring); - cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); - if (!dropped || dropped->type != cJSON_Number) { - dError("failed to read %s since dropped not found", file); - goto _OVER; - } - pData->dropped = dropped->valueint; + content[size] = '\0'; - cJSON *dnodes = cJSON_GetObjectItem(root, "dnodes"); - if (!dnodes || dnodes->type != cJSON_Array) { - dError("failed to read %s since dnodes not found", file); + pJson = tjsonParse(content); + if (pJson == NULL) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; goto _OVER; } - int32_t numOfDnodes = cJSON_GetArraySize(dnodes); - if (numOfDnodes <= 0) { - dError("failed to read %s since numOfDnodes:%d invalid", file, numOfDnodes); + if (dmDecodeEps(pJson, pData) < 0) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; goto _OVER; } - for (int32_t i = 0; i < numOfDnodes; ++i) { - cJSON *node = cJSON_GetArrayItem(dnodes, i); - if (node == NULL) break; - - SDnodeEp dnodeEp = {0}; - - cJSON *did = cJSON_GetObjectItem(node, "id"); - if (!did || did->type != cJSON_Number) { - dError("failed to read %s since dnodeId not found", file); - goto _OVER; - } - - dnodeEp.id = did->valueint; - - cJSON *dnodeFqdn = cJSON_GetObjectItem(node, "fqdn"); - if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) { - dError("failed to read %s since dnodeFqdn not found", file); - goto _OVER; - } - tstrncpy(dnodeEp.ep.fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN); - - cJSON *dnodePort = cJSON_GetObjectItem(node, "port"); - if (!dnodePort || dnodePort->type != cJSON_Number) { - dError("failed to read %s since dnodePort not found", file); - goto _OVER; - } - - dnodeEp.ep.port = dnodePort->valueint; - - cJSON *isMnode = cJSON_GetObjectItem(node, "isMnode"); - if (!isMnode || isMnode->type != cJSON_Number) { - dError("failed to read %s since isMnode not found", file); - goto _OVER; - } - dnodeEp.isMnode = isMnode->valueint; - - taosArrayPush(pData->dnodeEps, &dnodeEp); - } - code = 0; - dDebug("succcessed to read file %s", file); + dInfo("succceed to read mnode file %s", file); _OVER: if (content != NULL) taosMemoryFree(content); - if (root != NULL) cJSON_Delete(root); + if (pJson != NULL) cJSON_Delete(pJson); if (pFile != NULL) taosCloseFile(&pFile); + if (code != 0) { + dError("failed to read dnode file:%s since %s", file, terrstr()); + } + if (taosArrayGetSize(pData->dnodeEps) == 0) { SDnodeEp dnodeEp = {0}; dnodeEp.isMnode = 1; @@ -177,64 +163,77 @@ _OVER: return -1; } - terrno = code; return code; } -int32_t dmWriteEps(SDnodeData *pData) { - char file[PATH_MAX] = {0}; - char realfile[PATH_MAX] = {0}; - - snprintf(file, sizeof(file), "%s%sdnode%sdnode.json.bak", tsDataDir, TD_DIRSEP, TD_DIRSEP); - snprintf(realfile, sizeof(realfile), "%s%sdnode%sdnode.json", tsDataDir, TD_DIRSEP, TD_DIRSEP); - - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - dError("failed to write %s since %s", file, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - int32_t len = 0; - int32_t maxLen = 256 * 1024; - char *content = taosMemoryCalloc(1, maxLen + 1); +static int32_t dmEncodeEps(SJson *pJson, SDnodeData *pData) { + if (tjsonAddDoubleToObject(pJson, "dnodeId", pData->dnodeId) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "dnodeVer", pData->dnodeVer) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "clusterId", pData->clusterId) < 0) return -1; + if (tjsonAddDoubleToObject(pJson, "dropped", pData->dropped) < 0) return -1; - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", pData->dnodeId); - len += snprintf(content + len, maxLen - len, " \"dnodeVer\": \"%" PRId64 "\",\n", pData->dnodeVer); - len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%" PRId64 "\",\n", pData->clusterId); - len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pData->dropped); - len += snprintf(content + len, maxLen - len, " \"dnodes\": [{\n"); + SJson *dnodes = tjsonCreateArray(); + if (dnodes == NULL) return -1; + if (tjsonAddItemToObject(pJson, "dnodes", dnodes) < 0) return -1; int32_t numOfEps = (int32_t)taosArrayGetSize(pData->dnodeEps); for (int32_t i = 0; i < numOfEps; ++i) { SDnodeEp *pDnodeEp = taosArrayGet(pData->dnodeEps, i); - len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pDnodeEp->id); - len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pDnodeEp->ep.fqdn); - len += snprintf(content + len, maxLen - len, " \"port\": %u,\n", pDnodeEp->ep.port); - len += snprintf(content + len, maxLen - len, " \"isMnode\": %d\n", pDnodeEp->isMnode); - if (i < numOfEps - 1) { - len += snprintf(content + len, maxLen - len, " },{\n"); - } else { - len += snprintf(content + len, maxLen - len, " }]\n"); - } + SJson *dnode = tjsonCreateObject(); + if (dnode == NULL) return -1; + + if (tjsonAddDoubleToObject(dnode, "id", pDnodeEp->id) < 0) return -1; + if (tjsonAddStringToObject(dnode, "fqdn", pDnodeEp->ep.fqdn) < 0) return -1; + if (tjsonAddDoubleToObject(dnode, "port", pDnodeEp->ep.port) < 0) return -1; + if (tjsonAddDoubleToObject(dnode, "isMnode", pDnodeEp->isMnode) < 0) return -1; + if (tjsonAddItemToArray(dnodes, dnode) < 0) return -1; } - len += snprintf(content + len, maxLen - len, "}\n"); - taosWriteFile(pFile, content, len); - taosFsyncFile(pFile); - taosCloseFile(&pFile); - taosMemoryFree(content); + return 0; +} - if (taosRenameFile(file, realfile) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to rename %s since %s", file, terrstr()); - return -1; - } +int32_t dmWriteEps(SDnodeData *pData) { + int32_t code = -1; + char *buffer = NULL; + SJson *pJson = NULL; + TdFilePtr pFile = NULL; + char file[PATH_MAX] = {0}; + char realfile[PATH_MAX] = {0}; + snprintf(file, sizeof(file), "%s%sdnode%sdnode.json.bak", tsDataDir, TD_DIRSEP, TD_DIRSEP); + snprintf(realfile, sizeof(realfile), "%s%sdnode%sdnode.json", tsDataDir, TD_DIRSEP, TD_DIRSEP); + + terrno = TSDB_CODE_OUT_OF_MEMORY; + pJson = tjsonCreateObject(); + if (pJson == NULL) goto _OVER; + if (dmEncodeEps(pJson, pData) != 0) goto _OVER; + buffer = tjsonToString(pJson); + if (buffer == NULL) goto _OVER; + terrno = 0; + + pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) goto _OVER; + + int32_t len = strlen(buffer); + if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER; + if (taosFsyncFile(pFile) < 0) goto _OVER; + + taosCloseFile(&pFile); + if (taosRenameFile(file, realfile) != 0) goto _OVER; + code = 0; pData->updateTime = taosGetTimestampMs(); - dDebug("successed to write %s, dnodeVer:%" PRId64, realfile, pData->dnodeVer); - return 0; + dInfo("succeed to write dnode file:%s, dnodeVer:%" PRId64, realfile, pData->dnodeVer); + +_OVER: + if (pJson != NULL) tjsonDelete(pJson); + if (buffer != NULL) taosMemoryFree(buffer); + if (pFile != NULL) taosCloseFile(&pFile); + + if (code != 0) { + if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to write dnode file:%s since %s, dnodeVer:%" PRId64, realfile, terrstr(), pData->dnodeVer); + } + return code; } void dmUpdateEps(SDnodeData *pData, SArray *eps) { @@ -332,3 +331,41 @@ void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet) { dInfo("mnode index:%d %s:%u", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); } } + +void dmUpdateDnodeInfo(void *data, int32_t *dnodeId, int64_t *clusterId, char *fqdn, uint16_t *port) { + SDnodeData *pData = data; + int32_t ret = -1; + taosThreadRwlockRdlock(&pData->lock); + if (*dnodeId <= 0) { + for (int32_t i = 0; i < (int32_t)taosArrayGetSize(pData->dnodeEps); ++i) { + SDnodeEp *pDnodeEp = taosArrayGet(pData->dnodeEps, i); + if (strcmp(pDnodeEp->ep.fqdn, fqdn) == 0 && pDnodeEp->ep.port == *port) { + dInfo("dnode:%s:%u, update dnodeId from %d to %d", fqdn, *port, *dnodeId, pDnodeEp->id); + *dnodeId = pDnodeEp->id; + if (clusterId != NULL) *clusterId = pData->clusterId; + ret = 0; + } + } + if (ret != 0) { + dInfo("dnode:%s:%u, failed to update dnodeId:%d", fqdn, *port, *dnodeId); + } + } else { + SDnodeEp *pDnodeEp = taosHashGet(pData->dnodeHash, dnodeId, sizeof(int32_t)); + if (pDnodeEp) { + if (strcmp(pDnodeEp->ep.fqdn, fqdn) != 0) { + dInfo("dnode:%d, update port from %s to %s", *dnodeId, fqdn, pDnodeEp->ep.fqdn); + tstrncpy(fqdn, pDnodeEp->ep.fqdn, TSDB_FQDN_LEN); + } + if (pDnodeEp->ep.port != *port) { + dInfo("dnode:%d, update port from %u to %u", *dnodeId, *port, pDnodeEp->ep.port); + *port = pDnodeEp->ep.port; + } + if (clusterId != NULL) *clusterId = pData->clusterId; + ret = 0; + } else { + dInfo("dnode:%d, failed to update dnode info", *dnodeId); + } + } + taosThreadRwlockUnlock(&pData->lock); + // return ret; +} \ No newline at end of file diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index d387fe4a3f52400e3dcb5521d4c7a94c8802d607..fb05f08c0c5e26130c399e179052c81dae56ebf8 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -15,104 +15,133 @@ #define _DEFAULT_SOURCE #include "dmUtil.h" +#include "tjson.h" #define MAXLEN 1024 +static int32_t dmDecodeFile(SJson *pJson, bool *deployed) { + int32_t code = 0; + int32_t value = 0; + + tjsonGetInt32ValueFromDouble(pJson, "deployed", value, code); + if (code < 0) return -1; + + *deployed = (value != 0); + return code; +} + int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) { - int32_t code = TSDB_CODE_INVALID_JSON_FORMAT; - int64_t len = 0; - char content[MAXLEN + 1] = {0}; - cJSON *root = NULL; - char file[PATH_MAX] = {0}; + int32_t code = -1; TdFilePtr pFile = NULL; - + char *content = NULL; + SJson *pJson = NULL; + char file[PATH_MAX] = {0}; snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name); + + if (taosStatFile(file, NULL, NULL) < 0) { + dInfo("file:%s not exist", file); + code = 0; + goto _OVER; + } + pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - code = 0; + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to open file:%s since %s", file, terrstr()); goto _OVER; } - len = taosReadFile(pFile, content, MAXLEN); - if (len <= 0) { - dError("failed to read %s since content is null", file); + int64_t size = 0; + if (taosFStatFile(pFile, &size, NULL) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to fstat file:%s since %s", file, terrstr()); goto _OVER; } - root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read %s since invalid json format", file); + content = taosMemoryMalloc(size + 1); + if (content == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } - cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); - if (!deployed || deployed->type != cJSON_Number) { - dError("failed to read %s since deployed not found", file); + if (taosReadFile(pFile, content, size) != size) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to read file:%s since %s", file, terrstr()); + goto _OVER; + } + + content[size] = '\0'; + + pJson = tjsonParse(content); + if (pJson == NULL) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; + goto _OVER; + } + + if (dmDecodeFile(pJson, pDeployed) < 0) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; goto _OVER; } - *pDeployed = deployed->valueint != 0; - dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); code = 0; + dInfo("succceed to read mnode file %s", file); _OVER: - if (root != NULL) cJSON_Delete(root); + if (content != NULL) taosMemoryFree(content); + if (pJson != NULL) cJSON_Delete(pJson); if (pFile != NULL) taosCloseFile(&pFile); - terrno = code; + if (code != 0) { + dError("failed to read dnode file:%s since %s", file, terrstr()); + } return code; } +static int32_t dmEncodeFile(SJson *pJson, bool deployed) { + if (tjsonAddDoubleToObject(pJson, "deployed", deployed) < 0) return -1; + return 0; +} + int32_t dmWriteFile(const char *path, const char *name, bool deployed) { int32_t code = -1; - int32_t len = 0; - char content[MAXLEN + 1] = {0}; + char *buffer = NULL; + SJson *pJson = NULL; + TdFilePtr pFile = NULL; char file[PATH_MAX] = {0}; char realfile[PATH_MAX] = {0}; - TdFilePtr pFile = NULL; - snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name); snprintf(realfile, sizeof(realfile), "%s%s%s.json", path, TD_DIRSEP, name); - pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to write %s since %s", file, terrstr()); - goto _OVER; - } - - len += snprintf(content + len, MAXLEN - len, "{\n"); - len += snprintf(content + len, MAXLEN - len, " \"deployed\": %d\n", deployed); - len += snprintf(content + len, MAXLEN - len, "}\n"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + pJson = tjsonCreateObject(); + if (pJson == NULL) goto _OVER; + if (dmEncodeFile(pJson, deployed) != 0) goto _OVER; + buffer = tjsonToString(pJson); + if (buffer == NULL) goto _OVER; + terrno = 0; - if (taosWriteFile(pFile, content, len) != len) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to write file:%s since %s", file, terrstr()); - goto _OVER; - } + pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) goto _OVER; - if (taosFsyncFile(pFile) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to fsync file:%s since %s", file, terrstr()); - goto _OVER; - } + int32_t len = strlen(buffer); + if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER; + if (taosFsyncFile(pFile) < 0) goto _OVER; taosCloseFile(&pFile); + if (taosRenameFile(file, realfile) != 0) goto _OVER; - if (taosRenameFile(file, realfile) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to rename %s since %s", file, terrstr()); - return -1; - } - - dInfo("successed to write %s, deployed:%d", realfile, deployed); code = 0; + dInfo("succeed to write file:%s, deloyed:%d", realfile, deployed); _OVER: - if (pFile != NULL) { - taosCloseFile(&pFile); - } + if (pJson != NULL) tjsonDelete(pJson); + if (buffer != NULL) taosMemoryFree(buffer); + if (pFile != NULL) taosCloseFile(&pFile); + if (code != 0) { + if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to write file:%s since %s, deloyed:%d", realfile, terrstr(), deployed); + } return code; } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 4e93a1d96ee0eb3b70742eee69d892b8cb1dc451..1cbd9bfb66b92b63dd2a01285b609eed671805f7 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -193,6 +193,7 @@ typedef struct { int64_t lastAccessTime; int32_t accessTimes; int32_t numOfVnodes; + int32_t numOfOtherNodes; int32_t numOfSupportVnodes; float numOfCores; int64_t memTotal; @@ -443,6 +444,7 @@ typedef struct { STableMetaRsp* pMeta; bool sysDbRsp; char db[TSDB_DB_FNAME_LEN]; + char filterTb[TSDB_TABLE_NAME_LEN]; } SShowObj; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index cf7deba397556f8df3550067057a03f6ca374a2a..8943ba703ee47442a49fe25ab44ed7fcfb7867cd 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -34,6 +34,8 @@ SHashObj *mndDupDbHash(SHashObj *pOld); SHashObj *mndDupTopicHash(SHashObj *pOld); int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen); +int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db); +int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index ca03207d2b95bab562c48a02ca383d67bba7349d..e0d8ecb3eb949d4161f5330d62250b3267ed513b 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -20,6 +20,8 @@ #define CLUSTER_VER_NUMBE 1 #define CLUSTER_RESERVE_SIZE 60 +char tsVersionName[16] = "community"; +int64_t tsExpireTime = 0; static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster); static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw); @@ -291,6 +293,18 @@ static int32_t mndRetrieveClusters(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock * pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pCluster->createdTime, false); + char ver[12] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(ver, tsVersionName, pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)ver, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + if (tsExpireTime <= 0) { + colDataAppendNULL(pColInfo, numOfRows); + } else { + colDataAppend(pColInfo, numOfRows, (const char *)&tsExpireTime, false); + } + sdbRelease(pSdb, pCluster); numOfRows++; } diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index cf2386348ee8486984c0fbe2be1b8ea6b25ea536..b2ca1e7ad8f19c016a155e462c693fda50e47027 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -558,6 +558,10 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { goto SUBSCRIBE_OVER; } + if (mndCheckTopicPrivilege(pMnode, pMsg->info.conn.user, MND_OPER_SUBSCRIBE, pTopic) != 0) { + goto SUBSCRIBE_OVER; + } + mndReleaseTopic(pMnode, pTopic); } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 8a4f6c2195ba88b1adea293690b1ba209c6c91a8..7e5c29d56f11211b9475834e8c4a666547056398 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -112,7 +112,6 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT8(pRaw, dataPos, pDb->cfg.hashMethod, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfRetensions, _OVER) for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) { - ASSERT(taosArrayGetSize(pDb->cfg.pRetensions) == pDb->cfg.numOfRetensions); SRetention *pRetension = taosArrayGet(pDb->cfg.pRetensions, i); SDB_SET_INT64(pRaw, dataPos, pRetension->freq, _OVER) SDB_SET_INT64(pRaw, dataPos, pRetension->keep, _OVER) @@ -364,6 +363,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->hashPrefix < TSDB_MIN_HASH_PREFIX || pCfg->hashPrefix > TSDB_MAX_HASH_PREFIX) return -1; if (pCfg->hashSuffix < TSDB_MIN_HASH_SUFFIX || pCfg->hashSuffix > TSDB_MAX_HASH_SUFFIX) return -1; if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1; + if (taosArrayGetSize(pCfg->pRetensions) != pCfg->numOfRetensions) return -1; terrno = 0; return terrno; @@ -1051,17 +1051,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { if (mndDropStreamByDb(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndDropSmasByDb(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER; - - SUserObj *pUser = mndAcquireUser(pMnode, pDb->createUser); - if (pUser != NULL) { - pUser->authVersion++; - SSdbRaw *pCommitRaw = mndUserActionEncode(pUser); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - goto _OVER; - } - (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - } + if (mndUserRemoveDb(pMnode, pTrans, pDb->name) != 0) goto _OVER; int32_t rspLen = 0; void *pRsp = NULL; diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index b5c2fb05b3adb3f3e7d0c3ce19f3da055fabdb3c..8983d73c7078d09ec82cefc7d54592f9457330ef 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -167,6 +167,10 @@ void tFreeStreamObj(SStreamObj *pStream) { taosArrayDestroy(pLevel); } taosArrayDestroy(pStream->tasks); + // tagSchema.pSchema + if (pStream->tagSchema.nCols > 0) { + taosMemoryFree(pStream->tagSchema.pSchema); + } } SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) { @@ -489,7 +493,7 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { tlen += tEncodeSMqConsumerEp(buf, pConsumerEp); cnt++; } - ASSERT(cnt == sz); + if(cnt != sz) return -1; tlen += taosEncodeArray(buf, pSub->unassignedVgs, (FEncode)tEncodeSMqVgEp); tlen += taosEncodeString(buf, pSub->dbName); return tlen; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 58ae85a62865d56030724bc6f555a62256ab4575..ddb54a95ead10ec1e129cd4337a53335d8cf8da1 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -308,7 +308,8 @@ void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeEps) { void *pIter = NULL; while (1) { SDnodeObj *pDnode = NULL; - pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); + ESdbStatus objStatus = 0; + pIter = sdbFetchAll(pSdb, SDB_DNODE, pIter, (void **)&pDnode, &objStatus, true); if (pIter == NULL) break; SDnodeEp dnodeEp = {0}; @@ -397,8 +398,6 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { bool reboot = (pDnode->rebootTime != statusReq.rebootTime); bool needCheck = !online || dnodeChanged || reboot; - pDnode->accessTimes++; - pDnode->lastAccessTime = curMs; const STraceId *trace = &pReq->info.traceId; mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id, pDnode->accessTimes, needCheck, online, reboot, dnodeChanged, statusReq.statusSeq); @@ -534,6 +533,8 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { pReq->info.rsp = pHead; } + pDnode->accessTimes++; + pDnode->lastAccessTime = curMs; code = 0; _OVER: @@ -1050,7 +1051,7 @@ static int32_t mndRetrieveDnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB status = "offline"; } - char b1[9] = {0}; + char b1[16] = {0}; STR_TO_VARSTR(b1, status); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, b1, false); diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 31f31a15baf31813aeb071ff933ff5263d5976f8..244e6058d4a493a8d9f6b2cf794cca6a3db2cdad 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -293,7 +293,7 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { goto _OVER; } - mInfo("func:%s, start to create", createReq.name); + mInfo("func:%s, start to create, size:%d", createReq.name, createReq.codeLen); if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index b9448797eafd6570dcdcbaf04fe15379ae8f2bec..7dcd287fb77c66bf1c0cfea6e7cd7fa793c5b61f 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndMnode.h" +#include "mndCluster.h" #include "mndDnode.h" #include "mndPrivilege.h" #include "mndShow.h" @@ -180,9 +181,8 @@ _OVER: static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) { mTrace("mnode:%d, perform insert action, row:%p", pObj->id, pObj); - pObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pObj->id); + pObj->pDnode = sdbAcquireNotReadyObj(pSdb, SDB_DNODE, &pObj->id); if (pObj->pDnode == NULL) { - terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; mError("mnode:%d, failed to perform insert action since %s", pObj->id, terrstr()); return -1; } @@ -743,8 +743,12 @@ static void mndReloadSyncConfig(SMnode *pMnode) { if (objStatus == SDB_STATUS_READY || objStatus == SDB_STATUS_CREATING) { SNodeInfo *pNode = &cfg.nodeInfo[cfg.replicaNum]; - tstrncpy(pNode->nodeFqdn, pObj->pDnode->fqdn, sizeof(pNode->nodeFqdn)); + pNode->nodeId = pObj->pDnode->id; + pNode->clusterId = mndGetClusterId(pMnode); pNode->nodePort = pObj->pDnode->port; + tstrncpy(pNode->nodeFqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN); + tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + mInfo("vgId:1, ep:%s:%u dnode:%d", pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); if (pObj->pDnode->id == pMnode->selfDnodeId) { cfg.myIndex = cfg.replicaNum; } @@ -758,7 +762,6 @@ static void mndReloadSyncConfig(SMnode *pMnode) { mInfo("vgId:1, mnode sync not reconfig since readyMnodes:%d updatingMnodes:%d", readyMnodes, updatingMnodes); return; } - // ASSERT(0); if (cfg.myIndex == -1) { #if 1 @@ -775,14 +778,15 @@ static void mndReloadSyncConfig(SMnode *pMnode) { mInfo("vgId:1, mnode sync reconfig, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex); for (int32_t i = 0; i < cfg.replicaNum; ++i) { SNodeInfo *pNode = &cfg.nodeInfo[i]; - mInfo("vgId:1, index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort); + mInfo("vgId:1, index:%d, ep:%s:%u dnode:%d cluster:%" PRId64, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, + pNode->clusterId); } int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg); if (code != 0) { - mError("vgId:1, failed to reconfig mnode sync since %s", terrstr()); + mError("vgId:1, mnode sync reconfig failed since %s", terrstr()); } else { - mInfo("vgId:1, reconfig mnode sync success"); + mInfo("vgId:1, mnode sync reconfig success"); } } } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 6a7e2aaa51b2f7603c186ac75529e231561f442a..48d8e89bfe73e47dd89e75f8c13626ebd3d1ecf4 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -19,6 +19,7 @@ #include "systable.h" #define SHOW_STEP_SIZE 100 +#define SHOW_COLS_STEP_SIZE 4096 static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq); static void mndFreeShowObj(SShowObj *pShow); @@ -76,6 +77,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) { type = TSDB_MGMT_TABLE_TABLE; } else if (strncasecmp(name, TSDB_INS_TABLE_TAGS, len) == 0) { type = TSDB_MGMT_TABLE_TAG; + } else if (strncasecmp(name, TSDB_INS_TABLE_COLS, len) == 0) { + type = TSDB_MGMT_TABLE_COL; } else if (strncasecmp(name, TSDB_INS_TABLE_TABLE_DISTRIBUTED, len) == 0) { // type = TSDB_MGMT_TABLE_DIST; } else if (strncasecmp(name, TSDB_INS_TABLE_USERS, len) == 0) { @@ -111,7 +114,7 @@ static int32_t convertToRetrieveType(char *name, int32_t len) { } else if (strncasecmp(name, TSDB_INS_TABLE_USER_PRIVILEGES, len) == 0) { type = TSDB_MGMT_TABLE_PRIVILEGES; } else { - // ASSERT(0); + mError("invalid show name:%s len:%d", name, len); } return type; @@ -131,6 +134,7 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq) { showObj.pMnode = pMnode; showObj.type = convertToRetrieveType(pReq->tb, tListLen(pReq->tb)); memcpy(showObj.db, pReq->db, TSDB_DB_FNAME_LEN); + strncpy(showObj.filterTb, pReq->filterTb, TSDB_TABLE_NAME_LEN); int32_t keepTime = tsShellActivityTimer * 6 * 1000; SShowObj *pShow = taosCachePut(pMgmt->cache, &showId, sizeof(int64_t), &showObj, size, keepTime); @@ -190,13 +194,15 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { int32_t rowsToRead = SHOW_STEP_SIZE; int32_t size = 0; int32_t rowsRead = 0; - + mDebug("mndProcessRetrieveSysTableReq start"); SRetrieveTableReq retrieveReq = {0}; if (tDeserializeSRetrieveTableReq(pReq->pCont, pReq->contLen, &retrieveReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } + mDebug("mndProcessRetrieveSysTableReq tb:%s", retrieveReq.tb); + if (retrieveReq.showId == 0) { STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb)); if (pMeta == NULL) { @@ -226,6 +232,9 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { } } + if(pShow->type == TSDB_MGMT_TABLE_COL){ // expend capacity for ins_columns + rowsToRead = SHOW_COLS_STEP_SIZE; + } ShowRetrieveFp retrieveFp = pMgmt->retrieveFps[pShow->type]; if (retrieveFp == NULL) { mndReleaseShowObj(pShow, false); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 2d1f6eb5054c978195a2cb40341bef268e3f0fd1..fe0dc9e8578cd9afc46900e0530eb28788fa5c0b 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -202,11 +202,13 @@ static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) { _OVER: if (terrno != 0) { - mError("sma:%s, failed to decode from raw:%p since %s", pSma == NULL ? "null" : pSma->name, pRaw, terrstr()); - taosMemoryFreeClear(pSma->expr); - taosMemoryFreeClear(pSma->tagsFilter); - taosMemoryFreeClear(pSma->sql); - taosMemoryFreeClear(pSma->ast); + if (pSma != NULL) { + mError("sma:%s, failed to decode from raw:%p since %s", pSma->name, pRaw, terrstr()); + taosMemoryFreeClear(pSma->expr); + taosMemoryFreeClear(pSma->tagsFilter); + taosMemoryFreeClear(pSma->sql); + taosMemoryFreeClear(pSma->ast); + } taosMemoryFreeClear(pRow); return NULL; } @@ -488,7 +490,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea memcpy(smaObj.db, pDb->name, TSDB_DB_FNAME_LEN); smaObj.createdTime = taosGetTimestampMs(); smaObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); - ASSERT(smaObj.uid != 0); + char resultTbName[TSDB_TABLE_FNAME_LEN + 16] = {0}; snprintf(resultTbName, TSDB_TABLE_FNAME_LEN + 16, "%s_td_tsma_rst_tb", pCreate->name); memcpy(smaObj.dstTbName, resultTbName, TSDB_TABLE_FNAME_LEN); @@ -558,13 +560,15 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea SNode *pAst = NULL; if (nodesStringToNode(streamObj.ast, &pAst) < 0) { - ASSERT(0); + terrno = TSDB_CODE_MND_INVALID_SMA_OPTION; + mError("sma:%s, failed to create since parse ast error", smaObj.name); return -1; } // extract output schema from ast if (qExtractResultSchema(pAst, (int32_t *)&streamObj.outputSchema.nCols, &streamObj.outputSchema.pSchema) != 0) { - ASSERT(0); + terrno = TSDB_CODE_MND_INVALID_SMA_OPTION; + mError("sma:%s, failed to create since extract result schema error", smaObj.name); return -1; } @@ -579,15 +583,18 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea }; if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) { - ASSERT(0); + terrno = TSDB_CODE_MND_INVALID_SMA_OPTION; + mError("sma:%s, failed to create since create query plan error", smaObj.name); return -1; } // save physcial plan if (nodesNodeToString((SNode *)pPlan, false, &streamObj.physicalPlan, NULL) != 0) { - ASSERT(0); + terrno = TSDB_CODE_MND_INVALID_SMA_OPTION; + mError("sma:%s, failed to create since save physcial plan error", smaObj.name); return -1; } + if (pAst != NULL) nodesDestroyNode(pAst); nodesDestroyNode((SNode *)pPlan); @@ -826,14 +833,13 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { mError("stream:%s, failed to drop task since %s", pStream->name, terrstr()); sdbRelease(pMnode->pSdb, pStream); - ASSERT(0); goto _OVER; } // drop stream if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { + mError("stream:%s, failed to drop log since %s", pStream->name, terrstr()); sdbRelease(pMnode->pSdb, pStream); - ASSERT(0); goto _OVER; } } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 54b3202d246adde7d2bd053d8969554fcb7d0bce..c243e83a15333c877434c07ed961c1ce2f5af9e2 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -43,6 +43,7 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq); static int32_t mndProcessDropStbReq(SRpcMsg *pReq); static int32_t mndProcessTableMetaReq(SRpcMsg *pReq); static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); +static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextStb(SMnode *pMnode, void *pIter); static int32_t mndProcessTableCfgReq(SRpcMsg *pReq); static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp, @@ -69,10 +70,14 @@ int32_t mndInitStb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_TABLE_META, mndProcessTableMetaReq); mndSetMsgHandle(pMnode, TDMT_MND_TTL_TIMER, mndProcessTtlTimer); mndSetMsgHandle(pMnode, TDMT_MND_TABLE_CFG, mndProcessTableCfgReq); +// mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveStbReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STB, mndRetrieveStb); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STB, mndCancelGetNextStb); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_COL, mndRetrieveStbCol); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_COL, mndCancelGetNextStb); + return sdbSetTable(pMnode->pSdb, table); } @@ -1177,7 +1182,9 @@ static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName, SNode *pAst = NULL; if (nodesStringToNode(pTopic->ast, &pAst) != 0) { - ASSERT(0); + terrno = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC; + mError("topic:%s, create ast error", pTopic->name); + sdbRelease(pSdb, pTopic); return -1; } @@ -1222,7 +1229,9 @@ static int32_t mndCheckAlterColForStream(SMnode *pMnode, const char *stbFullName SNode *pAst = NULL; if (nodesStringToNode(pStream->ast, &pAst) != 0) { - ASSERT(0); + terrno = TSDB_CODE_MND_INVALID_STREAM_OPTION; + mError("stream:%s, create ast error", pStream->name); + sdbRelease(pSdb, pStream); return -1; } @@ -2094,7 +2103,9 @@ static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName, SNode *pAst = NULL; if (nodesStringToNode(pTopic->ast, &pAst) != 0) { - ASSERT(0); + terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; + mError("topic:%s, create ast error", pTopic->name); + sdbRelease(pSdb, pTopic); return -1; } @@ -2141,7 +2152,9 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, SNode *pAst = NULL; if (nodesStringToNode(pStream->ast, &pAst) != 0) { - ASSERT(0); + terrno = TSDB_CODE_MND_INVALID_STREAM_OPTION; + mError("stream:%s, create ast error", pStream->name); + sdbRelease(pSdb, pStream); return -1; } @@ -2481,6 +2494,283 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t } } +//static int32_t mndProcessRetrieveStbReq(SRpcMsg *pReq) { +// SMnode *pMnode = pReq->info.node; +// SShowMgmt *pMgmt = &pMnode->showMgmt; +// SShowObj *pShow = NULL; +// int32_t rowsToRead = SHOW_STEP_SIZE; +// int32_t rowsRead = 0; +// +// SRetrieveTableReq retrieveReq = {0}; +// if (tDeserializeSRetrieveTableReq(pReq->pCont, pReq->contLen, &retrieveReq) != 0) { +// terrno = TSDB_CODE_INVALID_MSG; +// return -1; +// } +// +// SMnode *pMnode = pReq->info.node; +// SSdb *pSdb = pMnode->pSdb; +// int32_t numOfRows = 0; +// SDbObj *pDb = NULL; +// ESdbStatus objStatus = 0; +// +// SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user); +// if (pUser == NULL) return 0; +// bool sysinfo = pUser->sysInfo; +// +// // Append the information_schema database into the result. +//// if (!pShow->sysDbRsp) { +//// SDbObj infoschemaDb = {0}; +//// setInformationSchemaDbCfg(pMnode, &infoschemaDb); +//// size_t numOfTables = 0; +//// getVisibleInfosTablesNum(sysinfo, &numOfTables); +//// mndDumpDbInfoData(pMnode, pBlock, &infoschemaDb, pShow, numOfRows, numOfTables, true, 0, 1); +//// +//// numOfRows += 1; +//// +//// SDbObj perfschemaDb = {0}; +//// setPerfSchemaDbCfg(pMnode, &perfschemaDb); +//// numOfTables = 0; +//// getPerfDbMeta(NULL, &numOfTables); +//// mndDumpDbInfoData(pMnode, pBlock, &perfschemaDb, pShow, numOfRows, numOfTables, true, 0, 1); +//// +//// numOfRows += 1; +//// pShow->sysDbRsp = true; +//// } +// +// SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_COLS); +// blockDataEnsureCapacity(p, rowsToRead); +// +// size_t size = 0; +// const SSysTableMeta* pSysDbTableMeta = NULL; +// +// getInfosDbMeta(&pSysDbTableMeta, &size); +// p->info.rows = buildDbColsInfoBlock(sysinfo, p, pSysDbTableMeta, size, TSDB_INFORMATION_SCHEMA_DB); +// +// getPerfDbMeta(&pSysDbTableMeta, &size); +// p->info.rows = buildDbColsInfoBlock(sysinfo, p, pSysDbTableMeta, size, TSDB_PERFORMANCE_SCHEMA_DB); +// +// blockDataDestroy(p); +// +// +// while (numOfRows < rowsToRead) { +// pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus, true); +// if (pShow->pIter == NULL) break; +// if (strncmp(retrieveReq.db, pDb->name, strlen(retrieveReq.db)) != 0){ +// continue; +// } +// if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) != 0) { +// continue; +// } +// +// while (numOfRows < rowsToRead) { +// pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb); +// if (pShow->pIter == NULL) break; +// +// if (pDb != NULL && pStb->dbUid != pDb->uid) { +// sdbRelease(pSdb, pStb); +// continue; +// } +// +// cols = 0; +// +// SName name = {0}; +// char stbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; +// mndExtractTbNameFromStbFullName(pStb->name, &stbName[VARSTR_HEADER_SIZE], TSDB_TABLE_NAME_LEN); +// varDataSetLen(stbName, strlen(&stbName[VARSTR_HEADER_SIZE])); +// +// SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// colDataAppend(pColInfo, numOfRows, (const char *)stbName, false); +// +// char db[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; +// tNameFromString(&name, pStb->db, T_NAME_ACCT | T_NAME_DB); +// tNameGetDbName(&name, varDataVal(db)); +// varDataSetLen(db, strlen(varDataVal(db))); +// +// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// colDataAppend(pColInfo, numOfRows, (const char *)db, false); +// +// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// colDataAppend(pColInfo, numOfRows, (const char *)&pStb->createdTime, false); +// +// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// colDataAppend(pColInfo, numOfRows, (const char *)&pStb->numOfColumns, false); +// +// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// colDataAppend(pColInfo, numOfRows, (const char *)&pStb->numOfTags, false); +// +// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// colDataAppend(pColInfo, numOfRows, (const char *)&pStb->updateTime, false); // number of tables +// +// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// if (pStb->commentLen > 0) { +// char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; +// STR_TO_VARSTR(comment, pStb->comment); +// colDataAppend(pColInfo, numOfRows, comment, false); +// } else if (pStb->commentLen == 0) { +// char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; +// STR_TO_VARSTR(comment, ""); +// colDataAppend(pColInfo, numOfRows, comment, false); +// } else { +// colDataAppendNULL(pColInfo, numOfRows); +// } +// +// char watermark[64 + VARSTR_HEADER_SIZE] = {0}; +// sprintf(varDataVal(watermark), "%" PRId64 "a,%" PRId64 "a", pStb->watermark[0], pStb->watermark[1]); +// varDataSetLen(watermark, strlen(varDataVal(watermark))); +// +// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// colDataAppend(pColInfo, numOfRows, (const char *)watermark, false); +// +// char maxDelay[64 + VARSTR_HEADER_SIZE] = {0}; +// sprintf(varDataVal(maxDelay), "%" PRId64 "a,%" PRId64 "a", pStb->maxdelay[0], pStb->maxdelay[1]); +// varDataSetLen(maxDelay, strlen(varDataVal(maxDelay))); +// +// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// colDataAppend(pColInfo, numOfRows, (const char *)maxDelay, false); +// +// char rollup[160 + VARSTR_HEADER_SIZE] = {0}; +// int32_t rollupNum = (int32_t)taosArrayGetSize(pStb->pFuncs); +// char *sep = ", "; +// int32_t sepLen = strlen(sep); +// int32_t rollupLen = sizeof(rollup) - VARSTR_HEADER_SIZE - 2; +// for (int32_t i = 0; i < rollupNum; ++i) { +// char *funcName = taosArrayGet(pStb->pFuncs, i); +// if (i) { +// strncat(varDataVal(rollup), sep, rollupLen); +// rollupLen -= sepLen; +// } +// strncat(varDataVal(rollup), funcName, rollupLen); +// rollupLen -= strlen(funcName); +// } +// varDataSetLen(rollup, strlen(varDataVal(rollup))); +// +// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// colDataAppend(pColInfo, numOfRows, (const char *)rollup, false); +// +// numOfRows++; +// sdbRelease(pSdb, pStb); +// } +// +// if (pDb != NULL) { +// mndReleaseDb(pMnode, pDb); +// } +// +// sdbRelease(pSdb, pDb); +// } +// +// pShow->numOfRows += numOfRows; +// mndReleaseUser(pMnode, pUser); +// +// +// +// +// +// +// +// +// ShowRetrieveFp retrieveFp = pMgmt->retrieveFps[pShow->type]; +// if (retrieveFp == NULL) { +// mndReleaseShowObj(pShow, false); +// terrno = TSDB_CODE_MSG_NOT_PROCESSED; +// mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, terrstr()); +// return -1; +// } +// +// mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type); +// if (retrieveReq.user[0] != 0) { +// memcpy(pReq->info.conn.user, retrieveReq.user, TSDB_USER_LEN); +// } else { +// memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, strlen(TSDB_DEFAULT_USER) + 1); +// } +// if (retrieveReq.db[0] && mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db) != 0) { +// return -1; +// } +// +// int32_t numOfCols = pShow->pMeta->numOfColumns; +// +// SSDataBlock *pBlock = createDataBlock(); +// for (int32_t i = 0; i < numOfCols; ++i) { +// SColumnInfoData idata = {0}; +// +// SSchema *p = &pShow->pMeta->pSchemas[i]; +// +// idata.info.bytes = p->bytes; +// idata.info.type = p->type; +// idata.info.colId = p->colId; +// blockDataAppendColInfo(pBlock, &idata); +// } +// +// blockDataEnsureCapacity(pBlock, rowsToRead); +// +// if (mndCheckRetrieveFinished(pShow)) { +// mDebug("show:0x%" PRIx64 ", read finished, numOfRows:%d", pShow->id, pShow->numOfRows); +// rowsRead = 0; +// } else { +// rowsRead = (*retrieveFp)(pReq, pShow, pBlock, rowsToRead); +// if (rowsRead < 0) { +// terrno = rowsRead; +// mDebug("show:0x%" PRIx64 ", retrieve completed", pShow->id); +// mndReleaseShowObj(pShow, true); +// blockDataDestroy(pBlock); +// return -1; +// } +// +// pBlock->info.rows = rowsRead; +// mDebug("show:0x%" PRIx64 ", stop retrieve data, rowsRead:%d numOfRows:%d", pShow->id, rowsRead, pShow->numOfRows); +// } +// +// size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pShow->pMeta->numOfColumns + +// blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)); +// +// SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); +// if (pRsp == NULL) { +// mndReleaseShowObj(pShow, false); +// terrno = TSDB_CODE_OUT_OF_MEMORY; +// mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, terrstr()); +// blockDataDestroy(pBlock); +// return -1; +// } +// +// pRsp->handle = htobe64(pShow->id); +// +// if (rowsRead > 0) { +// char *pStart = pRsp->data; +// SSchema *ps = pShow->pMeta->pSchemas; +// +// *(int32_t *)pStart = htonl(pShow->pMeta->numOfColumns); +// pStart += sizeof(int32_t); // number of columns +// +// for (int32_t i = 0; i < pShow->pMeta->numOfColumns; ++i) { +// SSysTableSchema *pSchema = (SSysTableSchema *)pStart; +// pSchema->bytes = htonl(ps[i].bytes); +// pSchema->colId = htons(ps[i].colId); +// pSchema->type = ps[i].type; +// +// pStart += sizeof(SSysTableSchema); +// } +// +// int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns); +// } +// +// pRsp->numOfRows = htonl(rowsRead); +// pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision +// pReq->info.rsp = pRsp; +// pReq->info.rspLen = size; +// +// if (rowsRead == 0 || rowsRead < rowsToRead) { +// pRsp->completed = 1; +// mDebug("show:0x%" PRIx64 ", retrieve completed", pShow->id); +// mndReleaseShowObj(pShow, true); +// } else { +// mDebug("show:0x%" PRIx64 ", retrieve not completed yet", pShow->id); +// mndReleaseShowObj(pShow, false); +// } +// +// blockDataDestroy(pBlock); +// return TSDB_CODE_SUCCESS; +//} + + static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; @@ -2591,6 +2881,187 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc return numOfRows; } +static int32_t buildDbColsInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, + const char* dbName, const char* tbName) { + char tName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + char dName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + int32_t numOfRows = p->info.rows; + + STR_TO_VARSTR(dName, dbName); + STR_TO_VARSTR(typeName, "SYSTEM_TABLE"); + + for (int32_t i = 0; i < size; ++i) { + const SSysTableMeta* pm = &pSysDbTableMeta[i]; +// if (pm->sysInfo) { +// continue; +// } + if(tbName[0] && strncmp(tbName, pm->name, TSDB_TABLE_NAME_LEN) != 0){ + continue; + } + + STR_TO_VARSTR(tName, pm->name); + + for(int32_t j = 0; j < pm->colNum; j++){ + // table name + SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); + colDataAppend(pColInfoData, numOfRows, tName, false); + + // database name + pColInfoData = taosArrayGet(p->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, dName, false); + + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, typeName, false); + + // col name + char colName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(colName, pm->schema[j].name); + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, colName, false); + + // col type + int8_t colType = pm->schema[j].type; + pColInfoData = taosArrayGet(p->pDataBlock, 4); + char colTypeStr[VARSTR_HEADER_SIZE + 32]; + int colTypeLen = sprintf(varDataVal(colTypeStr), "%s", tDataTypes[colType].name); + if (colType == TSDB_DATA_TYPE_VARCHAR) { + colTypeLen += sprintf(varDataVal(colTypeStr) + colTypeLen, "(%d)", + (int32_t)(pm->schema[j].bytes - VARSTR_HEADER_SIZE)); + } else if (colType == TSDB_DATA_TYPE_NCHAR) { + colTypeLen += sprintf( + varDataVal(colTypeStr) + colTypeLen, "(%d)", + (int32_t)((pm->schema[j].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); + } + varDataSetLen(colTypeStr, colTypeLen); + colDataAppend(pColInfoData, numOfRows, (char*)colTypeStr, false); + + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (const char*)&pm->schema[j].bytes, false); + for (int32_t k = 6; k <= 8; ++k) { + pColInfoData = taosArrayGet(p->pDataBlock, k); + colDataAppendNULL(pColInfoData, numOfRows); + } + + numOfRows += 1; + } + } + + return numOfRows; +} + +static int32_t buildSysDbColsInfo(SSDataBlock* p, char* db, char* tb) { + size_t size = 0; + const SSysTableMeta* pSysDbTableMeta = NULL; + + if(db[0] && strncmp(db, TSDB_INFORMATION_SCHEMA_DB, TSDB_DB_FNAME_LEN) != 0 && strncmp(db, TSDB_PERFORMANCE_SCHEMA_DB, TSDB_DB_FNAME_LEN) != 0){ + return p->info.rows; + } + + getInfosDbMeta(&pSysDbTableMeta, &size); + p->info.rows = buildDbColsInfoBlock(p, pSysDbTableMeta, size, TSDB_INFORMATION_SCHEMA_DB, tb); + + getPerfDbMeta(&pSysDbTableMeta, &size); + p->info.rows = buildDbColsInfoBlock(p, pSysDbTableMeta, size, TSDB_PERFORMANCE_SCHEMA_DB, tb); + + return p->info.rows; +} + +static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + SStbObj *pStb = NULL; + + int32_t numOfRows = buildSysDbColsInfo(pBlock, pShow->db, pShow->filterTb); + mDebug("mndRetrieveStbCol get system table cols, rows:%d, db:%s", numOfRows, pShow->db); + SDbObj *pDb = NULL; + if (strlen(pShow->db) > 0) { + pDb = mndAcquireDb(pMnode, pShow->db); + if (pDb == NULL) return terrno; + } + + char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(typeName, "SUPER_TABLE"); + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb); + if (pShow->pIter == NULL) break; + + if (pDb != NULL && pStb->dbUid != pDb->uid) { + sdbRelease(pSdb, pStb); + continue; + } + + SName name = {0}; + char stbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + mndExtractTbNameFromStbFullName(pStb->name, &stbName[VARSTR_HEADER_SIZE], TSDB_TABLE_NAME_LEN); + if(pShow->filterTb[0] && strncmp(pShow->filterTb, &stbName[VARSTR_HEADER_SIZE], TSDB_TABLE_NAME_LEN) != 0){ + sdbRelease(pSdb, pStb); + continue; + } + varDataSetLen(stbName, strlen(&stbName[VARSTR_HEADER_SIZE])); + + mDebug("mndRetrieveStbCol get stable cols, stable name:%s, db:%s", pStb->name, pStb->db); + + char db[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&name, pStb->db, T_NAME_ACCT | T_NAME_DB); + tNameGetDbName(&name, varDataVal(db)); + varDataSetLen(db, strlen(varDataVal(db))); + + for(int i = 0; i < pStb->numOfColumns; i++){ + int32_t cols = 0; + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)stbName, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)db, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, typeName, false); + + // col name + char colName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(colName, pStb->pColumns[i].name); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, colName, false); + + // col type + int8_t colType = pStb->pColumns[i].type; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + char colTypeStr[VARSTR_HEADER_SIZE + 32]; + int colTypeLen = sprintf(varDataVal(colTypeStr), "%s", tDataTypes[colType].name); + if (colType == TSDB_DATA_TYPE_VARCHAR) { + colTypeLen += sprintf(varDataVal(colTypeStr) + colTypeLen, "(%d)", + (int32_t)(pStb->pColumns[i].bytes - VARSTR_HEADER_SIZE)); + } else if (colType == TSDB_DATA_TYPE_NCHAR) { + colTypeLen += sprintf( + varDataVal(colTypeStr) + colTypeLen, "(%d)", + (int32_t)((pStb->pColumns[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); + } + varDataSetLen(colTypeStr, colTypeLen); + colDataAppend(pColInfo, numOfRows, (char*)colTypeStr, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char*)&pStb->pColumns[i].bytes, false); + while(cols < pShow->numOfColumns) { + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppendNULL(pColInfo, numOfRows); + } + numOfRows++; + } + + sdbRelease(pSdb, pStb); + } + + if (pDb != NULL) { + mndReleaseDb(pMnode, pDb); + } + + pShow->numOfRows += numOfRows; + mDebug("mndRetrieveStbCol success, rows:%d, pShow->numOfRows:%d", numOfRows, pShow->numOfRows); + + return numOfRows; +} + static void mndCancelGetNextStb(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 57f9c085ad3c2597a21029f3b1848eedf7c8fd77..6b54a36a6fc3b4135fb19f1bbe40ee13cdecf7c5 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -314,7 +314,11 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, } tstrncpy(pObj->targetDb, pTargetDb->name, TSDB_DB_FNAME_LEN); - pObj->targetStbUid = mndGenerateUid(pObj->targetSTbName, TSDB_TABLE_FNAME_LEN); + if (pCreate->createStb == STREAM_CREATE_STABLE_TRUE) { + pObj->targetStbUid = mndGenerateUid(pObj->targetSTbName, TSDB_TABLE_FNAME_LEN); + } else { + pObj->targetStbUid = pCreate->targetStbUid; + } pObj->targetDbUid = pTargetDb->uid; mndReleaseDb(pMnode, pTargetDb); @@ -334,6 +338,38 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, goto FAIL; } + int32_t numOfNULL = taosArrayGetSize(pCreate->fillNullCols); + if(numOfNULL > 0) { + pObj->outputSchema.nCols += numOfNULL; + SSchema* pFullSchema = taosMemoryCalloc(pObj->outputSchema.nCols, sizeof(SSchema)); + if (!pFullSchema) { + goto FAIL; + } + + int32_t nullIndex = 0; + int32_t dataIndex = 0; + for (int16_t i = 0; i < pObj->outputSchema.nCols; i++) { + SColLocation* pos = taosArrayGet(pCreate->fillNullCols, nullIndex); + if (i < pos->slotId) { + pFullSchema[i].bytes = pObj->outputSchema.pSchema[dataIndex].bytes; + pFullSchema[i].colId = i + 1; // pObj->outputSchema.pSchema[dataIndex].colId; + pFullSchema[i].flags = pObj->outputSchema.pSchema[dataIndex].flags; + strcpy(pFullSchema[i].name, pObj->outputSchema.pSchema[dataIndex].name); + pFullSchema[i].type = pObj->outputSchema.pSchema[dataIndex].type; + dataIndex++; + } else { + pFullSchema[i].bytes = 0; + pFullSchema[i].colId = pos->colId; + pFullSchema[i].flags = COL_SET_NULL; + memset(pFullSchema[i].name, 0, TSDB_COL_NAME_LEN); + pFullSchema[i].type = pos->type; + nullIndex++; + } + } + taosMemoryFree(pObj->outputSchema.pSchema); + pObj->outputSchema.pSchema = pFullSchema; + } + SPlanContext cxt = { .pAstRoot = pAst, .topicQuery = false, @@ -465,7 +501,6 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre SMCreateStbReq createReq = {0}; tstrncpy(createReq.name, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); createReq.numOfColumns = pStream->outputSchema.nCols; - createReq.numOfTags = 1; // group id createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField)); // build fields taosArraySetSize(createReq.pColumns, createReq.numOfColumns); @@ -476,14 +511,29 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre pField->type = pStream->outputSchema.pSchema[i].type; pField->bytes = pStream->outputSchema.pSchema[i].bytes; } - createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField)); - taosArraySetSize(createReq.pTags, 1); - // build tags - SField *pField = taosArrayGet(createReq.pTags, 0); - strcpy(pField->name, "group_id"); - pField->type = TSDB_DATA_TYPE_UBIGINT; - pField->flags = 0; - pField->bytes = 8; + + if (pStream->tagSchema.nCols == 0) { + createReq.numOfTags = 1; + createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField)); + taosArraySetSize(createReq.pTags, createReq.numOfTags); + // build tags + SField *pField = taosArrayGet(createReq.pTags, 0); + strcpy(pField->name, "group_id"); + pField->type = TSDB_DATA_TYPE_UBIGINT; + pField->flags = 0; + pField->bytes = 8; + } else { + createReq.numOfTags = pStream->tagSchema.nCols; + createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField)); + taosArraySetSize(createReq.pTags, createReq.numOfTags); + for (int32_t i = 0; i < createReq.numOfTags; i++) { + SField *pField = taosArrayGet(createReq.pTags, i); + pField->bytes = pStream->tagSchema.pSchema[i].bytes; + pField->flags = pStream->tagSchema.pSchema[i].flags; + pField->type = pStream->tagSchema.pSchema[i].type; + tstrncpy(pField->name, pStream->tagSchema.pSchema[i].name, TSDB_COL_NAME_LEN); + } + } if (mndCheckCreateStbReq(&createReq) != 0) { goto _OVER; @@ -637,7 +687,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; // create stb for stream - if (mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user) < 0) { + if (createStreamReq.createStb == STREAM_CREATE_STABLE_TRUE && mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user) < 0) { mError("trans:%d, failed to create stb for stream %s since %s", pTrans->id, createStreamReq.name, terrstr()); mndTransDrop(pTrans); goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 418474baa6c9759a2c4fb604e07e99af272539ae..b8ef185199c5b93c57ab4d2e68d30bb12c797373 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -96,18 +96,12 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic, pSub->subType = pTopic->subType; pSub->withMeta = pTopic->withMeta; - ASSERT(pSub->unassignedVgs->size == 0); - ASSERT(taosHashGetSize(pSub->consumerHash) == 0); - if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) { tDeleteSubscribeObj(pSub); taosMemoryFree(pSub); return NULL; } - ASSERT(pSub->unassignedVgs->size > 0); - ASSERT(taosHashGetSize(pSub->consumerHash) == 0); - return pSub; } @@ -144,7 +138,10 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg) { - ASSERT(pRebVg->oldConsumerId != pRebVg->newConsumerId); + if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { + terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; + return -1; + } void *buf; int32_t tlen; @@ -155,8 +152,8 @@ static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SM int32_t vgId = pRebVg->pVgEp->vgId; SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); if (pVgObj == NULL) { - ASSERT(0); taosMemoryFree(buf); + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -206,9 +203,9 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { } static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { - int32_t totalVgNum = pOutput->pSub->vgNum; - - mInfo("mq rebalance: subscription: %s, vgNum: %d", pOutput->pSub->key, pOutput->pSub->vgNum); + int32_t totalVgNum = pOutput->pSub->vgNum; + const char *sub = pOutput->pSub->key; + mInfo("sub:%s, mq rebalance vgNum:%d", sub, pOutput->pSub->vgNum); // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -218,11 +215,10 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR int32_t actualRemoved = 0; for (int32_t i = 0; i < removedNum; i++) { int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i); - ASSERT(consumerId > 0); + SMqConsumerEp *pConsumerEp = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); - ASSERT(pConsumerEp); + if (pConsumerEp) { - ASSERT(consumerId == pConsumerEp->consumerId); actualRemoved++; int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs); for (int32_t j = 0; j < consumerVgNum; j++) { @@ -233,7 +229,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64, pVgEp->vgId, consumerId); + mInfo("sub:%s, mq rebalance remove vgId:%d from consumer:%" PRId64, sub, pVgEp->vgId, consumerId); } taosArrayDestroy(pConsumerEp->vgs); taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); @@ -241,7 +237,10 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR taosArrayPush(pOutput->removedConsumers, &consumerId); } } - ASSERT(removedNum == actualRemoved); + + if (removedNum != actualRemoved) { + mError("sub:%s, mq rebalance removedNum:%d not matched with actual:%d", sub, removedNum, actualRemoved); + } // if previously no consumer, there are vgs not assigned { @@ -254,7 +253,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg)); - mInfo("mq rebalance: remove vgId:%d from unassigned", pVgEp->vgId); + mInfo("sub:%s, mq rebalance remove vgId:%d from unassigned", sub, pVgEp->vgId); } } @@ -268,8 +267,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR minVgCnt = totalVgNum / afterRebConsumerNum; imbConsumerNum = totalVgNum % afterRebConsumerNum; } - mInfo("mq rebalance: %d consumer after rebalance, at least %d vg each, %d consumer has more vg", afterRebConsumerNum, - minVgCnt, imbConsumerNum); + mInfo("sub:%s, mq rebalance %d consumer after rebalance, at least %d vg each, %d consumer has more vg", sub, + afterRebConsumerNum, minVgCnt, imbConsumerNum); // 4. first scan: remove consumer more than wanted, put to remove hash int32_t imbCnt = 0; @@ -278,7 +277,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); if (pIter == NULL) break; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; - ASSERT(pConsumerEp->consumerId > 0); + int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs); // all old consumers still existing are touched // TODO optimize: touch only consumer whose vgs changed @@ -298,7 +297,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, + mInfo("sub:%s, mq rebalance remove vgId:%d from consumer:%" PRId64 ",(first scan)", sub, pVgEp->vgId, pConsumerEp->consumerId); } imbCnt++; @@ -313,7 +312,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, + mInfo("sub:%s, mq rebalance remove vgId:%d from consumer:%" PRId64 ",(first scan)", sub, pVgEp->vgId, pConsumerEp->consumerId); } } @@ -325,13 +324,13 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR int32_t consumerNum = taosArrayGetSize(pInput->pRebInfo->newConsumers); for (int32_t i = 0; i < consumerNum; i++) { int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i); - ASSERT(consumerId > 0); + SMqConsumerEp newConsumerEp; newConsumerEp.consumerId = consumerId; newConsumerEp.vgs = taosArrayInit(0, sizeof(void *)); taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp)); taosArrayPush(pOutput->newConsumers, &consumerId); - mInfo("mq rebalance: add new consumer:%" PRId64, consumerId); + mInfo("sub:%s, mq rebalance add new consumer:%" PRId64, sub, consumerId); } } @@ -344,13 +343,16 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); if (pIter == NULL) break; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; - ASSERT(pConsumerEp->consumerId > 0); // push until equal minVg while (taosArrayGetSize(pConsumerEp->vgs) < minVgCnt) { // iter hash and find one vg pRemovedIter = taosHashIterate(pHash, pRemovedIter); - ASSERT(pRemovedIter); + if (pRemovedIter == NULL) { + mError("sub:%s, removed iter is null", sub); + continue; + } + pRebVg = (SMqRebOutputVg *)pRemovedIter; // push taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); @@ -361,7 +363,6 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR } } - ASSERT(pIter == NULL); // 7. handle unassigned vg if (taosHashGetSize(pOutput->pSub->consumerHash) != 0) { // if has consumer, assign all left vg @@ -377,9 +378,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR } while (1) { pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); - ASSERT(pIter); pConsumerEp = (SMqConsumerEp *)pIter; - ASSERT(pConsumerEp->consumerId > 0); + if (taosArrayGetSize(pConsumerEp->vgs) == minVgCnt) { break; } @@ -404,19 +404,19 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR pIter = taosHashIterate(pHash, pIter); if (pIter == NULL) break; pRebOutput = (SMqRebOutputVg *)pIter; - ASSERT(pRebOutput->newConsumerId == -1); + taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); taosArrayPush(pOutput->rebVgs, pRebOutput); - mInfo("mq rebalance: unassign vgId:%d (second scan)", pRebOutput->pVgEp->vgId); + mInfo("sub:%s, mq rebalance unassign vgId:%d (second scan)", sub, pRebOutput->pVgEp->vgId); } } // 8. generate logs - mInfo("mq rebalance: calculation completed, rebalanced vg:"); + mInfo("sub:%s, mq rebalance calculation completed, rebalanced vg", sub); for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) { SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i); - mInfo("mq rebalance: vgId:%d, moved from consumer:%" PRId64 ", to consumer:%" PRId64, pOutputRebVg->pVgEp->vgId, - pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId); + mInfo("sub:%s, mq rebalance vgId:%d, moved from consumer:%" PRId64 ", to consumer:%" PRId64, sub, + pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId); } { void *pIter = NULL; @@ -425,10 +425,11 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR if (pIter == NULL) break; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; int32_t sz = taosArrayGetSize(pConsumerEp->vgs); - mInfo("mq rebalance: final cfg: consumer %" PRId64 " has %d vg", pConsumerEp->consumerId, sz); + mInfo("sub:%s, mq rebalance final cfg: consumer %" PRId64 " has %d vg", sub, pConsumerEp->consumerId, sz); for (int32_t i = 0; i < sz; i++) { SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i); - mInfo("mq rebalance: final cfg: vg %d to consumer %" PRId64 "", pVgEp->vgId, pConsumerEp->consumerId); + mInfo("sub:%s, mq rebalance final cfg: vg %d to consumer %" PRId64 "", sub, pVgEp->vgId, + pConsumerEp->consumerId); } } } @@ -487,7 +488,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu consumerNum = taosArrayGetSize(pOutput->newConsumers); for (int32_t i = 0; i < consumerNum; i++) { int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->newConsumers, i); - ASSERT(consumerId > 0); + SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId); SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup); pConsumerNew->updateType = CONSUMER_UPDATE__ADD; @@ -497,7 +498,6 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu taosArrayPush(pConsumerNew->rebNewTopics, &topic); mndReleaseConsumer(pMnode, pConsumerOld); if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) { - ASSERT(0); tDeleteSMqConsumerObj(pConsumerNew); taosMemoryFree(pConsumerNew); goto REB_FAIL; @@ -510,7 +510,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu consumerNum = taosArrayGetSize(pOutput->removedConsumers); for (int32_t i = 0; i < consumerNum; i++) { int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->removedConsumers, i); - ASSERT(consumerId > 0); + SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId); SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup); pConsumerNew->updateType = CONSUMER_UPDATE__REMOVE; @@ -520,7 +520,6 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu taosArrayPush(pConsumerNew->rebRemovedTopics, &topic); mndReleaseConsumer(pMnode, pConsumerOld); if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) { - ASSERT(0); tDeleteSMqConsumerObj(pConsumerNew); taosMemoryFree(pConsumerNew); goto REB_FAIL; @@ -577,7 +576,6 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { char cgroup[TSDB_CGROUP_LEN]; mndSplitSubscribeKey(pRebInfo->key, topic, cgroup, true); SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); - /*ASSERT(pTopic);*/ if (pTopic == NULL) { mError("mq rebalance %s failed since topic %s not exist, abort", pRebInfo->key, topic); continue; @@ -585,8 +583,14 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { taosRLockLatch(&pTopic->lock); rebOutput.pSub = mndCreateSub(pMnode, pTopic, pRebInfo->key); + + if (rebOutput.pSub == NULL) { + mError("mq rebalance %s failed create sub since %s, abort", pRebInfo->key, terrstr()); + taosRUnLockLatch(&pTopic->lock); + mndReleaseTopic(pMnode, pTopic); + continue; + } memcpy(rebOutput.pSub->dbName, pTopic->db, TSDB_DB_FNAME_LEN); - ASSERT(taosHashGetSize(rebOutput.pSub->consumerHash) == 0); taosRUnLockLatch(&pTopic->lock); mndReleaseTopic(pMnode, pTopic); @@ -606,7 +610,6 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { // if add more consumer to balanced subscribe, // possibly no vg is changed - /*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/ if (mndPersistRebResult(pMnode, pMsg, &rebOutput) < 0) { mError("mq rebalance persist rebalance output error, possibly vnode splitted or dropped"); @@ -693,6 +696,7 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { terrno = TSDB_CODE_OUT_OF_MEMORY; void *buf = NULL; int32_t tlen = tEncodeSubscribeObj(NULL, pSub); + if (tlen <= 0) goto SUB_ENCODE_OVER; int32_t size = sizeof(int32_t) + tlen + MND_SUBSCRIBE_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_SUBSCRIBE, MND_SUBSCRIBE_VER_NUMBER, size); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 54d8aa7f609faae5dcc4e0789a03d1fca89d0446..6b675586e47a9116c9043991c17fa58984d0a44f 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndSync.h" +#include "mndCluster.h" #include "mndTrans.h" static int32_t mndSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { @@ -270,9 +271,11 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { int32_t mndInitSync(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; taosThreadMutexInit(&pMgmt->lock, NULL); + taosThreadMutexLock(&pMgmt->lock); pMgmt->transId = 0; pMgmt->transSec = 0; pMgmt->transSeq = 0; + taosThreadMutexUnlock(&pMgmt->lock); SSyncInfo syncInfo = { .snapshotStrategy = SYNC_STRATEGY_STANDARD_SNAPSHOT, @@ -297,9 +300,12 @@ int32_t mndInitSync(SMnode *pMnode) { pCfg->myIndex = pMgmt->selfIndex; for (int32_t i = 0; i < pMgmt->numOfReplicas; ++i) { SNodeInfo *pNode = &pCfg->nodeInfo[i]; - tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); + pNode->nodeId = pMgmt->replicas[i].id; pNode->nodePort = pMgmt->replicas[i].port; - mInfo("vgId:1, index:%d ep:%s:%u", i, pNode->nodeFqdn, pNode->nodePort); + tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); + tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + mInfo("vgId:1, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, + pNode->clusterId); } tsem_init(&pMgmt->syncSem, 0, 0); @@ -365,6 +371,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { if (pMgmt->transId != 0) { mError("trans:%d, can't be proposed since trans:%d already waiting for confirm", transId, pMgmt->transId); taosThreadMutexUnlock(&pMgmt->lock); + rpcFreeCont(req.pCont); terrno = TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED; return terrno; } diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 1d3209691adae6c917f430a2e6eb716951aefedf..679fafa28d89e398c1e5900f354925a50c38c06a 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -17,7 +17,6 @@ #include "mndTelem.h" #include "mndCluster.h" #include "mndSync.h" -#include "tbuffer.h" #include "thttp.h" #include "tjson.h" @@ -132,7 +131,7 @@ static int32_t mndProcessTelemTimer(SRpcMsg* pReq) { taosThreadMutexUnlock(&pMgmt->lock); if (pCont != NULL) { - if (taosSendHttpReport(tsTelemServer, tsTelemPort, pCont, strlen(pCont), HTTP_FLAT) != 0) { + if (taosSendHttpReport(tsTelemServer, tsTelemUri, tsTelemPort, pCont, strlen(pCont), HTTP_FLAT) != 0) { mError("failed to send telemetry report"); } else { mInfo("succeed to send telemetry report"); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 071edf992fe2ca38c6c74a698d91d9e88bfbf918..96dba24566d8a6f21ea19672d728b795ba0245d7 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -384,7 +384,11 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * topicObj.subType = pCreate->subType; topicObj.withMeta = pCreate->withMeta; if (topicObj.withMeta) { - ASSERT(topicObj.subType != TOPIC_SUB_TYPE__COLUMN); + if (topicObj.subType == TOPIC_SUB_TYPE__COLUMN) { + terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; + mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } } if (pCreate->subType == TOPIC_SUB_TYPE__COLUMN) { @@ -499,7 +503,6 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * if (code < 0) { sdbRelease(pSdb, pVgroup); mndTransDrop(pTrans); - ASSERT(0); return -1; } void *buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); @@ -601,22 +604,19 @@ _OVER: } static int32_t mndDropTopic(SMnode *pMnode, STrans *pTrans, SRpcMsg *pReq, SMqTopicObj *pTopic) { + int32_t code = -1; + if (mndUserRemoveTopic(pMnode, pTrans, pTopic->name) != 0) goto _OVER; + SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + code = 0; +_OVER: mndTransDrop(pTrans); - return 0; + return code; } static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { @@ -696,14 +696,9 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { sdbRelease(pSdb, pConsumer); } -#if 0 - if (pTopic->refConsumerCnt != 0) { - mndReleaseTopic(pMnode, pTopic); - terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED; - mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) { return -1; } -#endif STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq, "drop-topic"); if (pTrans == NULL) { @@ -723,7 +718,6 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { // TODO check if rebalancing if (mndDropSubByTopic(pMnode, pTrans, dropReq.name) < 0) { - /*ASSERT(0);*/ mError("topic:%s, failed to drop since %s", pTopic->name, terrstr()); mndTransDrop(pTrans); mndReleaseTopic(pMnode, pTopic); @@ -888,6 +882,7 @@ int32_t mndCheckTopicExist(SMnode *pMnode, SDbObj *pDb) { return 0; } +#if 0 int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { int32_t code = 0; SSdb *pSdb = pMnode->pSdb; @@ -915,3 +910,4 @@ int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { return code; } +#endif \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 67d3f7e11983f9550d59c042d1a0b84243884d4b..718fc5c73f27e96fb261effbdc54c73310a3e924 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -957,7 +957,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { for (int32_t i = 0; i < size; ++i) { SRpcHandleInfo *pInfo = taosArrayGet(pTrans->pRpcArray, i); if (pInfo->handle != NULL) { - if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED) { code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL; } if (code == TSDB_CODE_SYN_TIMEOUT) { diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index efce6255fbb31f790a876bc4506584e0156a4563..b965e1331649af7634385cea552f1d129750ad89 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -285,14 +285,35 @@ static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser) { return 0; } -static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { - mTrace("user:%s, perform delete action, row:%p", pUser->user, pUser); +static int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) { + memcpy(pNew, pUser, sizeof(SUserObj)); + pNew->authVersion++; + pNew->updateTime = taosGetTimestampMs(); + + taosRLockLatch(&pUser->lock); + pNew->readDbs = mndDupDbHash(pUser->readDbs); + pNew->writeDbs = mndDupDbHash(pUser->writeDbs); + pNew->topics = mndDupTopicHash(pUser->topics); + taosRUnLockLatch(&pUser->lock); + + if (pNew->readDbs == NULL || pNew->writeDbs == NULL || pNew->topics == NULL) { + return -1; + } + return 0; +} + +static void mndUserFreeObj(SUserObj *pUser) { taosHashCleanup(pUser->readDbs); taosHashCleanup(pUser->writeDbs); taosHashCleanup(pUser->topics); pUser->readDbs = NULL; pUser->writeDbs = NULL; pUser->topics = NULL; +} + +static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { + mTrace("user:%s, perform delete action, row:%p", pUser->user, pUser); + mndUserFreeObj(pUser); return 0; } @@ -516,19 +537,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { goto _OVER; } - memcpy(&newUser, pUser, sizeof(SUserObj)); - newUser.authVersion++; - newUser.updateTime = taosGetTimestampMs(); - - taosRLockLatch(&pUser->lock); - newUser.readDbs = mndDupDbHash(pUser->readDbs); - newUser.writeDbs = mndDupDbHash(pUser->writeDbs); - newUser.topics = mndDupTopicHash(pUser->topics); - taosRUnLockLatch(&pUser->lock); - - if (newUser.readDbs == NULL || newUser.writeDbs == NULL || newUser.topics == NULL) { - goto _OVER; - } + if (mndUserDupObj(pUser, &newUser) != 0) goto _OVER; if (alterReq.alterType == TSDB_ALTER_USER_PASSWD) { char pass[TSDB_PASSWORD_LEN + 1] = {0}; @@ -654,9 +663,7 @@ _OVER: mndReleaseUser(pMnode, pOperUser); mndReleaseUser(pMnode, pUser); - taosHashCleanup(newUser.writeDbs); - taosHashCleanup(newUser.readDbs); - taosHashCleanup(newUser.topics); + mndUserFreeObj(&newUser); return code; } @@ -884,9 +891,9 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock db = taosHashIterate(pUser->writeDbs, NULL); while (db != NULL) { cols = 0; - SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; + char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)userName, false); char privilege[20] = {0}; @@ -909,9 +916,9 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock char *topic = taosHashIterate(pUser->topics, NULL); while (topic != NULL) { cols = 0; - SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; + char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)userName, false); char privilege[20] = {0}; @@ -1007,3 +1014,74 @@ _OVER: tFreeSUserAuthBatchRsp(&batchRsp); return code; } + +int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db) { + int32_t code = 0; + SSdb *pSdb = pMnode->pSdb; + int32_t len = strlen(db) + 1; + void *pIter = NULL; + SUserObj *pUser = NULL; + SUserObj newUser = {0}; + + while (1) { + pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pUser); + if (pIter == NULL) break; + + code = -1; + if (mndUserDupObj(pUser, &newUser) != 0) break; + + bool inRead = (taosHashGet(newUser.readDbs, db, len) != NULL); + bool inWrite = (taosHashGet(newUser.writeDbs, db, len) != NULL); + if (inRead || inWrite) { + (void)taosHashRemove(newUser.readDbs, db, len); + (void)taosHashRemove(newUser.writeDbs, db, len); + + SSdbRaw *pCommitRaw = mndUserActionEncode(&newUser); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) break; + (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + } + + mndUserFreeObj(&newUser); + sdbRelease(pSdb, pUser); + code = 0; + } + + if (pUser != NULL) sdbRelease(pSdb, pUser); + if (pIter != NULL) sdbCancelFetch(pSdb, pIter); + mndUserFreeObj(&newUser); + return code; +} + +int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic) { + int32_t code = 0; + SSdb *pSdb = pMnode->pSdb; + int32_t len = strlen(topic) + 1; + void *pIter = NULL; + SUserObj *pUser = NULL; + SUserObj newUser = {0}; + + while (1) { + pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pUser); + if (pIter == NULL) break; + + code = -1; + if (mndUserDupObj(pUser, &newUser) != 0) break; + + bool inTopic = (taosHashGet(newUser.topics, topic, len) != NULL); + if (inTopic) { + (void)taosHashRemove(newUser.topics, topic, len); + SSdbRaw *pCommitRaw = mndUserActionEncode(&newUser); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) break; + (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + } + + mndUserFreeObj(&newUser); + sdbRelease(pSdb, pUser); + code = 0; + } + + if (pUser != NULL) sdbRelease(pSdb, pUser); + if (pIter != NULL) sdbCancelFetch(pSdb, pIter); + mndUserFreeObj(&newUser); + return code; +} diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 31ab1f3259cd45b0223c5962fbca5f27386da57a..54ea9e7b2437b303088fcdd9d003eb1aa19af8e5 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -425,6 +425,7 @@ void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgOb static bool mndResetDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { SDnodeObj *pDnode = pObj; pDnode->numOfVnodes = 0; + pDnode->numOfOtherNodes = 0; return true; } @@ -447,7 +448,7 @@ static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2 pDnode->numOfVnodes, pDnode->numOfSupportVnodes, isMnode, online, pDnode->memAvail, pDnode->memUsed); if (isMnode) { - pDnode->numOfVnodes++; + pDnode->numOfOtherNodes++; } if (online && pDnode->numOfSupportVnodes > 0) { @@ -468,14 +469,25 @@ SArray *mndBuildDnodesArray(SMnode *pMnode, int32_t exceptDnodeId) { sdbTraverse(pSdb, SDB_DNODE, mndResetDnodesArrayFp, NULL, NULL, NULL); sdbTraverse(pSdb, SDB_DNODE, mndBuildDnodesArrayFp, pArray, &exceptDnodeId, NULL); + + mDebug("build %d dnodes array", (int32_t)taosArrayGetSize(pArray)); + for (int32_t i = 0; i < (int32_t)taosArrayGetSize(pArray); ++i) { + SDnodeObj *pDnode = taosArrayGet(pArray, i); + mDebug("dnode:%d, vnodes:%d others:%d", pDnode->id, pDnode->numOfVnodes, pDnode->numOfOtherNodes); + } return pArray; } static int32_t mndCompareDnodeId(int32_t *dnode1Id, int32_t *dnode2Id) { return *dnode1Id >= *dnode2Id ? 1 : 0; } +static float mndGetDnodeScore(SDnodeObj *pDnode, int32_t additionDnodes, float ratio) { + float totalDnodes = pDnode->numOfVnodes + (float)pDnode->numOfOtherNodes * ratio + additionDnodes; + return totalDnodes / pDnode->numOfSupportVnodes; +} + static int32_t mndCompareDnodeVnodes(SDnodeObj *pDnode1, SDnodeObj *pDnode2) { - float d1Score = (float)pDnode1->numOfVnodes / pDnode1->numOfSupportVnodes; - float d2Score = (float)pDnode2->numOfVnodes / pDnode2->numOfSupportVnodes; + float d1Score = mndGetDnodeScore(pDnode1, 0, 0.9); + float d2Score = mndGetDnodeScore(pDnode2, 0, 0.9); return d1Score >= d2Score ? 1 : 0; } @@ -494,7 +506,12 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup int32_t allocedVnodes = 0; void *pIter = NULL; + mDebug("start to sort %d dnodes", (int32_t)taosArrayGetSize(pArray)); taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); + for (int32_t i = 0; i < (int32_t)taosArrayGetSize(pArray); ++i) { + SDnodeObj *pDnode = taosArrayGet(pArray, i); + mDebug("dnode:%d, score:%f", pDnode->id, mndGetDnodeScore(pDnode, 0, 0.9)); + } int32_t size = taosArrayGetSize(pArray); if (size < pVgroup->replica) { @@ -875,7 +892,7 @@ static int32_t mndAddVnodeToVgroup(SMnode *pMnode, STrans *pTrans, SVgObj *pVgro taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { SDnodeObj *pDnode = taosArrayGet(pArray, i); - mInfo("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); + mInfo("dnode:%d, equivalent vnodes:%d others:%d", pDnode->id, pDnode->numOfVnodes, pDnode->numOfOtherNodes); } SVnodeGid *pVgid = &pVgroup->vnodeGid[pVgroup->replica]; @@ -935,7 +952,7 @@ static int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, STrans *pTrans, SVgObj * taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { SDnodeObj *pDnode = taosArrayGet(pArray, i); - mInfo("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); + mInfo("dnode:%d, equivalent vnodes:%d others:%d", pDnode->id, pDnode->numOfVnodes, pDnode->numOfOtherNodes); } int32_t code = -1; @@ -1424,10 +1441,10 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, { SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL) return -1; + if (pRaw == NULL) goto _OVER; if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { sdbFreeRaw(pRaw); - return -1; + goto _OVER; } (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); } @@ -1970,16 +1987,16 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) { taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { SDnodeObj *pDnode = taosArrayGet(pArray, i); - mInfo("dnode:%d, equivalent vnodes:%d support:%d, score:%f", pDnode->id, pDnode->numOfVnodes, - pDnode->numOfSupportVnodes, (float)pDnode->numOfVnodes / pDnode->numOfSupportVnodes); + mInfo("dnode:%d, equivalent vnodes:%d others:%d support:%d, score:%f", pDnode->id, pDnode->numOfVnodes, + pDnode->numOfSupportVnodes, pDnode->numOfOtherNodes, mndGetDnodeScore(pDnode, 0, 1)); } SDnodeObj *pSrc = taosArrayGet(pArray, taosArrayGetSize(pArray) - 1); SDnodeObj *pDst = taosArrayGet(pArray, 0); - float srcScore = (float)(pSrc->numOfVnodes - 1) / pSrc->numOfSupportVnodes; - float dstScore = (float)(pDst->numOfVnodes + 1) / pDst->numOfSupportVnodes; - mInfo("trans:%d, after balance, src dnode:%d score:%f, dst dnode:%d score:%f", pTrans->id, pSrc->id, srcScore, + float srcScore = mndGetDnodeScore(pSrc, -1, 1); + float dstScore = mndGetDnodeScore(pDst, 1, 1); + mInfo("trans:%d, after balance, src dnode:%d score:%f, dst dnode:%d score:%f", pTrans->id, pSrc->id, dstScore, pDst->id, dstScore); if (srcScore > dstScore - 0.000001) { diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index e799f08a17208f107dd67835d2a2733c3a078b15..5a44e4279ff2456a9c0df4d134ecc6d4e49801dd 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -291,6 +291,7 @@ int32_t sdbWriteWithoutFree(SSdb *pSdb, SSdbRaw *pRaw); * @return void* The object of the row. */ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey); +void *sdbAcquireNotReadyObj(SSdb *pSdb, ESdbType type, const void *pKey); /** * @brief Release a row from sdb. diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index f43b6bdb2541c57987d48208da38cd3ea486f1f5..c2d7a9757abf522d67b88e29a0990b2c95f87e68 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -228,11 +228,12 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { int32_t readLen = 0; int64_t ret = 0; char file[PATH_MAX] = {0}; + int32_t bufLen = TSDB_MAX_MSG_SIZE; snprintf(file, sizeof(file), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP); mInfo("start to read sdb file:%s", file); - SSdbRaw *pRaw = taosMemoryMalloc(TSDB_MAX_MSG_SIZE + 100); + SSdbRaw *pRaw = taosMemoryMalloc(bufLen + 100); if (pRaw == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed read sdb file since %s", terrstr()); @@ -243,7 +244,7 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { if (pFile == NULL) { taosMemoryFree(pRaw); terrno = TAOS_SYSTEM_ERROR(errno); - mDebug("failed to read sdb file:%s since %s", file, terrstr()); + mInfo("read sdb file:%s finished since %s", file, terrstr()); return 0; } @@ -275,14 +276,15 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { } readLen = pRaw->dataLen + sizeof(int32_t); - if (readLen >= pRaw->dataLen) { - SSdbRaw *pNewRaw = taosMemoryMalloc(pRaw->dataLen + TSDB_MAX_MSG_SIZE); + if (readLen >= bufLen) { + bufLen = pRaw->dataLen * 2; + SSdbRaw *pNewRaw = taosMemoryMalloc(bufLen + 100); if (pNewRaw == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("failed read sdb file since malloc new sdbRaw size:%d failed", pRaw->dataLen + TSDB_MAX_MSG_SIZE); + mError("failed read sdb file since malloc new sdbRaw size:%d failed", bufLen); goto _OVER; } - mInfo("malloc new sdbRaw size:%d, type:%d", pRaw->dataLen + TSDB_MAX_MSG_SIZE, pRaw->type); + mInfo("malloc new sdb raw size:%d, type:%d", bufLen, pRaw->type); memcpy(pNewRaw, pRaw, sizeof(SSdbRaw)); sdbFreeRaw(pRaw); pRaw = pNewRaw; @@ -636,15 +638,20 @@ int32_t sdbStartWrite(SSdb *pSdb, SSdbIter **ppIter) { } int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, int64_t term, int64_t config) { - int32_t code = 0; + int32_t code = -1; if (!isApply) { mInfo("sdbiter:%p, not apply to sdb", pIter); - sdbCloseIter(pIter); - return 0; + code = 0; + goto _OVER; + } + + if (taosFsyncFile(pIter->file) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + mError("sdbiter:%p, failed to fasync file %s since %s", pIter, pIter->name, terrstr()); + goto _OVER; } - taosFsyncFile(pIter->file); taosCloseFile(&pIter->file); pIter->file = NULL; @@ -653,14 +660,12 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, i if (taosRenameFile(pIter->name, datafile) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); mError("sdbiter:%p, failed to rename file %s to %s since %s", pIter, pIter->name, datafile, terrstr()); - sdbCloseIter(pIter); - return -1; + goto _OVER; } if (sdbReadFile(pSdb) != 0) { mError("sdbiter:%p, failed to read from %s since %s", pIter, datafile, terrstr()); - sdbCloseIter(pIter); - return -1; + goto _OVER; } if (config > 0) { @@ -674,8 +679,11 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, i } mInfo("sdbiter:%p, success applyed to sdb", pIter); + code = 0; + +_OVER: sdbCloseIter(pIter); - return 0; + return code; } int32_t sdbDoWrite(SSdb *pSdb, SSdbIter *pIter, void *pBuf, int32_t len) { diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 32b34ea3a3b01193b0b23a2a2b4d80d938417a18..505dee3d87053d2b406ba5679419f12bdc5d837b 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -270,7 +270,7 @@ int32_t sdbWrite(SSdb *pSdb, SSdbRaw *pRaw) { return code; } -void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey) { +void *sdbAcquireAll(SSdb *pSdb, ESdbType type, const void *pKey, bool onlyReady) { terrno = 0; SHashObj *hash = sdbGetHash(pSdb, type); @@ -306,10 +306,24 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey) { break; } + if (pRet == NULL) { + if (!onlyReady) { + terrno = 0; + atomic_add_fetch_32(&pRow->refCount, 1); + pRet = pRow->pObj; + sdbPrintOper(pSdb, pRow, "acquire"); + } + } + sdbUnLock(pSdb, type); return pRet; } +void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey) { return sdbAcquireAll(pSdb, type, pKey, true); } +void *sdbAcquireNotReadyObj(SSdb *pSdb, ESdbType type, const void *pKey) { + return sdbAcquireAll(pSdb, type, pKey, false); +} + static void sdbCheckRow(SSdb *pSdb, SSdbRow *pRow) { int32_t type = pRow->type; sdbWriteLock(pSdb, type); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index b133226ed39bb5c20ed96b56d95881d009e1e872..860db20fa87e225f43e33e7b25f728a13f65846a 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -124,6 +124,7 @@ FAIL: } void sndClose(SSnode *pSnode) { + streamMetaCommit(pSnode->pMeta); streamMetaClose(pSnode->pMeta); taosMemoryFree(pSnode->path); taosMemoryFree(pSnode); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index cbee70ad0318d190c76d89197a0096aadb4594d9..3f3e287bb9f6b450ad9470d9935a76c65cc03e19 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -54,7 +54,8 @@ int32_t vnodeAlter(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs); void vnodeDestroy(const char *path, STfs *pTfs); SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb); void vnodePreClose(SVnode *pVnode); -void vnodeSyncCheckTimeout(SVnode* pVnode); +void vnodePostClose(SVnode *pVnode); +void vnodeSyncCheckTimeout(SVnode *pVnode); void vnodeClose(SVnode *pVnode); int32_t vnodeStart(SVnode *pVnode); @@ -88,6 +89,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg); int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo); void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs); void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs); +void vnodeProposeCommitOnNeed(SVnode *pVnode); // meta typedef struct SMeta SMeta; // todo: remove @@ -150,7 +152,7 @@ typedef struct SMTbCursor SMTbCursor; SMTbCursor *metaOpenTbCursor(SMeta *pMeta); void metaCloseTbCursor(SMTbCursor *pTbCur); -int32_t metaTbCursorNext(SMTbCursor *pTbCur); +int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); #endif // tsdb @@ -175,7 +177,8 @@ int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableL void tsdbReaderClose(STsdbReader *pReader); bool tsdbNextDataBlock(STsdbReader *pReader); void tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow); -int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock* pDataBlock, bool *allHave); +int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave); +void tsdbReleaseDataBlock(STsdbReader *pReader); SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); @@ -185,7 +188,7 @@ void *tsdbGetIvtIdx(SMeta *pMeta); uint64_t getReaderMaxVersion(STsdbReader *pReader); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, - uint64_t suid, void **pReader, const char* idstr); + uint64_t suid, void **pReader, const char *idstr); int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, SArray *pTableUids); void *tsdbCacherowsReaderClose(void *pReader); int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); @@ -222,11 +225,19 @@ typedef struct SSnapContext { } SSnapContext; typedef struct STqReader { - int64_t ver; - const SSubmitReq *pMsg; - SSubmitBlk *pBlock; - SSubmitMsgIter msgIter; - SSubmitBlkIter blkIter; + // const SSubmitReq *pMsg; + // SSubmitBlk *pBlock; + // SSubmitMsgIter msgIter; + // SSubmitBlkIter blkIter; + + int64_t ver; + SPackedData msg2; + + int8_t setMsg; + SSubmitReq2 submit; + int32_t nextBlk; + + int64_t lastBlkUid; SWalReader *pWalReader; @@ -251,11 +262,14 @@ int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqSeekVer(STqReader *pReader, int64_t ver); int32_t tqNextBlock(STqReader *pReader, SFetchRet *ret); -int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t ver); -bool tqNextDataBlock(STqReader *pReader); -bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); -int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader); -int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas); +int32_t tqReaderSetSubmitReq2(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); +// int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t ver); +bool tqNextDataBlock2(STqReader *pReader); +bool tqNextDataBlockFilterOut2(STqReader *pReader, SHashObj *filterOutUids); +int32_t tqRetrieveDataBlock2(SSDataBlock *pBlock, STqReader *pReader); +int32_t tqRetrieveTaosxBlock2(STqReader *pReader, SArray *blocks, SArray *schemas); +// int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader); +// int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas); int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 828341ddd8666bbb55c2cedce1b34c50f6b3362a..104e0945ba978815e38f17ecf107ff73c2f2905a 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -154,7 +154,8 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum); // tqExec -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp); +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp); +// int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp); int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision); int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp); int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry); @@ -181,8 +182,9 @@ int32_t tqOffsetDelete(STqOffsetStore* pStore, const char* subscribeKey) int32_t tqOffsetCommitFile(STqOffsetStore* pStore); // tqSink -// void tqSinkToTableMerge(SStreamTask* pTask, void* vnode, int64_t ver, void* data); -void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* data); +int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBlock* pDataBlock, + SBatchDeleteReq* deleteReq); +void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* data); // tqOffset char* tqOffsetBuildFName(const char* path, int32_t fVer); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 5a63af41afd97c74fc86e5371e0cf477df5361ee..8c8775548d21702a31d4adc7e9d7f1c6aa30fa4c 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -70,6 +70,9 @@ typedef struct SDiskData SDiskData; typedef struct SDiskDataBuilder SDiskDataBuilder; typedef struct SBlkInfo SBlkInfo; +#define TSDBROW_ROW_FMT ((int8_t)0x0) +#define TSDBROW_COL_FMT ((int8_t)0x1) + #define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_MAX_SUBBLOCKS 8 #define TSDB_FHDR_SIZE 512 @@ -102,26 +105,29 @@ static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) { // tsdbUtil.c ============================================================================================== // TSDBROW -#define TSDBROW_TS(ROW) (((ROW)->type == 0) ? (ROW)->pTSRow->ts : (ROW)->pBlockData->aTSKEY[(ROW)->iRow]) -#define TSDBROW_VERSION(ROW) (((ROW)->type == 0) ? (ROW)->version : (ROW)->pBlockData->aVersion[(ROW)->iRow]) -#define TSDBROW_SVERSION(ROW) TD_ROW_SVER((ROW)->pTSRow) -#define TSDBROW_KEY(ROW) ((TSDBKEY){.version = TSDBROW_VERSION(ROW), .ts = TSDBROW_TS(ROW)}) -#define tsdbRowFromTSRow(VERSION, TSROW) ((TSDBROW){.type = 0, .version = (VERSION), .pTSRow = (TSROW)}) -#define tsdbRowFromBlockData(BLOCKDATA, IROW) ((TSDBROW){.type = 1, .pBlockData = (BLOCKDATA), .iRow = (IROW)}) -void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); -// int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); +#define TSDBROW_TS(ROW) (((ROW)->type == TSDBROW_ROW_FMT) ? (ROW)->pTSRow->ts : (ROW)->pBlockData->aTSKEY[(ROW)->iRow]) +#define TSDBROW_VERSION(ROW) \ + (((ROW)->type == TSDBROW_ROW_FMT) ? (ROW)->version : (ROW)->pBlockData->aVersion[(ROW)->iRow]) +#define TSDBROW_SVERSION(ROW) ((ROW)->type == TSDBROW_ROW_FMT ? (ROW)->pTSRow->sver : -1) +#define TSDBROW_KEY(ROW) ((TSDBKEY){.version = TSDBROW_VERSION(ROW), .ts = TSDBROW_TS(ROW)}) +#define tsdbRowFromTSRow(VERSION, TSROW) ((TSDBROW){.type = TSDBROW_ROW_FMT, .version = (VERSION), .pTSRow = (TSROW)}) +#define tsdbRowFromBlockData(BLOCKDATA, IROW) \ + ((TSDBROW){.type = TSDBROW_COL_FMT, .pBlockData = (BLOCKDATA), .iRow = (IROW)}) + +void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); int32_t tsdbRowCmprFn(const void *p1, const void *p2); // STSDBRowIter -void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); +int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); +void tsdbRowClose(STSDBRowIter *pIter); SColVal *tsdbRowIterNext(STSDBRowIter *pIter); // SRowMerger -int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); -int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); +int32_t tsdbRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); +int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); -int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); -void tRowMergerClear(SRowMerger *pMerger); -int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow); -int32_t tRowMergerGetRow(SRowMerger *pMerger, STSRow **ppRow); +int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); +void tsdbRowMergerClear(SRowMerger *pMerger); +int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow); +int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow); // TABLEID int32_t tTABLEIDCmprFn(const void *p1, const void *p2); // TSDBKEY @@ -146,24 +152,22 @@ int32_t tGetBlockIdx(uint8_t *p, void *ph); int32_t tCmprBlockIdx(void const *lhs, void const *rhs); int32_t tCmprBlockL(void const *lhs, void const *rhs); // SBlockData -#define tBlockDataFirstRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, 0) -#define tBlockDataLastRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1) -#define tBlockDataFirstKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataFirstRow(PBLOCKDATA)) -#define tBlockDataLastKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA)) - -int32_t tBlockDataCreate(SBlockData *pBlockData); -void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear); -int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid); -void tBlockDataReset(SBlockData *pBlockData); -int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); -void tBlockDataClear(SBlockData *pBlockData); -SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx); -void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); -int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); -int32_t tBlockDataAddColData(SBlockData *pBlockData, SColData **ppColData); -int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[], - int32_t aBufN[]); -int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]); +#define tBlockDataFirstRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, 0) +#define tBlockDataLastRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1) +#define tBlockDataFirstKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataFirstRow(PBLOCKDATA)) +#define tBlockDataLastKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA)) +#define tBlockDataGetColDataByIdx(PBLOCKDATA, IDX) (&(PBLOCKDATA)->aColData[IDX]) + +int32_t tBlockDataCreate(SBlockData *pBlockData); +void tBlockDataDestroy(SBlockData *pBlockData); +int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid); +void tBlockDataReset(SBlockData *pBlockData); +int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); +void tBlockDataClear(SBlockData *pBlockData); +void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); +int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[], + int32_t aBufN[]); +int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]); // SDiskDataHdr int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr); int32_t tGetDiskDataHdr(uint8_t *p, void *ph); @@ -205,10 +209,10 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in // tsdbMemTable ============================================================================================== // SMemTable int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); -void tsdbMemTableDestroy(SMemTable *pMemTable); +void tsdbMemTableDestroy(SMemTable *pMemTable, bool proactive); STbData *tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid); -void tsdbRefMemTable(SMemTable *pMemTable); -void tsdbUnrefMemTable(SMemTable *pMemTable); +int32_t tsdbRefMemTable(SMemTable *pMemTable, SQueryNode *pQNode); +int32_t tsdbUnrefMemTable(SMemTable *pMemTable, SQueryNode *pNode, bool proactive); SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable); // STbDataIter int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter); @@ -286,8 +290,8 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader); int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData); int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); // tsdbRead.c ============================================================================================== -int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap, const char *id); -void tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap, const char *id); +int32_t tsdbTakeReadSnap(STsdbReader *pReader, _query_reseek_func_t reseek, STsdbReadSnap **ppSnap); +void tsdbUntakeReadSnap(STsdbReader *pReader, STsdbReadSnap *pSnap, bool proactive); // tsdbMerge.c ============================================================================================== int32_t tsdbMerge(STsdb *pTsdb); @@ -335,10 +339,13 @@ struct SVersionRange { typedef struct SMemSkipListNode SMemSkipListNode; struct SMemSkipListNode { int8_t level; + int8_t flag; // TSDBROW_ROW_FMT for row format, TSDBROW_COL_FMT for col format + int32_t iRow; int64_t version; - STSRow *pTSRow; + void *pData; SMemSkipListNode *forwards[0]; }; + typedef struct SMemSkipList { int64_t size; uint32_t seed; @@ -364,6 +371,8 @@ struct SMemTable { STsdb *pTsdb; SVBufPool *pPool; volatile int32_t nRef; + int64_t minVer; + int64_t maxVer; TSKEY minKey; TSKEY maxKey; int64_t nRow; @@ -376,11 +385,11 @@ struct SMemTable { }; struct TSDBROW { - int8_t type; // 0 for row from tsRow, 1 for row from block data + int8_t type; // TSDBROW_ROW_FMT for row from tsRow, TSDBROW_COL_FMT for row from block data union { struct { int64_t version; - STSRow *pTSRow; + SRow *pTSRow; }; struct { SBlockData *pBlockData; @@ -466,14 +475,14 @@ struct SSttBlk { // (SBlockData){.suid = suid, .uid = 0}: block data for N child tables int .last file // (SBlockData){.suid = 0, .uid = uid}: block data for 1 normal table int .last/.data file struct SBlockData { - int64_t suid; // 0 means normal table block data, otherwise child table block data - int64_t uid; // 0 means block data in .last file, otherwise in .data file - int32_t nRow; // number of rows - int64_t *aUid; // uids of each row, only exist in block data in .last file (uid == 0) - int64_t *aVersion; // versions of each row - TSKEY *aTSKEY; // timestamp of each row - int32_t nColData; - SArray *aColData; // SArray + int64_t suid; // 0 means normal table block data, otherwise child table block data + int64_t uid; // 0 means block data in .last file, otherwise in .data file + int32_t nRow; // number of rows + int64_t *aUid; // uids of each row, only exist in block data in .last file (uid == 0) + int64_t *aVersion; // versions of each row + TSKEY *aTSKEY; // timestamp of each row + int32_t nColData; + SColData *aColData; }; struct TABLEID { @@ -565,10 +574,14 @@ struct SDFileSet { }; struct STSDBRowIter { - TSDBROW *pRow; - STSchema *pTSchema; - SColVal colVal; - int32_t i; + TSDBROW *pRow; + union { + SRowIter *pIter; + struct { + int32_t iColData; + SColVal cv; + }; + }; }; struct SRowMerger { STSchema *pTSchema; @@ -594,9 +607,11 @@ struct SDelFWriter { }; struct STsdbReadSnap { - SMemTable *pMem; - SMemTable *pIMem; - STsdbFS fs; + SMemTable *pMem; + SQueryNode *pNode; + SMemTable *pIMem; + SQueryNode *pINode; + STsdbFS fs; }; struct SDataFWriter { @@ -715,6 +730,9 @@ void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); // tsdbCache ============================================================================================== typedef struct SCacheRowsReader { + STsdb *pTsdb; + SVersionRange verRange; + TdThreadMutex readerMutex; SVnode *pVnode; STSchema *pSchema; uint64_t uid; @@ -739,8 +757,8 @@ typedef struct { int32_t tsdbOpenCache(STsdb *pTsdb); void tsdbCloseCache(STsdb *pTsdb); -int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb); -int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, STSRow *row, bool dup); +int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, TSDBROW *row, STsdb *pTsdb); +int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, TSDBROW *row, bool dup); int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, LRUHandle **h); int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, LRUHandle **h); int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h); @@ -752,6 +770,8 @@ int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); size_t tsdbCacheGetCapacity(SVnode *pVnode); +// int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema); + // ========== inline functions ========== static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { TSDBKEY *pKey1 = (TSDBKEY *)p1; @@ -772,8 +792,8 @@ static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { return 0; } -#define SL_NODE_FORWARD(n, l) ((n)->forwards[l]) -#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) +// #define SL_NODE_FORWARD(n, l) ((n)->forwards[l]) +// #define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) static FORCE_INLINE TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) { if (pIter == NULL) return NULL; @@ -793,8 +813,13 @@ static FORCE_INLINE TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) { } pIter->pRow = &pIter->row; - pIter->pRow->version = pIter->pNode->version; - pIter->pRow->pTSRow = pIter->pNode->pTSRow; + if (pIter->pNode->flag == TSDBROW_ROW_FMT) { + pIter->row = tsdbRowFromTSRow(pIter->pNode->version, pIter->pNode->pData); + } else if (pIter->pNode->flag == TSDBROW_COL_FMT) { + pIter->row = tsdbRowFromBlockData(pIter->pNode->pData, pIter->pNode->iRow); + } else { + ASSERT(0); + } return pIter->pRow; } diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 28797c536166d554aaa99f1d4866e76e85564971..e0d0e9408b1dde6edf29b895a4c3c916105de646 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -61,10 +61,19 @@ struct SVBufPoolNode { }; struct SVBufPool { - SVBufPool* next; + SVBufPool* freeNext; + SVBufPool* recycleNext; + SVBufPool* recyclePrev; + + // query handle list + TdThreadMutex mutex; + int32_t nQuery; + SQueryNode qList; + SVnode* pVnode; - TdThreadSpinlock* lock; + int32_t id; volatile int32_t nRef; + TdThreadSpinlock* lock; int64_t size; uint8_t* ptr; SVBufPoolNode* pTail; @@ -74,6 +83,8 @@ struct SVBufPool { int32_t vnodeOpenBufPool(SVnode* pVnode); int32_t vnodeCloseBufPool(SVnode* pVnode); void vnodeBufPoolReset(SVBufPool* pPool); +void vnodeBufPoolAddToFreeList(SVBufPool* pPool); +int32_t vnodeBufPoolRecycle(SVBufPool* pPool); // vnodeQuery.c int32_t vnodeQueryOpen(SVnode* pVnode); @@ -86,6 +97,7 @@ int32_t vnodeGetBatchMeta(SVnode* pVnode, SRpcMsg* pMsg); // vnodeCommit.c int32_t vnodeBegin(SVnode* pVnode); int32_t vnodeShouldCommit(SVnode* pVnode); +void vnodeUpdCommitSched(SVnode* pVnode); void vnodeRollback(SVnode* pVnode); int32_t vnodeSaveInfo(const char* dir, const SVnodeInfo* pCfg); int32_t vnodeCommitInfo(const char* dir, const SVnodeInfo* pInfo); @@ -98,10 +110,12 @@ bool vnodeShouldRollback(SVnode* pVnode); int32_t vnodeSyncOpen(SVnode* pVnode, char* path); int32_t vnodeSyncStart(SVnode* pVnode); void vnodeSyncPreClose(SVnode* pVnode); +void vnodeSyncPostClose(SVnode* pVnode); void vnodeSyncClose(SVnode* pVnode); void vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg, int32_t code); bool vnodeIsLeader(SVnode* pVnode); bool vnodeIsRoleLeader(SVnode* pVnode); +int vnodeShouldCommit(SVnode* pVnode); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index b2e3245fca5c9f6f5aa774684c5b9ea7863f4b0a..9c5f47b731e2393aca1d21099fbfec6373d97cd0 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -76,6 +76,7 @@ typedef struct SRSmaSnapReader SRSmaSnapReader; typedef struct SRSmaSnapWriter SRSmaSnapWriter; typedef struct SSnapDataHdr SSnapDataHdr; typedef struct SCommitInfo SCommitInfo; +typedef struct SQueryNode SQueryNode; #define VNODE_META_DIR "meta" #define VNODE_TSDB_DIR "tsdb" @@ -87,16 +88,29 @@ typedef struct SCommitInfo SCommitInfo; #define VNODE_RSMA1_DIR "rsma1" #define VNODE_RSMA2_DIR "rsma2" +#define VNODE_BUFPOOL_SEGMENTS 3 + #define VND_INFO_FNAME "vnode.json" // vnd.h +typedef int32_t (*_query_reseek_func_t)(void* pQHandle); +struct SQueryNode { + SQueryNode* pNext; + SQueryNode** ppNext; + void* pQHandle; + _query_reseek_func_t reseek; +}; + void* vnodeBufPoolMalloc(SVBufPool* pPool, int size); void* vnodeBufPoolMallocAligned(SVBufPool* pPool, int size); void vnodeBufPoolFree(SVBufPool* pPool, void* p); void vnodeBufPoolRef(SVBufPool* pPool); -void vnodeBufPoolUnRef(SVBufPool* pPool); +void vnodeBufPoolUnRef(SVBufPool* pPool, bool proactive); int vnodeDecodeInfo(uint8_t* pData, SVnodeInfo* pInfo); +int32_t vnodeBufPoolRegisterQuery(SVBufPool* pPool, SQueryNode* pQNode); +void vnodeBufPoolDeregisterQuery(SVBufPool* pPool, SQueryNode* pQNode, bool proactive); + // meta typedef struct SMCtbCursor SMCtbCursor; typedef struct SMStbCursor SMStbCursor; @@ -162,10 +176,9 @@ int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo); int32_t tsdbFinishCommit(STsdb* pTsdb); int32_t tsdbRollbackCommit(STsdb* pTsdb); int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now); -int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg); -int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); -int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, - SSubmitBlkRsp* pRsp); +int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg); +int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp); +int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows); int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey); int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg); @@ -190,7 +203,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msg int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t version, char* msg, int32_t msgLen); -int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* data, int64_t ver); +int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit); int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver); int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec); @@ -203,10 +216,6 @@ int32_t tqProcessTaskRecoverFinishReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRecoverFinishRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqCheckLogInWal(STQ* pTq, int64_t version); -SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pSchema, - SSchemaWrapper* pTagSchemaWrapper, bool createTb, int64_t suid, const char* stbFullName, - SBatchDeleteReq* pDeleteReq); - // sma int32_t smaInit(); void smaCleanUp(); @@ -223,7 +232,7 @@ int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); int32_t tdProcessRSmaCreate(SSma* pSma, SVCreateStbReq* pReq); -int32_t tdProcessRSmaSubmit(SSma* pSma, void* pMsg, int32_t inputType); +int32_t tdProcessRSmaSubmit(SSma* pSma, int64_t version, void* pReq, void* pMsg, int32_t len, int32_t inputType); int32_t tdProcessRSmaDrop(SSma* pSma, SVDropStbReq* pReq); int32_t tdFetchTbUidList(SSma* pSma, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid); int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore, bool isAdd); @@ -329,17 +338,30 @@ struct STsdbKeepCfg { int32_t keep2; }; +typedef struct SVCommitSched { + int64_t commitMs; + int64_t maxWaitMs; +} SVCommitSched; + struct SVnode { - char* path; - SVnodeCfg config; - SVState state; - SVStatis statis; - STfs* pTfs; - SMsgCb msgCb; + char* path; + SVnodeCfg config; + SVState state; + SVStatis statis; + STfs* pTfs; + SMsgCb msgCb; + + // Buffer Pool TdThreadMutex mutex; TdThreadCond poolNotEmpty; - SVBufPool* pPool; + SVBufPool* aBufPool[VNODE_BUFPOOL_SEGMENTS]; + SVBufPool* freeList; SVBufPool* inUse; + SVBufPool* onCommit; + SVBufPool* recycleHead; + SVBufPool* recycleTail; + SVBufPool* onRecycle; + SMeta* pMeta; SSma* pSma; STsdb* pTsdb; @@ -347,6 +369,7 @@ struct SVnode { STQ* pTq; SSink* pSink; tsem_t canCommit; + SVCommitSched commitSched; int64_t sync; TdThreadMutex lock; bool blocked; diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 513ee5a1c23d140463384971e971e59bad6d6a75..85d8f031fb3f4ab32c806cebec583c5b21656c11 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -55,9 +55,8 @@ struct SMetaCache { // query cache struct STagFilterResCache { TdThreadMutex lock; - SHashObj* pTableEntry; - SLRUCache* pUidResCache; - uint64_t keyBuf[3]; + SHashObj* pTableEntry; + SLRUCache* pUidResCache; } sTagFilterResCache; }; @@ -211,7 +210,7 @@ _exit: int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo) { int32_t code = 0; - // ASSERT(metaIsWLocked(pMeta)); + // meta is wlocked for calling this func. // search SMetaCache* pCache = pMeta->pCache; @@ -222,7 +221,10 @@ int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo) { } if (*ppEntry) { // update - ASSERT(pInfo->suid == (*ppEntry)->info.suid); + if (pInfo->suid != (*ppEntry)->info.suid) { + metaError("meta/cache: suid should be same as the one in cache."); + return TSDB_CODE_FAILED; + } if (pInfo->version > (*ppEntry)->info.version) { (*ppEntry)->info.version = pInfo->version; (*ppEntry)->info.skmVer = pInfo->skmVer; @@ -341,7 +343,7 @@ _exit: int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo) { int32_t code = 0; - // ASSERT(metaIsWLocked(pMeta)); + // meta is wlocked for calling this func. // search SMetaCache* pCache = pMeta->pCache; @@ -429,20 +431,20 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK bool* acquireRes) { // generate the composed key for LRU cache SLRUCache* pCache = pMeta->pCache->sTagFilterResCache.pUidResCache; - uint64_t* pBuf = pMeta->pCache->sTagFilterResCache.keyBuf; SHashObj* pTableMap = pMeta->pCache->sTagFilterResCache.pTableEntry; TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock; + uint64_t buf[3] = {0}; uint32_t times = 0; *acquireRes = 0; - pBuf[0] = suid; - memcpy(&pBuf[1], pKey, keyLen); + buf[0] = suid; + memcpy(&buf[1], pKey, keyLen); taosThreadMutexLock(pLock); int32_t len = keyLen + sizeof(uint64_t); - LRUHandle* pHandle = taosLRUCacheLookup(pCache, pBuf, len); + LRUHandle* pHandle = taosLRUCacheLookup(pCache, buf, len); if (pHandle == NULL) { taosThreadMutexUnlock(pLock); return TSDB_CODE_SUCCESS; @@ -450,7 +452,11 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK // do some book mark work after acquiring the filter result from cache STagFilterResEntry** pEntry = taosHashGet(pTableMap, &suid, sizeof(uint64_t)); - ASSERT(pEntry != NULL); + if (NULL == pEntry) { + metaError("meta/cache: pEntry should not be NULL."); + return TSDB_CODE_FAILED; + } + *acquireRes = 1; const char* p = taosLRUCacheValue(pCache, pHandle); @@ -476,10 +482,10 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK SListNode* pNode = NULL; while ((pNode = tdListNext(&iter)) != NULL) { - memcpy(&pBuf[1], pNode->data, keyLen); + memcpy(&buf[1], pNode->data, keyLen); // check whether it is existed in LRU cache, and remove it from linked list if not. - LRUHandle* pRes = taosLRUCacheLookup(pCache, pBuf, len); + LRUHandle* pRes = taosLRUCacheLookup(pCache, buf, len); if (pRes == NULL) { // remove the item in the linked list taosArrayPush(pInvalidRes, &pNode); } else { @@ -495,7 +501,7 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK taosMemoryFree(*p1); } - atomic_store_32(&(*pEntry)->qTimes, 0); // reset the query times + atomic_store_32(&(*pEntry)->qTimes, 0); // reset the query times taosArrayDestroy(pInvalidRes); taosThreadMutexUnlock(pLock); @@ -547,20 +553,23 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int tdListAppend(&(*pEntry)->list, pKey); } - uint64_t* pBuf = pMeta->pCache->sTagFilterResCache.keyBuf; - pBuf[0] = suid; + uint64_t buf[3] = {0}; + buf[0] = suid; - memcpy(&pBuf[1], pKey, keyLen); - ASSERT(sizeof(uint64_t) + keyLen == 24); + memcpy(&buf[1], pKey, keyLen); + if (sizeof(uint64_t) + keyLen != 24) { + metaError("meta/cache: incorrect keyLen:%" PRId32 " length.", keyLen); + return TSDB_CODE_FAILED; + } // add to cache. - taosLRUCacheInsert(pCache, pBuf, sizeof(uint64_t) + keyLen, pPayload, payloadLen, freePayload, NULL, - TAOS_LRU_PRIORITY_LOW); + int32_t ret = taosLRUCacheInsert(pCache, buf, sizeof(uint64_t) + keyLen, pPayload, payloadLen, freePayload, NULL, + TAOS_LRU_PRIORITY_LOW); taosThreadMutexUnlock(pLock); - metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", TD_VID(pMeta->pVnode), suid, - (int32_t)taosLRUCacheGetUsage(pCache), taosHashGetSize(pTableEntry)); + metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d, ret:%d", TD_VID(pMeta->pVnode), + suid, (int32_t)taosLRUCacheGetUsage(pCache), taosHashGetSize(pTableEntry), ret); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index 72f7365a1e3304c8fa5c387eea24a968ebb87c14..e50931ac0618252eea58afc0d1a734d252a05d1e 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -51,7 +51,9 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { } else if (pME->type == TSDB_TSMA_TABLE) { if (tEncodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1; } else { - ASSERT(0); + metaError("meta/entry: invalide table type: %" PRId8 " encode failed.", pME->type); + + return -1; } tEndEncode(pCoder); @@ -99,7 +101,9 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { } if (tDecodeTSma(pCoder, pME->smaEntry.tsma, true) < 0) return -1; } else { - ASSERT(0); + metaError("meta/entry: invalide table type: %" PRId8 " decode failed.", pME->type); + + return -1; } tEndDecode(pCoder); diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 1b5f74255948da71938eb274fe3cfd05700d8aa9..609ffc58c3d4ed00c3b5db729179a3559b36e93d 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -203,6 +203,7 @@ _err: int metaClose(SMeta *pMeta) { if (pMeta) { + if (pMeta->pEnv) tdbAbort(pMeta->pEnv, pMeta->txn); if (pMeta->pCache) metaCacheClose(pMeta); if (pMeta->pIdx) metaCloseIdx(pMeta); if (pMeta->pStreamDb) tdbTbClose(pMeta->pStreamDb); @@ -358,7 +359,10 @@ static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL return -1; } - ASSERT(pTagIdxKey1->type == pTagIdxKey2->type); + if (pTagIdxKey1->type != pTagIdxKey2->type) { + metaError("meta/open: incorrect tag idx type."); + return TSDB_CODE_FAILED; + } // check NULL, NULL is always the smallest if (pTagIdxKey1->isNull && !pTagIdxKey2->isNull) { diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 6c468b89f66e438d2f923a034f18f1f02282d349..4b280a32f1f5915876dbeb8fd0ff4b6beaa4c658 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -310,7 +310,7 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) { } } -int metaTbCursorNext(SMTbCursor *pTbCur) { +int metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType) { int ret; void *pBuf; STbCfg tbCfg; @@ -324,7 +324,7 @@ int metaTbCursorNext(SMTbCursor *pTbCur) { tDecoderClear(&pTbCur->mr.coder); metaGetTableEntryByVersion(&pTbCur->mr, ((SUidIdxVal *)pTbCur->pVal)[0].version, *(tb_uid_t *)pTbCur->pKey); - if (pTbCur->mr.me.type == TSDB_SUPER_TABLE) { + if (pTbCur->mr.me.type == jumpTableType) { continue; } @@ -611,23 +611,14 @@ tb_uid_t metaStbCursorNext(SMStbCursor *pStbCur) { } STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) { - // SMetaReader mr = {0}; STSchema *pTSchema = NULL; SSchemaWrapper *pSW = NULL; - STSchemaBuilder sb = {0}; - SSchema *pSchema; + SSchema *pSchema = NULL; pSW = metaGetTableSchema(pMeta, uid, sver, lock); if (!pSW) return NULL; - tdInitTSchemaBuilder(&sb, pSW->version); - for (int i = 0; i < pSW->nCols; i++) { - pSchema = pSW->pSchema + i; - tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes); - } - pTSchema = tdGetSchemaFromBuilder(&sb); - - tdDestroyTSchemaBuilder(&sb); + pTSchema = tBuildTSchema(pSW->pSchema, pSW->nCols, pSW->version); taosMemoryFree(pSW->pSchema); taosMemoryFree(pSW); @@ -661,7 +652,13 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv goto _exit; } - ASSERT(c); + if (c == 0) { + metaULock(pMeta); + tdbTbcClose(pSkmDbC); + code = TSDB_CODE_FAILED; + metaError("meta/query: incorrect c: %" PRId32 ".", c); + goto _exit; + } if (c < 0) { tdbTbcMoveToPrev(pSkmDbC); @@ -685,7 +682,11 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv } } - ASSERT(sver > 0); + if (sver <= 0) { + metaError("meta/query: incorrect sver: %" PRId32 ".", sver); + code = TSDB_CODE_FAILED; + goto _exit; + } skmDbKey.uid = suid ? suid : uid; skmDbKey.sver = sver; @@ -708,21 +709,11 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv tdbFree(pData); // convert - STSchemaBuilder sb = {0}; - - tdInitTSchemaBuilder(&sb, pSchemaWrapper->version); - for (int i = 0; i < pSchemaWrapper->nCols; i++) { - SSchema *pSchema = pSchemaWrapper->pSchema + i; - tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes); - } - - STSchema *pTSchema = tdGetSchemaFromBuilder(&sb); + STSchema *pTSchema = tBuildTSchema(pSchemaWrapper->pSchema, pSchemaWrapper->nCols, pSchemaWrapper->version); if (pTSchema == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; } - tdDestroyTSchemaBuilder(&sb); - *ppTSchema = pTSchema; taosMemoryFree(pSchemaWrapper->pSchema); diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 054e785980370f85afd2782896cbb4095349b692..6e505dfde512e53607e7410dbf5f31c61f5ffdd6 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -100,7 +100,10 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { break; } - ASSERT(pData && nData); + if (!pData || !nData) { + metaError("meta/snap: invalide nData: %" PRId32 " meta snap read failed.", nData); + goto _exit; + } *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + nData); if (*ppData == NULL) { @@ -113,8 +116,8 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { pHdr->size = nData; memcpy(pHdr->data, pData, nData); - metaInfo("vgId:%d, vnode snapshot meta read data, version:%" PRId64 " uid:%" PRId64 " nData:%d", - TD_VID(pReader->pMeta->pVnode), key.version, key.uid, nData); + metaDebug("vgId:%d, vnode snapshot meta read data, version:%" PRId64 " uid:%" PRId64 " blockLen:%d", + TD_VID(pReader->pMeta->pVnode), key.version, key.uid, nData); _exit: return code; @@ -189,7 +192,8 @@ int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) SDecoder* pDecoder = &(SDecoder){0}; tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); - metaDecodeEntry(pDecoder, &metaEntry); + code = metaDecodeEntry(pDecoder, &metaEntry); + if (code) goto _err; code = metaHandleEntry(pMeta, &metaEntry); if (code) goto _err; @@ -198,6 +202,7 @@ int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) return code; _err: + tDecoderClear(pDecoder); metaError("vgId:%d, vnode snapshot meta write failed since %s", TD_VID(pMeta->pVnode), tstrerror(code)); return code; } @@ -356,7 +361,11 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t for (int i = 0; i < taosArrayGetSize(ctx->idList); i++) { int64_t* uid = taosArrayGet(ctx->idList, i); SIdInfo* idData = (SIdInfo*)taosHashGet(ctx->idVersion, uid, sizeof(int64_t)); - ASSERT(idData); + if (!idData) { + metaError("meta/snap: null idData"); + return TSDB_CODE_FAILED; + } + idData->index = i; metaDebug("tmqsnap init idVersion uid:%" PRIi64 " version:%" PRIi64 " index:%d", *uid, idData->version, idData->index); @@ -473,7 +482,10 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in int64_t* uidTmp = taosArrayGet(ctx->idList, ctx->index); ctx->index++; SIdInfo* idInfo = (SIdInfo*)taosHashGet(ctx->idVersion, uidTmp, sizeof(tb_uid_t)); - ASSERT(idInfo); + if (!idInfo) { + metaError("meta/snap: null idInfo"); + return TSDB_CODE_FAILED; + } *uid = *uidTmp; ret = MoveToPosition(ctx, idInfo->version, *uidTmp); @@ -507,7 +519,11 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in (ctx->subType == TOPIC_SUB_TYPE__TABLE && me.type == TSDB_CHILD_TABLE && me.ctbEntry.suid == ctx->suid)) { STableInfoForChildTable* data = (STableInfoForChildTable*)taosHashGet(ctx->suidInfo, &me.ctbEntry.suid, sizeof(tb_uid_t)); - ASSERT(data); + if (!data) { + metaError("meta/snap: null data"); + return TSDB_CODE_FAILED; + } + SVCreateTbReq req = {0}; req.type = TSDB_CHILD_TABLE; @@ -528,7 +544,8 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in } else { SArray* pTagVals = NULL; if (tTagToValArray((const STag*)p, &pTagVals) != 0) { - ASSERT(0); + metaError("meta/snap: tag to val array failed."); + return TSDB_CODE_FAILED; } int16_t nCols = taosArrayGetSize(pTagVals); for (int j = 0; j < nCols; ++j) { @@ -572,7 +589,8 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in ret = buildNormalChildTableInfo(&req, pBuf, contLen); *type = TDMT_VND_CREATE_TABLE; } else { - ASSERT(0); + metaError("meta/snap: invalid topic sub type: %" PRId8 " get meta from snap failed.", ctx->subType); + ret = -1; } tDecoderClear(&dc); @@ -593,7 +611,10 @@ SMetaTableInfo getUidfromSnapShot(SSnapContext* ctx) { int64_t* uidTmp = taosArrayGet(ctx->idList, ctx->index); ctx->index++; SIdInfo* idInfo = (SIdInfo*)taosHashGet(ctx->idVersion, uidTmp, sizeof(tb_uid_t)); - ASSERT(idInfo); + if (!idInfo) { + metaError("meta/snap: null idInfo"); + return result; + } int32_t ret = MoveToPosition(ctx, idInfo->version, *uidTmp); if (ret != 0) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 60a27bec102d78983747aa89fa1dd11c9f0c10f3..78e0643e1076c04529e04c98e347016aeec0076b 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -46,7 +46,7 @@ static void metaGetEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) { pInfo->suid = 0; pInfo->skmVer = pEntry->ntbEntry.schemaRow.version; } else { - ASSERT(0); + metaError("meta/table: invalide table type: %" PRId8 " get entry info failed.", pEntry->type); } } @@ -342,10 +342,23 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c); - ASSERT(ret == 0 && c == 0); + if (!(ret == 0 && c == 0)) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + + terrno = TSDB_CODE_TDB_STB_NOT_EXIST; + metaError("meta/table: invalide ret: %" PRId32 " or c: %" PRId32 "alter stb failed.", ret, c); + return -1; + } ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); - ASSERT(ret == 0); + if (ret < 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + + terrno = TSDB_CODE_TDB_STB_NOT_EXIST; + return -1; + } oStbEntry.pBuf = taosMemoryMalloc(nData); memcpy(oStbEntry.pBuf, pData, nData); @@ -558,7 +571,8 @@ static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME) { ctime = pME->ntbEntry.ctime; ttlDays = pME->ntbEntry.ttlDays; } else { - ASSERT(0); + metaError("meta/table: invalide table type: %" PRId8 " build ttl idx key failed.", pME->type); + return; } if (ttlDays <= 0) return; @@ -773,7 +787,11 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); - ASSERT(c == 0); + if (c != 0) { + tdbTbcClose(pUidIdxc); + metaError("meta/table: invalide c: %" PRId32 " alt tb column failed.", c); + return -1; + } tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); oversion = ((SUidIdxVal *)pData)[0].version; @@ -783,7 +801,13 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); - ASSERT(c == 0); + if (c != 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + metaError("meta/table: invalide c: %" PRId32 " alt tb column failed.", c); + return -1; + } + tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); // get table entry @@ -792,7 +816,13 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl memcpy(entry.pBuf, pData, nData); tDecoderInit(&dc, entry.pBuf, nData); ret = metaDecodeEntry(&dc, &entry); - ASSERT(ret == 0); + if (ret != 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + tDecoderClear(&dc); + metaError("meta/table: invalide ret: %" PRId32 " alt tb column failed.", ret); + return -1; + } if (entry.type != TSDB_NORMAL_TABLE) { terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; @@ -812,7 +842,11 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl if (iCol >= pSchema->nCols) break; pColumn = &pSchema->pSchema[iCol]; - ASSERT(pAlterTbReq->colName); + if (NULL == pAlterTbReq->colName) { + metaError("meta/table: null pAlterTbReq->colName"); + return -1; + } + if (strcmp(pColumn->name, pAlterTbReq->colName) == 0) break; iCol++; } @@ -964,7 +998,12 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); - ASSERT(c == 0); + if (c != 0) { + tdbTbcClose(pUidIdxc); + terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); + return -1; + } tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); oversion = ((SUidIdxVal *)pData)[0].version; @@ -977,7 +1016,14 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA /* get ctbEntry */ tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); - ASSERT(c == 0); + if (c != 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); + return -1; + } + tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); ctbEntry.pBuf = taosMemoryMalloc(nData); @@ -1075,7 +1121,11 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA metaUpdateTagIdx(pMeta, &ctbEntry); } - ASSERT(ctbEntry.ctbEntry.pTags); + if (NULL == ctbEntry.ctbEntry.pTags) { + metaError("meta/table: null tags, update tag val failed."); + goto _err; + } + SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, ((STag *)(ctbEntry.ctbEntry.pTags))->len, pMeta->txn); @@ -1130,7 +1180,11 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); - ASSERT(c == 0); + if (c != 0) { + tdbTbcClose(pUidIdxc); + metaError("meta/table: invalide c: %" PRId32 " update tb options failed.", c); + return -1; + } tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); oversion = ((SUidIdxVal *)pData)[0].version; @@ -1140,7 +1194,13 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); - ASSERT(c == 0); + if (c != 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + metaError("meta/table: invalide c: %" PRId32 " update tb options failed.", c); + return -1; + } + tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); // get table entry @@ -1149,7 +1209,13 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p memcpy(entry.pBuf, pData, nData); tDecoderInit(&dc, entry.pBuf, nData); ret = metaDecodeEntry(&dc, &entry); - ASSERT(ret == 0); + if (ret != 0) { + tDecoderClear(&dc); + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + metaError("meta/table: invalide ret: %" PRId32 " alt tb options failed.", ret); + return -1; + } entry.version = version; metaWLock(pMeta); @@ -1408,7 +1474,8 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { } else if (pME->type == TSDB_NORMAL_TABLE) { pSW = &pME->ntbEntry.schemaRow; } else { - ASSERT(0); + metaError("meta/table: invalide table type: %" PRId8 " save skm db failed.", pME->type); + return TSDB_CODE_FAILED; } skmDbKey.uid = pME->uid; diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 8b43de7421a6329273ede2eefb0a047d94ba5c98..38f04bb8e8bea7f12e7fb128775d5a80f54bb51a 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -116,68 +116,6 @@ _exit: return code; } -// SQTaskFile ====================================================== - -/** - * @brief At most time, there is only one qtaskinfo file committed latest in aTaskFile. Sometimes, there would be - * multiple qtaskinfo files supporting snapshot replication. - * - * @param pSma - * @param pStat - * @return int32_t - */ -static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pStat) { -#if 0 - SVnode *pVnode = pSma->pVnode; - SRSmaFS *pFS = RSMA_FS(pStat); - int64_t committed = pStat->commitAppliedVer; - int64_t fsMaxVer = -1; - char qTaskInfoFullName[TSDB_FILENAME_LEN]; - - taosWLockLatch(RSMA_FS_LOCK(pStat)); - - for (int32_t i = 0; i < taosArrayGetSize(pFS->aQTaskInf);) { - SQTaskFile *pTaskF = taosArrayGet(pFS->aQTaskInf, i); - int32_t oldVal = atomic_fetch_sub_32(&pTaskF->nRef, 1); - if ((oldVal <= 1) && (pTaskF->version < committed)) { - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->version, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); - if (taosRemoveFile(qTaskInfoFullName) < 0) { - smaWarn("vgId:%d, cleanup qinf, committed %" PRIi64 ", failed to remove %s since %s", TD_VID(pVnode), committed, - qTaskInfoFullName, tstrerror(TAOS_SYSTEM_ERROR(errno))); - } else { - smaDebug("vgId:%d, cleanup qinf, committed %" PRIi64 ", success to remove %s", TD_VID(pVnode), committed, - qTaskInfoFullName); - } - taosArrayRemove(pFS->aQTaskInf, i); - continue; - } - ++i; - } - - if (taosArrayGetSize(pFS->aQTaskInf) > 0) { - fsMaxVer = ((SQTaskFile *)taosArrayGetLast(pFS->aQTaskInf))->version; - } - - if (fsMaxVer < committed) { - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), committed, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); - if (taosCheckExistFile(qTaskInfoFullName)) { - SQTaskFile qFile = {.nRef = 1, .padding = 0, .version = committed, .size = 0}; - if (!taosArrayPush(pFS->aQTaskInf, &qFile)) { - taosWUnLockLatch(RSMA_FS_LOCK(pStat)); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - } - } else { - smaDebug("vgId:%d, update qinf, no need as committed %" PRIi64 " not larger than fsMaxVer %" PRIi64, TD_VID(pVnode), - committed, fsMaxVer); - } - - taosWUnLockLatch(RSMA_FS_LOCK(pStat)); -#endif - return TSDB_CODE_SUCCESS; -} - /** * @brief Rsma async commit implementation(only do some necessary light weighted task) * 1) set rsma stat TASK_TRIGGER_STAT_PAUSED @@ -187,7 +125,8 @@ static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pStat) { * @return int32_t */ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { - int32_t code = 0; + int32_t code = 0; + int32_t lino = 0; SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); if (!pEnv) { @@ -208,7 +147,11 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { } } pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied; - // ASSERT(pRSmaStat->commitAppliedVer > 0); + if (ASSERTS(pRSmaStat->commitAppliedVer >= 0, "commit applied version %" PRIi64 " < 0", + pRSmaStat->commitAppliedVer)) { + code = TSDB_CODE_APP_ERROR; + TSDB_CHECK_CODE(code, lino, _exit); + } // step 2: wait for all triggered fetch tasks to finish nLoops = 0; @@ -242,9 +185,9 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { } } smaInfo("vgId:%d, rsma commit, all items are consumed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); - if ((code = tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat))) != 0) { - return code; - } + code = tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)); + TSDB_CHECK_CODE(code, lino, _exit); + smaInfo("vgId:%d, rsma commit, operator state committed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); #if 0 // consuming task of qTaskInfo clone @@ -252,8 +195,6 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { // lock taosWLockLatch(SMA_ENV_LOCK(pEnv)); - ASSERT(RSMA_INFO_HASH(pRSmaStat)); - void *pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL); while (pIter) { @@ -271,9 +212,19 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { // all rsma results are written completely STsdb *pTsdb = NULL; - if ((pTsdb = VND_RSMA1(pSma->pVnode))) tsdbPrepareCommit(pTsdb); - if ((pTsdb = VND_RSMA2(pSma->pVnode))) tsdbPrepareCommit(pTsdb); + if ((pTsdb = VND_RSMA1(pSma->pVnode))) { + code = tsdbPrepareCommit(pTsdb); + TSDB_CHECK_CODE(code, lino, _exit); + } + if ((pTsdb = VND_RSMA2(pSma->pVnode))) { + code = tsdbPrepareCommit(pTsdb); + TSDB_CHECK_CODE(code, lino, _exit); + } +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pSma), __func__, lino, tstrerror(code)); + } return code; } @@ -288,6 +239,11 @@ static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma, SCommitInfo *pInfo) { int32_t lino = 0; SVnode *pVnode = pSma->pVnode; + SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); + if (!pSmaEnv) { + goto _exit; + } + code = tdRSmaFSCommit(pSma); TSDB_CHECK_CODE(code, lino, _exit); @@ -360,8 +316,6 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); } - tdUpdateQTaskInfoFiles(pSma, pRSmaStat); - atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 0); return TSDB_CODE_SUCCESS; diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index e613b5833fa76dbfe2adc03e0a57d32261ec6156..a980b653a9c9d7e2e3f71c2502464e0fe68ff986 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -131,7 +131,7 @@ static int32_t tdNewSmaEnv(SSma *pSma, int8_t smaType, SSmaEnv **ppEnv) { (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&SMA_TSMA_ENV(pSma), *ppEnv) : atomic_store_ptr(&SMA_RSMA_ENV(pSma), *ppEnv); - if (tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType, pSma) != TSDB_CODE_SUCCESS) { + if ((terrno = tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType, pSma)) != TSDB_CODE_SUCCESS) { tdFreeSmaEnv(pEnv); *ppEnv = NULL; (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&SMA_TSMA_ENV(pSma), NULL) @@ -193,10 +193,16 @@ static void tRSmaInfoHashFreeNode(void *data) { } static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pSma) { - ASSERT(pSmaStat != NULL); + int32_t code = 0; + int32_t lino = 0; + + if (ASSERTS(pSmaStat != NULL, "pSmaStat is NULL")) { + terrno = TSDB_CODE_RSMA_INVALID_ENV; + TSDB_CHECK_CODE(code, lino, _exit); + } if (*pSmaStat) { // no lock - return TSDB_CODE_SUCCESS; + return code; // success, return directly } /** @@ -207,8 +213,8 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS if (!(*pSmaStat)) { *pSmaStat = (SSmaStat *)taosMemoryCalloc(1, sizeof(SSmaStat) + sizeof(TdThread) * tsNumOfVnodeRsmaThreads); if (!(*pSmaStat)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } if (smaType == TSDB_SMA_TYPE_ROLLUP) { @@ -224,7 +230,8 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS if (refId < 0) { smaError("vgId:%d, taosAddRef refId:%" PRIi64 " to rsetId rsetId:%d max:%d failed since:%s", SMA_VID(pSma), refId, smaMgmt.rsetId, SMA_MGMT_REF_NUM, tstrerror(terrno)); - return TSDB_CODE_FAILED; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } else { smaDebug("vgId:%d, taosAddRef refId:%" PRIi64 " to rsetId rsetId:%d max:%d succeed", SMA_VID(pSma), refId, smaMgmt.rsetId, SMA_MGMT_REF_NUM); @@ -235,22 +242,30 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS RSMA_INFO_HASH(pRSmaStat) = taosHashInit( RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (!RSMA_INFO_HASH(pRSmaStat)) { - return TSDB_CODE_FAILED; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } taosHashSetFreeFp(RSMA_INFO_HASH(pRSmaStat), tRSmaInfoHashFreeNode); if (tdRsmaStartExecutor(pSma) < 0) { - return TSDB_CODE_FAILED; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } taosInitRWLatch(RSMA_FS_LOCK(pRSmaStat)); } else if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { // TODO } else { - ASSERT(0); + ASSERTS(0, "unknown smaType:%" PRIi8, smaType); + code = TSDB_CODE_APP_ERROR; + TSDB_CHECK_CODE(code, lino, _exit); } } - return TSDB_CODE_SUCCESS; +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pSma), __func__, lino, tstrerror(code)); + } + return code; } static void tdDestroyTSmaStat(STSmaStat *pStat) { @@ -339,7 +354,10 @@ static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { smaDebug("vgId:%d, remove refId:%" PRIi64 " from rsmaRef:%" PRIi32 " succeed", vid, refId, smaMgmt.rsetId); } } else { - ASSERT(0); + ASSERTS(0, "unknown smaType:%" PRIi8, smaType); + terrno = TSDB_CODE_APP_ERROR; + smaError("%s failed at line %d since %s", __func__, __LINE__, terrstr()); + return -1; } } return 0; @@ -348,7 +366,7 @@ static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { int32_t tdLockSma(SSma *pSma) { int code = taosThreadMutexLock(&pSma->mutex); if (code != 0) { - smaError("vgId:%d, failed to lock td since %s", SMA_VID(pSma), strerror(errno)); + smaError("vgId:%d, failed to lock since %s", SMA_VID(pSma), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); return -1; } @@ -357,12 +375,17 @@ int32_t tdLockSma(SSma *pSma) { } int32_t tdUnLockSma(SSma *pSma) { - ASSERT(SMA_LOCKED(pSma)); + if (ASSERTS(SMA_LOCKED(pSma), "pSma %p is not locked:%d", pSma, pSma->locked)) { + terrno = TSDB_CODE_APP_ERROR; + smaError("vgId:%d, failed to unlock since %s", SMA_VID(pSma), tstrerror(terrno)); + return -1; + } + pSma->locked = false; int code = taosThreadMutexUnlock(&pSma->mutex); if (code != 0) { - smaError("vgId:%d, failed to unlock td since %s", SMA_VID(pSma), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); + smaError("vgId:%d, failed to unlock since %s", SMA_VID(pSma), strerror(errno)); return -1; } return 0; diff --git a/source/dnode/vnode/src/sma/smaFS.c b/source/dnode/vnode/src/sma/smaFS.c index b7c2538b416ac59e7173c7ad7e8b0df9c59f9234..ef872d055e5eec598af61bb9f410f6361b01c024 100644 --- a/source/dnode/vnode/src/sma/smaFS.c +++ b/source/dnode/vnode/src/sma/smaFS.c @@ -77,7 +77,7 @@ static int32_t tsdbBinaryToFS(uint8_t *pData, int64_t nData, SRSmaFS *pFS) { int32_t nt = tdRSmaGetQTaskF(pData + n, &qTaskF); if (nt < 0) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_FILE_CORRUPTED; goto _exit; } @@ -88,7 +88,11 @@ static int32_t tsdbBinaryToFS(uint8_t *pData, int64_t nData, SRSmaFS *pFS) { } } - ASSERT(n + sizeof(TSCKSUM) == nData); + if (ASSERTS(n + sizeof(TSCKSUM) == nData, "n:%d + sizeof(TSCKSUM):%d != nData:%d", n, (int32_t)sizeof(TSCKSUM), + nData)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } _exit: return code; @@ -545,87 +549,6 @@ int32_t tdRSmaFSUpsertQTaskFile(SSma *pSma, SRSmaFS *pFS, SQTaskFile *qTaskFile, _exit: return code; } -#if 0 -int32_t tdRSmaFSRef(SSma *pSma, SRSmaStat *pStat, int64_t suid, int8_t level, int64_t version) { - SArray *aQTaskInf = RSMA_FS(pStat)->aQTaskInf; - SQTaskFile qTaskF = {.level = level, .suid = suid, .version = version}; - SQTaskFile *pTaskF = NULL; - int32_t oldVal = 0; - - taosRLockLatch(RSMA_FS_LOCK(pStat)); - if (suid > 0 && level > 0) { - ASSERT(version > 0); - if ((pTaskF = taosArraySearch(aQTaskInf, &qTaskF, tdQTaskInfCmprFn1, TD_EQ))) { - oldVal = atomic_fetch_add_32(&pTaskF->nRef, 1); - ASSERT(oldVal > 0); - } - } else { - // ref all - int32_t size = taosArrayGetSize(aQTaskInf); - for (int32_t i = 0; i < size; ++i) { - pTaskF = TARRAY_GET_ELEM(aQTaskInf, i); - oldVal = atomic_fetch_add_32(&pTaskF->nRef, 1); - ASSERT(oldVal > 0); - } - } - taosRUnLockLatch(RSMA_FS_LOCK(pStat)); - return oldVal; -} - -void tdRSmaFSUnRef(SSma *pSma, SRSmaStat *pStat, int64_t suid, int8_t level, int64_t version) { - SVnode *pVnode = pSma->pVnode; - SArray *aQTaskInf = RSMA_FS(pStat)->aQTaskInf; - char qTaskFullName[TSDB_FILENAME_LEN]; - SQTaskFile qTaskF = {.level = level, .suid = suid, .version = version}; - SQTaskFile *pTaskF = NULL; - int32_t idx = -1; - - taosWLockLatch(RSMA_FS_LOCK(pStat)); - if (suid > 0 && level > 0) { - ASSERT(version > 0); - if ((idx = taosArraySearchIdx(aQTaskInf, &qTaskF, tdQTaskInfCmprFn1, TD_EQ)) >= 0) { - ASSERT(idx < taosArrayGetSize(aQTaskInf)); - pTaskF = taosArrayGet(aQTaskInf, idx); - if (atomic_sub_fetch_32(&pTaskF->nRef, 1) <= 0) { - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, level, pTaskF->version, - tfsGetPrimaryPath(pVnode->pTfs), qTaskFullName); - if (taosRemoveFile(qTaskFullName) < 0) { - smaWarn("vgId:%d, failed to remove %s since %s", TD_VID(pVnode), qTaskFullName, - tstrerror(TAOS_SYSTEM_ERROR(errno))); - } else { - smaDebug("vgId:%d, success to remove %s", TD_VID(pVnode), qTaskFullName); - } - taosArrayRemove(aQTaskInf, idx); - } - } - } else { - for (int32_t i = 0; i < taosArrayGetSize(aQTaskInf);) { - pTaskF = TARRAY_GET_ELEM(aQTaskInf, i); - int32_t nRef = INT32_MAX; - if (pTaskF->version == version) { - nRef = atomic_sub_fetch_32(&pTaskF->nRef, 1); - } else if (pTaskF->version < version) { - nRef = atomic_load_32(&pTaskF->nRef); - } - if (nRef <= 0) { - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, pTaskF->level, pTaskF->version, - tfsGetPrimaryPath(pVnode->pTfs), qTaskFullName); - if (taosRemoveFile(qTaskFullName) < 0) { - smaWarn("vgId:%d, failed to remove %s since %s", TD_VID(pVnode), qTaskFullName, - tstrerror(TAOS_SYSTEM_ERROR(errno))); - } else { - smaDebug("vgId:%d, success to remove %s", TD_VID(pVnode), qTaskFullName); - } - taosArrayRemove(aQTaskInf, i); - continue; - } - ++i; - } - } - - taosWUnLockLatch(RSMA_FS_LOCK(pStat)); -} -#endif int32_t tdRSmaFSRef(SSma *pSma, SRSmaFS *pFS) { int32_t code = 0; diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index 3923a5dd5b37c3787ba64543b15071c4b73bba43..21c283994345a00badb3e5c12c80f1e3b86bdc6f 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -34,13 +34,16 @@ static int32_t rsmaRestore(SSma *pSma); SRetention *r = (SRetention *)VND_RETENTIONS(v) + l; \ if (!RETENTION_VALID(r)) { \ if (l == 0) { \ - goto _err; \ + code = TSDB_CODE_INVALID_PARA; \ + TSDB_CHECK_CODE(code, lino, _exit); \ } \ break; \ } \ - smaSetKeepCfg(v, &keepCfg, pCfg, TSDB_TYPE_RSMA_L##l); \ + code = smaSetKeepCfg(v, &keepCfg, pCfg, TSDB_TYPE_RSMA_L##l); \ + TSDB_CHECK_CODE(code, lino, _exit); \ if (tsdbOpen(v, &SMA_RSMA_TSDB##l(pSma), VNODE_RSMA##l##_DIR, &keepCfg, rollback) < 0) { \ - goto _err; \ + code = terrno; \ + TSDB_CHECK_CODE(code, lino, _exit); \ } \ } while (0) @@ -68,12 +71,10 @@ static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t p days = keepDuration; } - if (level == TSDB_RETENTION_L0) { - goto end; + if (level < TSDB_RETENTION_L1 || level > TSDB_RETENTION_L2) { + goto _exit; } - ASSERT(level >= TSDB_RETENTION_L1 && level <= TSDB_RETENTION_L2); - freqDuration = convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE); keepDuration = convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE); @@ -91,16 +92,18 @@ static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t p if (days < freqDuration) { days = freqDuration; } -end: +_exit: smaInfo("vgId:%d, evaluated duration for level %d is %d, raw val:%d", TD_VID(pVnode), level + 1, days, duration); return days; } int smaSetKeepCfg(SVnode *pVnode, STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type) { + terrno = 0; pKeepCfg->precision = pCfg->precision; switch (type) { case TSDB_TYPE_TSMA: - ASSERT(0); + ASSERTS(0, "undefined smaType:%d", (int32_t)type); + terrno = TSDB_CODE_APP_ERROR; break; case TSDB_TYPE_RSMA_L0: SMA_SET_KEEP_CFG(pVnode, 0); @@ -112,19 +115,22 @@ int smaSetKeepCfg(SVnode *pVnode, STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int ty SMA_SET_KEEP_CFG(pVnode, 2); break; default: - ASSERT(0); + ASSERTS(0, "unknown smaType:%d", (int32_t)type); + terrno = TSDB_CODE_APP_ERROR; break; } - return 0; + return terrno; } int32_t smaOpen(SVnode *pVnode, int8_t rollback) { + int32_t code = 0; + int32_t lino = 0; STsdbCfg *pCfg = &pVnode->config.tsdbCfg; SSma *pSma = taosMemoryCalloc(1, sizeof(SSma)); if (!pSma) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } pVnode->pSma = pSma; @@ -142,22 +148,21 @@ int32_t smaOpen(SVnode *pVnode, int8_t rollback) { SMA_OPEN_RSMA_IMPL(pVnode, 1); } else if (i == TSDB_RETENTION_L2) { SMA_OPEN_RSMA_IMPL(pVnode, 2); - } else { - terrno = TSDB_CODE_APP_ERROR; - smaError("vgId:%d, sma open failed since %s, level:%d", TD_VID(pVnode), terrstr(), i); - goto _err; } } // restore the rsma if (tdRSmaRestore(pSma, RSMA_RESTORE_REBOOT, pVnode->state.committed, rollback) < 0) { - goto _err; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } } - return 0; -_err: - return -1; +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } + return code; } int32_t smaClose(SSma *pSma) { diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 044e1d5bab36361455fa3df6f52573e43baee339..809e553ab665b9473e833759c7f0b278df62354e 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -129,12 +129,17 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { } static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) { - ASSERT(*pStore == NULL); + if (ASSERTS(*pStore == NULL, "*pStore:%p != NULL", *pStore)) { + terrno = TSDB_CODE_APP_ERROR; + return TSDB_CODE_FAILED; + } + *pStore = taosMemoryCalloc(1, sizeof(STbUidStore)); if (*pStore == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_FAILED; } + return TSDB_CODE_SUCCESS; } @@ -163,15 +168,14 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pRSmaInfo->taskInfo[i]) { - if (((terrno = qUpdateQualifiedTableId(pRSmaInfo->taskInfo[i], tbUids, isAdd)) < 0)) { + if ((terrno = qUpdateQualifiedTableId(pRSmaInfo->taskInfo[i], tbUids, isAdd)) < 0) { tdReleaseRSmaInfo(pSma, pRSmaInfo); smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " level %d since %s", SMA_VID(pSma), *suid, i, terrstr()); return TSDB_CODE_FAILED; - } else { - smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 " uid:%" PRIi64 " level %d", - SMA_VID(pSma), pRSmaInfo->taskInfo[0], *suid, *(int64_t *)taosArrayGet(tbUids, 0), i); } + smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 " uid:%" PRIi64 " level %d", + SMA_VID(pSma), pRSmaInfo->taskInfo[i], *suid, *(int64_t *)taosArrayGet(tbUids, 0), i); } } @@ -232,8 +236,6 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui return TSDB_CODE_SUCCESS; } - ASSERT(ppStore != NULL); - if (!(*ppStore)) { if (tdUidStoreInit(ppStore) < 0) { return TSDB_CODE_FAILED; @@ -300,12 +302,15 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat } pItem->level = idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2; - ASSERT(pItem->level > 0); + + if (ASSERTS(pItem->level > 0, "pItem level:%" PRIi8 " should > 0", pItem->level)) { + terrno = TSDB_CODE_APP_ERROR; + return TSDB_CODE_FAILED; + } SRSmaRef rsmaRef = {.refId = pStat->refId, .suid = pRSmaInfo->suid}; taosHashPut(smaMgmt.refHash, &pItem, POINTER_BYTES, &rsmaRef, sizeof(rsmaRef)); - pItem->fetchLevel = pItem->level; taosTmrReset(tdRSmaFetchTrigger, RSMA_FETCH_INTERVAL, pItem, smaMgmt.tmrHandle, &pItem->tmrId); smaInfo("vgId:%d, item:%p table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64 @@ -565,8 +570,8 @@ static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { return TSDB_CODE_FAILED; } - SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; - // TODO: spin lock for race condition + SSubmitReq2 *pSubmitReq = (SSubmitReq2 *)pReq; + // spin lock for race condition during insert data if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { return TSDB_CODE_FAILED; } @@ -574,29 +579,19 @@ static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { return TSDB_CODE_SUCCESS; } -static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SSubmitBlkIter blkIter = {0}; - STSRow *row = NULL; +static int32_t tdFetchSubmitReqSuids(SSubmitReq2 *pMsg, STbUidStore *pStore) { + SArray *pSubmitTbData = pMsg ? pMsg->aSubmitTbData : NULL; + int32_t size = taosArrayGetSize(pSubmitTbData); terrno = TSDB_CODE_SUCCESS; - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { - return -1; - } - while (true) { - if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) { + for (int32_t i = 0; i < size; ++i) { + SSubmitTbData *pData = TARRAY_GET_ELEM(pSubmitTbData, i); + if ((terrno = tdUidStorePut(pStore, pData->suid, NULL)) < 0) { return -1; } - - if (!pBlock) break; - tdUidStorePut(pStore, msgIter.suid, NULL); } - if (terrno != TSDB_CODE_SUCCESS) { - return -1; - } return 0; } @@ -675,11 +670,11 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%d", output->info.id.uid, output->info.id.groupId, output->info.rows); - STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); - SSubmitReq *pReq = NULL; + STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); + SSubmitReq2 *pReq = NULL; // TODO: the schema update should be handled later(TD-17965) - if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, SMA_VID(pSma), suid) < 0) { + if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, output->info.id.groupId, SMA_VID(pSma), suid) < 0) { smaError("vgId:%d, build submit req for rsma table suid:%" PRIu64 ", uid:%" PRIu64 ", level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, terrstr()); @@ -687,19 +682,21 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma } if (pReq && tdProcessSubmitReq(sinkTsdb, output->info.version, pReq) < 0) { - taosMemoryFreeClear(pReq); + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); smaError("vgId:%d, process submit req for rsma suid:%" PRIu64 ", uid:%" PRIu64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, terrstr()); goto _err; } - smaDebug("vgId:%d, process submit req for rsma suid:%" PRIu64 ",uid:%" PRIu64 ", level %" PRIi8 " ver %" PRIi64 - " len %" PRIu32, - SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, output->info.version, - htonl(pReq->header.contLen)); + smaDebug("vgId:%d, process submit req for rsma suid:%" PRIu64 ",uid:%" PRIu64 ", level %" PRIi8 " ver %" PRIi64, + SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, output->info.version); - taosMemoryFreeClear(pReq); + if (pReq) { + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); + } } } @@ -717,22 +714,29 @@ _err: * @brief Copy msg to rsmaQueueBuffer for batch process * * @param pSma + * @param version * @param pMsg + * @param len * @param inputType * @param pInfo * @param suid * @return int32_t */ -static int32_t tdExecuteRSmaImplAsync(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfo *pInfo, - tb_uid_t suid) { - const SSubmitReq *pReq = (const SSubmitReq *)pMsg; +static int32_t tdExecuteRSmaImplAsync(SSma *pSma, int64_t version, const void *pMsg, int32_t len, int32_t inputType, + SRSmaInfo *pInfo, tb_uid_t suid) { + int32_t size = sizeof(int32_t) + sizeof(int64_t) + len; + void *qItem = taosAllocateQitem(size, DEF_QITEM, 0); - void *qItem = taosAllocateQitem(pReq->header.contLen, DEF_QITEM, 0); if (!qItem) { return TSDB_CODE_FAILED; } - memcpy(qItem, pMsg, pReq->header.contLen); + void *pItem = qItem; + + *(int32_t *)pItem = len; + pItem = POINTER_SHIFT(pItem, sizeof(int32_t)); + *(int64_t *)pItem = version; + memcpy(POINTER_SHIFT(pItem, sizeof(int64_t)), pMsg, len); taosWriteQitem(pInfo->queue, qItem); @@ -832,50 +836,51 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t srcTaskInfo, SRSmaParam *param, tb_uid_t suid, int8_t idx) { + int32_t code = 0; + int32_t lino = 0; SVnode *pVnode = pSma->pVnode; char *pOutput = NULL; int32_t len = 0; if (!srcTaskInfo) { - terrno = TSDB_CODE_INVALID_PTR; + code = TSDB_CODE_INVALID_PTR; smaWarn("vgId:%d, rsma clone, table %" PRIi64 ", no need since srcTaskInfo is NULL", TD_VID(pVnode), suid); - return TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _exit); } - if ((terrno = qSerializeTaskStatus(srcTaskInfo, &pOutput, &len)) < 0) { - smaError("vgId:%d, rsma clone, table %" PRIi64 " serialize qTaskInfo failed since %s", TD_VID(pVnode), suid, - terrstr()); - goto _err; - } + code = qSerializeTaskStatus(srcTaskInfo, &pOutput, &len); + TSDB_CHECK_CODE(code, lino, _exit); SReadHandle handle = { .meta = pVnode->pMeta, .vnode = pVnode, .initTqReader = 1, }; - ASSERT(!dstTaskInfo); + + if (ASSERTS(!dstTaskInfo, "dstTaskInfo:%p is not NULL", dstTaskInfo)) { + code = TSDB_CODE_APP_ERROR; + TSDB_CHECK_CODE(code, lino, _exit); + } + dstTaskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle); if (!dstTaskInfo) { - terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE; - goto _err; + code = TSDB_CODE_RSMA_QTASKINFO_CREATE; + TSDB_CHECK_CODE(code, lino, _exit); } - if (qDeserializeTaskStatus(dstTaskInfo, pOutput, len) < 0) { - smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " failed since %s", TD_VID(pVnode), suid, - terrstr()); - goto _err; - } + code = qDeserializeTaskStatus(dstTaskInfo, pOutput, len); + TSDB_CHECK_CODE(code, lino, _exit); smaDebug("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " succeed", TD_VID(pVnode), suid); +_exit: taosMemoryFreeClear(pOutput); - return TSDB_CODE_SUCCESS; -_err: - taosMemoryFreeClear(pOutput); - tdRSmaQTaskInfoFree(dstTaskInfo, TD_VID(pVnode), idx + 1); - smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " failed since %s", TD_VID(pVnode), suid, - terrstr()); - return TSDB_CODE_FAILED; + if (code) { + tdRSmaQTaskInfoFree(dstTaskInfo, TD_VID(pVnode), idx + 1); + smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " failed since %s", TD_VID(pVnode), suid, + terrstr()); + } + return code; } /** @@ -886,43 +891,53 @@ _err: * @return int32_t */ static int32_t tdRSmaInfoClone(SSma *pSma, SRSmaInfo *pInfo) { + int32_t code = 0; + int32_t lino = 0; SRSmaParam *param = NULL; + SMetaReader mr = {0}; + if (!pInfo) { return TSDB_CODE_SUCCESS; } - SMetaReader mr = {0}; metaReaderInit(&mr, SMA_META(pSma), 0); smaDebug("vgId:%d, rsma clone qTaskInfo for suid:%" PRIi64, SMA_VID(pSma), pInfo->suid); if (metaGetTableEntryByUidCache(&mr, pInfo->suid) < 0) { - smaError("vgId:%d, rsma clone, failed to get table meta for %" PRIi64 " since %s", SMA_VID(pSma), pInfo->suid, - terrstr()); - goto _err; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (mr.me.type != TSDB_SUPER_TABLE) { + code = TSDB_CODE_RSMA_INVALID_SCHEMA; + TSDB_CHECK_CODE(code, lino, _exit); } - ASSERT(mr.me.type == TSDB_SUPER_TABLE); - ASSERT(mr.me.uid == pInfo->suid); + if (mr.me.uid != pInfo->suid) { + code = TSDB_CODE_RSMA_INVALID_SCHEMA; + TSDB_CHECK_CODE(code, lino, _exit); + } + if (TABLE_IS_ROLLUP(mr.me.flags)) { param = &mr.me.stbEntry.rsmaParam; for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (!pInfo->iTaskInfo[i]) { continue; } - if (tdCloneQTaskInfo(pSma, pInfo->taskInfo[i], pInfo->iTaskInfo[i], param, pInfo->suid, i) < 0) { - goto _err; - } + code = tdCloneQTaskInfo(pSma, pInfo->taskInfo[i], pInfo->iTaskInfo[i], param, pInfo->suid, i); + TSDB_CHECK_CODE(code, lino, _exit); } smaDebug("vgId:%d, rsma clone env success for %" PRIi64, SMA_VID(pSma), pInfo->suid); } else { - terrno = TSDB_CODE_RSMA_INVALID_SCHEMA; - goto _err; + code = TSDB_CODE_RSMA_INVALID_SCHEMA; + TSDB_CHECK_CODE(code, lino, _exit); } +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s, suid:%" PRIi64 ", flags:%" PRIi8 ",type:%" PRIi8 ", uid:%" PRIi64, + SMA_VID(pSma), __func__, lino, tstrerror(code), pInfo->suid, mr.me.flags, mr.me.type, mr.me.uid); + } metaReaderClear(&mr); - return TSDB_CODE_SUCCESS; -_err: - metaReaderClear(&mr); - smaError("vgId:%d, rsma clone env failed for %" PRIi64 " since %s", SMA_VID(pSma), pInfo->suid, terrstr()); - return TSDB_CODE_FAILED; + return code; } /** @@ -933,10 +948,14 @@ _err: * @return SRSmaInfo* */ static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) { + int32_t code = 0; + int32_t lino = 0; SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pStat = NULL; SRSmaInfo *pRSmaInfo = NULL; + terrno = 0; + if (!pEnv) { terrno = TSDB_CODE_RSMA_INVALID_ENV; return NULL; @@ -956,14 +975,17 @@ static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) { return NULL; } if (!pRSmaInfo->taskInfo[0]) { - if (tdRSmaInfoClone(pSma, pRSmaInfo) < 0) { + if ((terrno = tdRSmaInfoClone(pSma, pRSmaInfo)) < 0) { taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); return NULL; } } tdRefRSmaInfo(pSma, pRSmaInfo); taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); - ASSERT(pRSmaInfo->suid == suid); + if (ASSERTS(pRSmaInfo->suid == suid, "suid:%" PRIi64 " != %" PRIi64, pRSmaInfo->suid, suid)) { + terrno = TSDB_CODE_APP_ERROR; + return NULL; + } return pRSmaInfo; } taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); @@ -981,12 +1003,14 @@ static FORCE_INLINE void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { * @brief async mode * * @param pSma + * @param version * @param pMsg * @param inputType * @param suid * @return int32_t */ -static int32_t tdExecuteRSmaAsync(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) { +static int32_t tdExecuteRSmaAsync(SSma *pSma, int64_t version, const void *pMsg, int32_t len, int32_t inputType, + tb_uid_t suid) { SRSmaInfo *pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, suid); if (!pRSmaInfo) { smaDebug("vgId:%d, execute rsma, no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); @@ -994,7 +1018,7 @@ static int32_t tdExecuteRSmaAsync(SSma *pSma, const void *pMsg, int32_t inputTyp } if (inputType == STREAM_INPUT__DATA_SUBMIT) { - if (tdExecuteRSmaImplAsync(pSma, pMsg, inputType, pRSmaInfo, suid) < 0) { + if (tdExecuteRSmaImplAsync(pSma, version, pMsg, len, inputType, pRSmaInfo, suid) < 0) { tdReleaseRSmaInfo(pSma, pRSmaInfo); return TSDB_CODE_FAILED; } @@ -1009,14 +1033,18 @@ static int32_t tdExecuteRSmaAsync(SSma *pSma, const void *pMsg, int32_t inputTyp } } } else { - ASSERT(0); + terrno = TSDB_CODE_APP_ERROR; + tdReleaseRSmaInfo(pSma, pRSmaInfo); + smaError("vgId:%d, execute rsma, failed for suid:%" PRIu64 " since %s, type:%d", SMA_VID(pSma), suid, + tstrerror(terrno), inputType); + return TSDB_CODE_FAILED; } tdReleaseRSmaInfo(pSma, pRSmaInfo); return TSDB_CODE_SUCCESS; } -int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { +int32_t tdProcessRSmaSubmit(SSma *pSma, int64_t version, void *pReq, void *pMsg, int32_t len, int32_t inputType) { SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); if (!pEnv) { // only applicable when rsma env exists @@ -1026,19 +1054,22 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { STbUidStore uidStore = {0}; if (inputType == STREAM_INPUT__DATA_SUBMIT) { - if (tdFetchSubmitReqSuids(pMsg, &uidStore) < 0) { + if (tdFetchSubmitReqSuids(pReq, &uidStore) < 0) { + smaError("vgId:%d, failed to process rsma submit fetch suid since: %s", SMA_VID(pSma), terrstr()); goto _err; } if (uidStore.suid != 0) { - if (tdExecuteRSmaAsync(pSma, pMsg, inputType, uidStore.suid) < 0) { + if (tdExecuteRSmaAsync(pSma, version, pMsg, len, inputType, uidStore.suid) < 0) { + smaError("vgId:%d, failed to process rsma submit exec 1 since: %s", SMA_VID(pSma), terrstr()); goto _err; } void *pIter = NULL; while ((pIter = taosHashIterate(uidStore.uidHash, pIter))) { tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - if (tdExecuteRSmaAsync(pSma, pMsg, inputType, *pTbSuid) < 0) { + if (tdExecuteRSmaAsync(pSma, version, pMsg, len, inputType, *pTbSuid) < 0) { + smaError("vgId:%d, failed to process rsma submit exec 2 since: %s", SMA_VID(pSma), terrstr()); goto _err; } } @@ -1048,7 +1079,6 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { return TSDB_CODE_SUCCESS; _err: tdUidStoreDestory(&uidStore); - smaError("vgId:%d, failed to process rsma submit since: %s", SMA_VID(pSma), terrstr()); return TSDB_CODE_FAILED; } @@ -1060,19 +1090,22 @@ _err: * @return int32_t */ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { + int32_t code = 0; + int32_t lino = 0; SVnode *pVnode = pSma->pVnode; SArray *suidList = NULL; STbUidStore uidStore = {0}; SMetaReader mr = {0}; + tb_uid_t suid = 0; if (!(suidList = taosArrayInit(1, sizeof(tb_uid_t)))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } if (vnodeGetStbIdList(pSma->pVnode, 0, suidList) < 0) { - smaError("vgId:%d, failed to restore rsma env since get stb id list error: %s", TD_VID(pVnode), terrstr()); - goto _err; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } int64_t arrSize = taosArrayGetSize(suidList); @@ -1089,19 +1122,26 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { int64_t nRsmaTables = 0; metaReaderInit(&mr, SMA_META(pSma), 0); if (!(uidStore.tbUids = taosArrayInit(1024, sizeof(tb_uid_t)))) { - goto _err; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } + for (int64_t i = 0; i < arrSize; ++i) { - tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); + suid = *(tb_uid_t *)taosArrayGet(suidList, i); smaDebug("vgId:%d, rsma restore, suid is %" PRIi64, TD_VID(pVnode), suid); if (metaGetTableEntryByUidCache(&mr, suid) < 0) { - smaError("vgId:%d, rsma restore, failed to get table meta for %" PRIi64 " since %s", TD_VID(pVnode), suid, - terrstr()); - goto _err; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } tDecoderClear(&mr.coder); - ASSERT(mr.me.type == TSDB_SUPER_TABLE); - ASSERT(mr.me.uid == suid); + if (mr.me.type != TSDB_SUPER_TABLE) { + code = TSDB_CODE_RSMA_INVALID_SCHEMA; + TSDB_CHECK_CODE(code, lino, _exit); + } + if (mr.me.uid != suid) { + code = TSDB_CODE_RSMA_INVALID_SCHEMA; + TSDB_CHECK_CODE(code, lino, _exit); + } if (TABLE_IS_ROLLUP(mr.me.flags)) { ++nRsmaTables; SRSmaParam *param = &mr.me.stbEntry.rsmaParam; @@ -1111,22 +1151,20 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { TD_VID(pVnode), suid, i, param->maxdelay[i], param->watermark[i], param->qmsgLen[i]); } if (tdRSmaProcessCreateImpl(pSma, &mr.me.stbEntry.rsmaParam, suid, mr.me.name) < 0) { - smaError("vgId:%d, rsma restore env failed for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr()); - goto _err; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } // reload all ctbUids for suid uidStore.suid = suid; if (vnodeGetCtbIdList(pVnode, suid, uidStore.tbUids) < 0) { - smaError("vgId:%d, rsma restore, get ctb idlist failed for %" PRIi64 " since %s", TD_VID(pVnode), suid, - terrstr()); - goto _err; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } if (tdUpdateTbUidList(pVnode->pSma, &uidStore, true) < 0) { - smaError("vgId:%d, rsma restore, update tb uid list failed for %" PRIi64 " since %s", TD_VID(pVnode), suid, - terrstr()); - goto _err; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } taosArrayClear(uidStore.tbUids); @@ -1135,21 +1173,18 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { } } - metaReaderClear(&mr); - taosArrayDestroy(suidList); - tdUidStoreDestory(&uidStore); - if (nTables) { *nTables = nRsmaTables; } - - return TSDB_CODE_SUCCESS; -_err: +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s, suid:%" PRIi64 ", type:%" PRIi8 ", uid:%" PRIi64, TD_VID(pVnode), + __func__, lino, tstrerror(code), suid, mr.me.type, mr.me.uid); + } metaReaderClear(&mr); taosArrayDestroy(suidList); tdUidStoreDestory(&uidStore); - - return TSDB_CODE_FAILED; + return code; } /** @@ -1251,7 +1286,6 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code, lino, _exit); } - ASSERT(size > 0); int64_t offset = 0; if (taosFSendFile(pOutFD, pInFD, &offset, size) < 0) { @@ -1371,10 +1405,11 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { // async process pItem->fetchLevel = pItem->level; #if 0 + // debugging codes SRSmaInfo *qInfo = tdAcquireRSmaInfoBySuid(pSma, pRSmaInfo->suid); SRSmaInfoItem *qItem = RSMA_INFO_ITEM(qInfo, pItem->level - 1); - ASSERT(qItem->level == pItem->level); - ASSERT(qItem->fetchLevel == pItem->fetchLevel); + make sure(qItem->level == pItem->level); + make sure(qItem->fetchLevel == pItem->fetchLevel); #endif if (atomic_load_8(&pRSmaInfo->assigned) == 0) { tsem_post(&(pStat->notEmpty)); @@ -1403,7 +1438,8 @@ _end: static void tdFreeRSmaSubmitItems(SArray *pItems) { for (int32_t i = 0; i < taosArrayGetSize(pItems); ++i) { - taosFreeQitem(*(void **)taosArrayGet(pItems, i)); + SPackedData *packData = taosArrayGet(pItems, i); + taosFreeQitem(POINTER_SHIFT(packData->msgStr, -sizeof(int32_t) - sizeof(int64_t))); } taosArrayClear(pItems); } @@ -1472,8 +1508,12 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA void *msg = NULL; taosGetQitem(qall, (void **)&msg); if (msg) { - if (!taosArrayPush(pSubmitArr, &msg)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + SPackedData packData = {.msgLen = *(int32_t *)msg, + .ver = *(int64_t *)POINTER_SHIFT(msg, sizeof(int32_t)), + .msgStr = POINTER_SHIFT(msg, sizeof(int32_t) + sizeof(int64_t))}; + + if (!taosArrayPush(pSubmitArr, &packData)) { + tdFreeRSmaSubmitItems(pSubmitArr); goto _err; } } else { @@ -1516,6 +1556,8 @@ _err: */ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { + int32_t code = 0; + int32_t lino = 0; SVnode *pVnode = pSma->pVnode; SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); @@ -1524,14 +1566,14 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { bool isFetchAll = false; if (!pRSmaStat || !(infoHash = RSMA_INFO_HASH(pRSmaStat))) { - terrno = TSDB_CODE_RSMA_INVALID_STAT; - goto _err; + code = TSDB_CODE_RSMA_INVALID_STAT; + TSDB_CHECK_CODE(code, lino, _exit); } if (!(pSubmitArr = - taosArrayInit(TMIN(RSMA_SUBMIT_BATCH_SIZE, atomic_load_64(&pRSmaStat->nBufItems)), POINTER_BYTES))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + taosArrayInit(TMIN(RSMA_SUBMIT_BATCH_SIZE, atomic_load_64(&pRSmaStat->nBufItems)), sizeof(SPackedData)))) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } while (true) { @@ -1562,7 +1604,11 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { if (oldStat == 0 || ((oldStat == 2) && atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)) < TASK_TRIGGER_STAT_PAUSED)) { int32_t oldVal = atomic_fetch_add_32(&pRSmaStat->nFetchAll, 1); - ASSERT(oldVal >= 0); + + if (ASSERTS(oldVal >= 0, "oldVal of nFetchAll: %d < 0", oldVal)) { + code = TSDB_CODE_APP_ERROR; + TSDB_CHECK_CODE(code, lino, _exit); + } int8_t curStat = atomic_load_8(RSMA_COMMIT_STAT(pRSmaStat)); if (curStat == 1) { @@ -1592,7 +1638,9 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { } } } else { - ASSERT(0); + ASSERTS(0, "unknown rsma exec type:%d", (int32_t)type); + code = TSDB_CODE_APP_ERROR; + TSDB_CHECK_CODE(code, lino, _exit); } if (atomic_load_64(&pRSmaStat->nBufItems) <= 0) { @@ -1611,10 +1659,10 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { } // end of while(true) -_end: - taosArrayDestroy(pSubmitArr); - return TSDB_CODE_SUCCESS; -_err: +_exit: taosArrayDestroy(pSubmitArr); - return TSDB_CODE_FAILED; + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } + return code; } diff --git a/source/dnode/vnode/src/sma/smaSnapshot.c b/source/dnode/vnode/src/sma/smaSnapshot.c index c68525a493b83a8baadeb86c8133c00f2dad09a0..de3d93395aa3790dcd231b461470ee35462abb46 100644 --- a/source/dnode/vnode/src/sma/smaSnapshot.c +++ b/source/dnode/vnode/src/sma/smaSnapshot.c @@ -237,8 +237,8 @@ int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData) { _exit: if (code) { - rsmaSnapReaderClose(&pReader); smaError("vgId:%d, vnode snapshot rsma read failed since %s", SMA_VID(pReader->pSma), tstrerror(code)); + rsmaSnapReaderClose(&pReader); } else { smaInfo("vgId:%d, vnode snapshot rsma read succeed", SMA_VID(pReader->pSma)); } @@ -432,7 +432,7 @@ _exit: if (pInFD) taosCloseFile(&pInFD); smaError("vgId:%d, vnode snapshot rsma writer close failed since %s", SMA_VID(pSma), tstrerror(code)); } else { - smaInfo("vgId:%d, vnode snapshot rsma writer close succeed", SMA_VID(pSma)); + smaInfo("vgId:%d, vnode snapshot rsma writer close succeed", pSma ? SMA_VID(pSma) : 0); } return code; diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 9ce4a2077171b25ec3d98d970db99fee09aa1240..65c3bf3095e223f78166ebbd7c8b0d161d51e43d 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -14,6 +14,7 @@ */ #include "sma.h" +#include "tq.h" #include "tsdb.h" #define SMA_STORAGE_MINUTES_MAX 86400 @@ -155,6 +156,200 @@ _exit: return code; } +int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *pTSchema, + SSchemaWrapper *pTagSchemaWrapper, bool createTb, int64_t suid, const char *stbFullName, + SBatchDeleteReq *pDeleteReq, void **ppData, int32_t *pLen) { + void *pBuf = NULL; + int32_t len = 0; + SSubmitReq2 *pReq = NULL; + SArray *tagArray = NULL; + SArray *createTbArray = NULL; + SArray *pVals = NULL; + + int32_t sz = taosArrayGetSize(pBlocks); + + if (!(tagArray = taosArrayInit(1, sizeof(STagVal)))) { + goto _end; + } + + if (!(createTbArray = taosArrayInit(sz, POINTER_BYTES))) { + goto _end; + } + + if (!(pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { + goto _end; + } + + // create table req + if (createTb) { + for (int32_t i = 0; i < sz; ++i) { + SSDataBlock *pDataBlock = taosArrayGet(pBlocks, i); + SVCreateTbReq *pCreateTbReq = NULL; + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + taosArrayPush(createTbArray, &pCreateTbReq); + continue; + } + + if (!(pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq)))) { + goto _end; + }; + + // don't move to the end of loop as to destroy in the end of func when error occur + taosArrayPush(createTbArray, &pCreateTbReq); + + // set const + pCreateTbReq->flags = 0; + pCreateTbReq->type = TSDB_CHILD_TABLE; + pCreateTbReq->ctb.suid = suid; + + // set super table name + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + pCreateTbReq->ctb.stbName = strdup((char *)tNameGetTableName(&name)); // strdup(stbFullName); + + // set tag content + taosArrayClear(tagArray); + STagVal tagVal = { + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.id.groupId, + }; + taosArrayPush(tagArray, &tagVal); + pCreateTbReq->ctb.tagNum = taosArrayGetSize(tagArray); + + STag *pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + if (pTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + pCreateTbReq->ctb.pTag = (uint8_t *)pTag; + + // set tag name + SArray *tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = {0}; + strcpy(tagNameStr, "group_id"); + taosArrayPush(tagName, tagNameStr); + pCreateTbReq->ctb.tagName = tagName; + + // set table name + if (pDataBlock->info.parTbName[0]) { + pCreateTbReq->name = strdup(pDataBlock->info.parTbName); + } else { + pCreateTbReq->name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId); + } + } + } + + // SSubmitTbData req + for (int32_t i = 0; i < sz; ++i) { + SSDataBlock *pDataBlock = taosArrayGet(pBlocks, i); + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + pDeleteReq->suid = suid; + pDeleteReq->deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); + tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, pDeleteReq); + continue; + } + + int32_t rows = pDataBlock->info.rows; + + SSubmitTbData *pTbData = (SSubmitTbData *)taosMemoryCalloc(1, sizeof(SSubmitTbData)); + if (!pTbData) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + if (!(pTbData->aRowP = taosArrayInit(rows, sizeof(SRow *)))) { + taosMemoryFree(pTbData); + goto _end; + } + pTbData->suid = suid; + pTbData->uid = 0; // uid is assigned by vnode + pTbData->sver = pTSchema->version; + + if (createTb) { + pTbData->pCreateTbReq = taosArrayGetP(createTbArray, i); + if (pTbData->pCreateTbReq) pTbData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; + } + + if (!pVals && !(pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)))) { + taosArrayDestroy(pTbData->aRowP); + taosMemoryFree(pTbData); + goto _end; + } + + for (int32_t j = 0; j < rows; ++j) { + taosArrayClear(pVals); + for (int32_t k = 0; k < pTSchema->numOfCols; k++) { + const STColumn *pCol = &pTSchema->columns[k]; + SColumnInfoData *pColData = taosArrayGet(pDataBlock->pDataBlock, k); + if (colDataIsNull_s(pColData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + void *data = colDataGetData(pColData, j); + if (IS_STR_DATA_TYPE(pCol->type)) { + SValue sv = (SValue){.nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } else { + SValue sv; + memcpy(&sv.val, data, tDataTypes[pCol->type].bytes); + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } + } + } + SRow *pRow = NULL; + if ((terrno = tRowBuild(pVals, (STSchema *)pTSchema, &pRow)) < 0) { + tDestroySSubmitTbData(pTbData, TSDB_MSG_FLG_ENCODE); + goto _end; + } + ASSERT(pRow); + taosArrayPush(pTbData->aRowP, &pRow); + } + + taosArrayPush(pReq->aSubmitTbData, pTbData); + } + + // encode + tEncodeSize(tEncodeSSubmitReq2, pReq, len, terrno); + if (TSDB_CODE_SUCCESS == terrno) { + SEncoder encoder; + len += sizeof(SMsgHead); + pBuf = rpcMallocCont(len); + if (NULL == pBuf) { + goto _end; + } + ((SMsgHead *)pBuf)->vgId = TD_VID(pVnode); + ((SMsgHead *)pBuf)->contLen = htonl(len); + tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SMsgHead)), len - sizeof(SMsgHead)); + if (tEncodeSSubmitReq2(&encoder, pReq) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + /*vError("failed to encode submit req since %s", terrstr());*/ + } + tEncoderClear(&encoder); + } +_end: + taosArrayDestroy(tagArray); + taosArrayDestroy(pVals); + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + + if (terrno != 0) { + rpcFreeCont(pBuf); + taosArrayDestroy(pDeleteReq->deleteReqs); + return TSDB_CODE_FAILED; + } + if (ppData) *ppData = pBuf; + if (pLen) *pLen = len; + return TSDB_CODE_SUCCESS; +} + /** * @brief Insert/Update Time-range-wise SMA data. * @@ -217,19 +412,19 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char } SBatchDeleteReq deleteReq = {0}; - SSubmitReq *pSubmitReq = - tqBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true, - pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq); - // TODO deleteReq - taosArrayDestroy(deleteReq.deleteReqs); - + void *pSubmitReq = NULL; + int32_t contLen = 0; - if (!pSubmitReq) { - smaError("vgId:%d, failed to gen submit blk while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), + if (smaBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true, + pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq, &pSubmitReq, + &contLen) < 0) { + smaError("vgId:%d, failed to gen submit msg while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid, tstrerror(terrno)); goto _err; } - + + // TODO deleteReq + taosArrayDestroy(deleteReq.deleteReqs); #if 0 if (!strncasecmp("td.tsma.rst.tb", pTsmaStat->pTSma->dstTbName, 14)) { terrno = TSDB_CODE_APP_ERROR; @@ -242,7 +437,7 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char SRpcMsg submitReqMsg = { .msgType = TDMT_VND_SUBMIT, .pCont = pSubmitReq, - .contLen = ntohl(pSubmitReq->length), + .contLen = contLen, }; if (tmsgPutToQueue(&pSma->pVnode->msgCb, WRITE_QUEUE, &submitReqMsg) < 0) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ee00e2961624cc62a7d19ce3f509e02e263c3080..a27b6988c58d9173ca4250440a7abbac98f71114 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -677,9 +677,12 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { req.epoch, TD_VID(pTq->pVnode), fetchVer, pHead->msgType); if (pHead->msgType == TDMT_VND_SUBMIT) { - SSubmitReq* pCont = (SSubmitReq*)&pHead->body; - - if (tqTaosxScanLog(pTq, pHandle, pCont, &taosxRsp) < 0) { + SPackedData submit = { + .msgStr = POINTER_SHIFT(pHead->body, sizeof(SMsgHead)), + .msgLen = pHead->bodyLen - sizeof(SMsgHead), + .ver = pHead->version, + }; + if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp) < 0) { } if (taosxRsp.blockNum > 0 /* threshold */) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); @@ -722,6 +725,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) { SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg; + tqDebug("vgId:%d, tq process delete sub req %s", pTq->pVnode->config.vgId, pReq->subKey); + taosWLockLatch(&pTq->pushLock); int32_t code = taosHashRemove(pTq->pPushMgr, pReq->subKey, strlen(pReq->subKey)); if (code != 0) { @@ -731,6 +736,7 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t version, char* msg, int32_t msgL STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey)); if (pHandle) { + // walCloseRef(pHandle->pWalReader->pWal, pHandle->pRef->refId); if (pHandle->pRef) { walCloseRef(pTq->pVnode->pWal, pHandle->pRef->refId); } @@ -787,6 +793,9 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t version, char* msg, int32_t msgL SMqRebVgReq req = {0}; tDecodeSMqRebVgReq(msg, &req); // todo lock + + tqDebug("vgId:%d, tq process sub req %s", pTq->pVnode->config.vgId, req.subKey); + STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); if (pHandle == NULL) { if (req.oldConsumerId != -1) { @@ -877,6 +886,9 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t version, char* msg, int32_t msgL atomic_store_64(&pHandle->consumerId, req.newConsumerId); atomic_add_fetch_32(&pHandle->epoch, 1); taosMemoryFree(req.qmsg); + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + qStreamCloseTsdbReader(pHandle->execHandle.task); + } if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) { } // close handle @@ -954,14 +966,14 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { pTask->smaSink.smaSink = smaHandleRes; } else if (pTask->outputType == TASK_OUTPUT__TABLE) { pTask->tbSink.vnode = pTq->pVnode; - pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline; + pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline2; /*A(pTask->tbSink.pSchemaWrapper);*/ /*A(pTask->tbSink.pSchemaWrapper->pSchema);*/ pTask->tbSink.pTSchema = - tdGetSTSChemaFromSSChema(pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols, 1); - /*A(pTask->tbSink.pTSchema);*/ + tBuildTSchema(pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols, 1); + ASSERT(pTask->tbSink.pTSchema); } streamSetupTrigger(pTask); @@ -1010,6 +1022,7 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { tqError("unable to encode rsp %d", __LINE__); return -1; } + void* buf = rpcMallocCont(sizeof(SMsgHead) + len); ((SMsgHead*)buf)->vgId = htonl(req.upstreamNodeId); @@ -1112,6 +1125,11 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { // do recovery step 1 streamSourceRecoverScanStep1(pTask); + if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + streamMetaReleaseTask(pTq->pStreamMeta, pTask); + return 0; + } + // build msg to launch next step SStreamRecoverStep2Req req; code = streamBuildSourceRecover2Req(pTask, &req); @@ -1122,6 +1140,10 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { streamMetaReleaseTask(pTq->pStreamMeta, pTask); + if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + return 0; + } + // serialize msg int32_t len = sizeof(SStreamRecoverStep1Req); @@ -1160,6 +1182,11 @@ int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t version, char* msg, int32_t m return -1; } + if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + streamMetaReleaseTask(pTq->pStreamMeta, pTask); + return 0; + } + // restore param code = streamRestoreParam(pTask); if (code < 0) { @@ -1334,12 +1361,12 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { return 0; } -int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* pReq, int64_t ver) { - void* pIter = NULL; - bool failed = false; - SStreamDataSubmit* pSubmit = NULL; +int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { + void* pIter = NULL; + bool failed = false; + SStreamDataSubmit2* pSubmit = NULL; - pSubmit = streamDataSubmitNew(pReq); + pSubmit = streamDataSubmitNew(submit); if (pSubmit == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to create data submit for stream since out of memory"); @@ -1356,7 +1383,7 @@ int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* pReq, int64_t ver) { continue; } - tqDebug("data submit enqueue stream task: %d, ver: %" PRId64, pTask->taskId, ver); + tqDebug("data submit enqueue stream task: %d, ver: %" PRId64, pTask->taskId, submit.ver); if (!failed) { if (streamTaskInput(pTask, (SStreamQueueItem*)pSubmit) < 0) { diff --git a/source/dnode/vnode/src/tq/tqCommit.c b/source/dnode/vnode/src/tq/tqCommit.c index dabd97a345f375c6774d37e4f4a408bd0bd44940..7fc66c49192eb011ce674bc7a2d396825ba1b435 100644 --- a/source/dnode/vnode/src/tq/tqCommit.c +++ b/source/dnode/vnode/src/tq/tqCommit.c @@ -15,4 +15,11 @@ #include "tq.h" -int tqCommit(STQ* pTq) { return tqOffsetCommitFile(pTq->pOffsetStore); } +int tqCommit(STQ* pTq) { + if (streamMetaCommit(pTq->pStreamMeta) < 0) { + tqError("vgId:%d, failed to commit stream meta since %s", TD_VID(pTq->pVnode), terrstr()); + return -1; + } + + return tqOffsetCommitFile(pTq->pOffsetStore); +} diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 9a56df7cb0f535852c35fab8b6a5d56b5c7a865c..40a82cc8e82edbab0a776f0f5f69d6013b5e2de3 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -113,14 +113,8 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs return -1; } - if (pRsp->withTbName) { - if (pRsp->rspOffset.type == TMQ_OFFSET__LOG) { - int64_t uid = pExec->pExecReader->msgIter.uid; - tqAddTbNameToRsp(pTq, uid, pRsp, 1); - } else { - pRsp->withTbName = false; - } - } + ASSERT(pRsp->withTbName == false); + ASSERT(pRsp->withSchema == false); return 0; } @@ -157,9 +151,8 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta if (pDataBlock != NULL) { if (pRsp->withTbName) { - int64_t uid = 0; if (pOffset->type == TMQ_OFFSET__LOG) { - uid = pExec->pExecReader->msgIter.uid; + int64_t uid = pExec->pExecReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp, 1) < 0) { continue; } @@ -229,7 +222,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta return 0; } -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp) { +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp) { STqExecHandle* pExec = &pHandle->execHandle; /*A(pExec->subType != TOPIC_SUB_TYPE__COLUMN);*/ @@ -238,8 +231,9 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { STqReader* pReader = pExec->pExecReader; - tqReaderSetDataMsg(pReader, pReq, 0); - while (tqNextDataBlock(pReader)) { + /*tqReaderSetDataMsg(pReader, pReq, 0);*/ + tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver); + while (tqNextDataBlock2(pReader)) { /*SSDataBlock block = {0};*/ /*if (tqRetrieveDataBlock(&block, pReader) < 0) {*/ /*if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;*/ @@ -247,11 +241,12 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp taosArrayClear(pBlocks); taosArrayClear(pSchemas); - if (tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas) < 0) { + if (tqRetrieveTaosxBlock2(pReader, pBlocks, pSchemas) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; } if (pRsp->withTbName) { - int64_t uid = pExec->pExecReader->msgIter.uid; + /*int64_t uid = pExec->pExecReader->msgIter.uid;*/ + int64_t uid = pExec->pExecReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlocks)) < 0) { taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper); @@ -261,7 +256,9 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp } } if (pHandle->fetchMeta) { +#if 0 SSubmitBlk* pBlk = pReader->pBlock; + int64_t uid = pExec->pExecReader->lastBlkUid; int32_t schemaLen = htonl(pBlk->schemaLen); if (schemaLen > 0) { if (pRsp->createTableNum == 0) { @@ -274,6 +271,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp taosArrayPush(pRsp->createTableReq, &createReq); pRsp->createTableNum++; } +#endif } for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) { SSDataBlock* pBlock = taosArrayGet(pBlocks, i); @@ -287,19 +285,20 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp } } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { STqReader* pReader = pExec->pExecReader; - tqReaderSetDataMsg(pReader, pReq, 0); - while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) { + /*tqReaderSetDataMsg(pReader, pReq, 0);*/ + tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver); + while (tqNextDataBlockFilterOut2(pReader, pExec->execDb.pFilterOutTbUid)) { /*SSDataBlock block = {0};*/ /*if (tqRetrieveDataBlock(&block, pReader) < 0) {*/ /*if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;*/ /*}*/ taosArrayClear(pBlocks); taosArrayClear(pSchemas); - if (tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas) < 0) { + if (tqRetrieveTaosxBlock2(pReader, pBlocks, pSchemas) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; } if (pRsp->withTbName) { - int64_t uid = pExec->pExecReader->msgIter.uid; + int64_t uid = pExec->pExecReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlocks)) < 0) { taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper); @@ -309,6 +308,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp } } if (pHandle->fetchMeta) { +#if 0 SSubmitBlk* pBlk = pReader->pBlock; int32_t schemaLen = htonl(pBlk->schemaLen); if (schemaLen > 0) { @@ -322,6 +322,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp taosArrayPush(pRsp->createTableReq, &createReq); pRsp->createTableNum++; } +#endif } /*tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock),*/ /*pTq->pVnode->config.tsdbCfg.precision);*/ diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 5a4d414ab7f9591a095148635634fb350f8f078d..66d1ac2c7e530e4b8c732deaaa83afc6191b5e44 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -45,19 +45,37 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { } int32_t size = htonl(head.size); void* memBuf = taosMemoryCalloc(1, size); + if (memBuf == NULL) { + return -1; + } if ((code = taosReadFile(pFile, memBuf, size)) != size) { + taosMemoryFree(memBuf); return -1; } STqOffset offset; SDecoder decoder; tDecoderInit(&decoder, memBuf, size); if (tDecodeSTqOffset(&decoder, &offset) < 0) { + taosMemoryFree(memBuf); + tDecoderClear(&decoder); return -1; } + tDecoderClear(&decoder); if (taosHashPut(pStore->pHash, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset)) < 0) { return -1; } + + if (offset.val.type == TMQ_OFFSET__LOG) { + STqHandle* pHandle = taosHashGet(pStore->pTq->pHandle, offset.subKey, strlen(offset.subKey)); + if (pHandle) { + if (walRefVer(pHandle->pRef, offset.val.version) < 0) { + tqError("vgId: %d, tq handle %s ref ver %" PRId64 "error", pStore->pTq->pVnode->config.vgId, + pHandle->subKey, offset.val.version); + } + } + } + taosMemoryFree(memBuf); } @@ -123,6 +141,7 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) { const char* sysErrStr = strerror(errno); tqError("vgId:%d, cannot open file %s when commit offset since %s", pStore->pTq->pVnode->config.vgId, fname, sysErrStr); + taosMemoryFree(fname); return -1; } taosMemoryFree(fname); diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 63c3c25218c0532e58c1b52e0ac2ace30eef3d11..559a3b76fec3a532d2e40fe62a203e515ce38252 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -207,7 +207,11 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ #endif int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { - tqDebug("vgId:%d, tq push msg ver %" PRId64 ", type: %s", pTq->pVnode->config.vgId, ver, TMSG_INFO(msgType)); + void* pReq = POINTER_SHIFT(msg, sizeof(SMsgHead)); + int32_t len = msgLen - sizeof(SMsgHead); + + tqDebug("vgId:%d, tq push msg ver %" PRId64 ", type: %s, p head %p, p body %p, len %d", pTq->pVnode->config.vgId, ver, + TMSG_INFO(msgType), msg, pReq, len); if (msgType == TDMT_VND_SUBMIT) { // lock push mgr to avoid potential msg lost @@ -216,7 +220,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) if (taosHashGetSize(pTq->pPushMgr) != 0) { SArray* cachedKeys = taosArrayInit(0, sizeof(void*)); SArray* cachedKeyLens = taosArrayInit(0, sizeof(size_t)); - void* data = taosMemoryMalloc(msgLen); + void* data = taosMemoryMalloc(len); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to copy data for stream since out of memory"); @@ -224,9 +228,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) taosArrayDestroy(cachedKeyLens); return -1; } - memcpy(data, msg, msgLen); - SSubmitReq* pReq = (SSubmitReq*)data; - pReq->version = ver; + memcpy(data, pReq, len); void* pIter = NULL; while (1) { @@ -250,7 +252,12 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) SMqDataRsp* pRsp = &pPushEntry->dataRsp; // prepare scan mem data - qStreamScanMemData(task, pReq); + SPackedData submit = { + .msgStr = data, + .msgLen = len, + .ver = ver, + }; + qStreamSetScanMemData(task, submit); // exec while (1) { @@ -304,17 +311,22 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) if (vnodeIsRoleLeader(pTq->pVnode)) { if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) return 0; if (msgType == TDMT_VND_SUBMIT) { - void* data = taosMemoryMalloc(msgLen); + void* data = taosMemoryMalloc(len); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to copy data for stream since out of memory"); return -1; } - memcpy(data, msg, msgLen); - SSubmitReq* pReq = (SSubmitReq*)data; - pReq->version = ver; + memcpy(data, pReq, len); + SPackedData submit = { + .msgStr = data, + .msgLen = len, + .ver = ver, + }; + + tqDebug("tq copy write msg %p %d %" PRId64 " from %p", data, len, ver, pReq); - tqProcessSubmitReq(pTq, data, ver); + tqProcessSubmitReq(pTq, submit); } if (msgType == TDMT_VND_DELETE) { tqProcessDelReq(pTq, POINTER_SHIFT(msg, sizeof(SMsgHead)), msgLen - sizeof(SMsgHead), ver); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index e5392bbd0263e724617fbba47ac28924bc9bc8e4..eb9c0c3eeb252b53caf2b04a0674e879242cca42 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -247,7 +247,7 @@ END: } STqReader* tqOpenReader(SVnode* pVnode) { - STqReader* pReader = taosMemoryMalloc(sizeof(STqReader)); + STqReader* pReader = taosMemoryCalloc(1, sizeof(STqReader)); if (pReader == NULL) { return NULL; } @@ -259,7 +259,7 @@ STqReader* tqOpenReader(SVnode* pVnode) { } pReader->pVnodeMeta = pVnode->pMeta; - pReader->pMsg = NULL; + /*pReader->pMsg = NULL;*/ pReader->ver = -1; pReader->pColIdList = NULL; pReader->cachedSchemaVer = 0; @@ -298,7 +298,7 @@ int32_t tqSeekVer(STqReader* pReader, int64_t ver) { } int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { - bool fromProcessedMsg = pReader->pMsg != NULL; + bool fromProcessedMsg = pReader->msg2.msgStr != NULL; while (1) { if (!fromProcessedMsg) { @@ -311,7 +311,9 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { tqDebug("return offset %" PRId64 ", no more valid", ret->offset.version); return -1; } - void* body = pReader->pWalReader->pHead->head.body; + void* body = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SMsgHead)); + int32_t bodyLen = pReader->pWalReader->pHead->head.bodyLen - sizeof(SMsgHead); + int64_t ver = pReader->pWalReader->pHead->head.version; #if 0 if (pReader->pWalReader->pHead->head.msgType != TDMT_VND_SUBMIT) { // TODO do filter @@ -320,16 +322,17 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { return 0; } else { #endif - tqReaderSetDataMsg(pReader, body, pReader->pWalReader->pHead->head.version); + tqReaderSetSubmitReq2(pReader, body, bodyLen, ver); + /*tqReaderSetDataMsg(pReader, body, pReader->pWalReader->pHead->head.version);*/ #if 0 } #endif } - while (tqNextDataBlock(pReader)) { + while (tqNextDataBlock2(pReader)) { // TODO mem free memset(&ret->data, 0, sizeof(SSDataBlock)); - int32_t code = tqRetrieveDataBlock(&ret->data, pReader); + int32_t code = tqRetrieveDataBlock2(&ret->data, pReader); if (code != 0 || ret->data.info.rows == 0) { continue; } @@ -348,6 +351,7 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { } } +#if 0 int32_t tqReaderSetDataMsg(STqReader* pReader, const SSubmitReq* pMsg, int64_t ver) { pReader->pMsg = pMsg; @@ -362,7 +366,33 @@ int32_t tqReaderSetDataMsg(STqReader* pReader, const SSubmitReq* pMsg, int64_t v memset(&pReader->blkIter, 0, sizeof(SSubmitBlkIter)); return 0; } +#endif + +int32_t tqReaderSetSubmitReq2(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) { + ASSERT(pReader->msg2.msgStr == NULL); + ASSERT(msgStr); + ASSERT(msgLen); + ASSERT(ver >= 0); + pReader->msg2.msgStr = msgStr; + pReader->msg2.msgLen = msgLen; + pReader->msg2.ver = ver; + pReader->ver = ver; + + tqDebug("tq reader set msg %p %d", msgStr, msgLen); + if (pReader->setMsg == 0) { + SDecoder decoder; + tDecoderInit(&decoder, pReader->msg2.msgStr, pReader->msg2.msgLen); + if (tDecodeSSubmitReq2(&decoder, &pReader->submit) < 0) { + ASSERT(0); + } + tDecoderClear(&decoder); + pReader->setMsg = 1; + } + return 0; +} + +#if 0 bool tqNextDataBlock(STqReader* pReader) { if (pReader->pMsg == NULL) return false; while (1) { @@ -386,6 +416,59 @@ bool tqNextDataBlock(STqReader* pReader) { } return false; } +#endif + +bool tqNextDataBlock2(STqReader* pReader) { + if (pReader->msg2.msgStr == NULL) return false; + ASSERT(pReader->setMsg == 1); + + tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg2.msgStr, pReader->msg2.msgLen, + pReader->msg2.ver, pReader->nextBlk); + + int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); + while (pReader->nextBlk < blockSz) { + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); + ASSERT(pSubmitTbData->uid); + + if (pReader->tbIdHash == NULL) return true; + + void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); + if (ret != NULL) { + return true; + } + pReader->nextBlk++; + } + + tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); + pReader->setMsg = 0; + pReader->nextBlk = 0; + pReader->msg2.msgStr = NULL; + + return false; +} + +bool tqNextDataBlockFilterOut2(STqReader* pReader, SHashObj* filterOutUids) { + if (pReader->msg2.msgStr == NULL) return false; + ASSERT(pReader->setMsg == 1); + + int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); + while (pReader->nextBlk < blockSz) { + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); + if (pReader->tbIdHash == NULL) return true; + + void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); + if (ret == NULL) { + return true; + } + } + + tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); + pReader->setMsg = 0; + pReader->nextBlk = 0; + pReader->msg2.msgStr = NULL; + + return false; +} int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrapper* pSrc, char* mask) { int32_t code; @@ -416,6 +499,7 @@ int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrap return 0; } +#if 0 bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) { while (1) { if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { @@ -431,6 +515,253 @@ bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) { return false; } +int32_t tqScanSubmitSplit(SArray* pBlocks, SArray* schemas, STqReader* pReader) { + // + int32_t sversion = htonl(pReader->pBlock->sversion); + if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || + pReader->cachedSchemaSuid != pReader->msgIter.suid) { + taosMemoryFree(pReader->pSchema); + pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); + if (pReader->pSchema == NULL) { + tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 + "), version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, pReader->msgIter.uid, pReader->msgIter.suid, sversion); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + tDeleteSSchemaWrapper(pReader->pSchemaWrapper); + pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); + if (pReader->pSchemaWrapper == NULL) { + tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, pReader->msgIter.uid, pReader->cachedSchemaVer); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + STSchema* pTschema = pReader->pSchema; + SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; + + int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList); + } + return 0; +} +#endif + +int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader) { + int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); + ASSERT(pReader->nextBlk < blockSz); + + tqDebug("tq reader retrieve data block %p, %d", pReader->msg2.msgStr, pReader->nextBlk); + + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); + pReader->nextBlk++; + + int32_t sversion = pSubmitTbData->sver; + int64_t suid = pSubmitTbData->suid; + int64_t uid = pSubmitTbData->uid; + pReader->lastBlkUid = uid; + + pBlock->info.id.uid = uid; + pBlock->info.version = pReader->msg2.ver; + + if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || pReader->cachedSchemaSuid != suid) { + taosMemoryFree(pReader->pSchema); + pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, uid, sversion, 1); + if (pReader->pSchema == NULL) { + tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 + "), version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, uid, suid, sversion); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + tDeleteSSchemaWrapper(pReader->pSchemaWrapper); + pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1); + if (pReader->pSchemaWrapper == NULL) { + tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, uid, pReader->cachedSchemaVer); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + STSchema* pTschema = pReader->pSchema; + SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; + + int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList); + + if (colNumNeed == 0) { + int32_t colMeta = 0; + while (colMeta < pSchemaWrapper->nCols) { + SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { + goto FAIL; + } + colMeta++; + } + } else { + if (colNumNeed > pSchemaWrapper->nCols) { + colNumNeed = pSchemaWrapper->nCols; + } + + int32_t colMeta = 0; + int32_t colNeed = 0; + while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { + SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; + col_id_t colIdSchema = pColSchema->colId; + col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pReader->pColIdList, colNeed); + if (colIdSchema < colIdNeed) { + colMeta++; + } else if (colIdSchema > colIdNeed) { + colNeed++; + } else { + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { + goto FAIL; + } + colMeta++; + colNeed++; + } + } + } + + int32_t numOfRows = 0; + + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + SArray* pCols = pSubmitTbData->aCol; + SColData* pCol = taosArrayGet(pCols, 0); + numOfRows = pCol->nVal; + } else { + SArray* pRows = pSubmitTbData->aRowP; + numOfRows = taosArrayGetSize(pRows); + } + + if (blockDataEnsureCapacity(pBlock, numOfRows) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto FAIL; + } + pBlock->info.rows = numOfRows; + + int32_t colActual = blockDataGetNumOfCols(pBlock); + + // convert and scan one block + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + SArray* pCols = pSubmitTbData->aCol; + int32_t numOfCols = taosArrayGetSize(pCols); + int32_t targetIdx = 0; + int32_t sourceIdx = 0; + while (targetIdx < colActual) { + ASSERT(sourceIdx < numOfCols); + + SColData* pCol = taosArrayGet(pCols, sourceIdx); + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx); + SColVal colVal; + + ASSERT(pCol->nVal == numOfRows); + + if (pCol->cid < pColData->info.colId) { + sourceIdx++; + } else if (pCol->cid == pColData->info.colId) { + for (int32_t i = 0; i < pCol->nVal; i++) { + tColDataGetValue(pCol, sourceIdx, &colVal); +#if 0 + void* val = NULL; + if (IS_STR_DATA_TYPE(colVal.type)) { + val = colVal.value.pData; + } else { + val = &colVal.value.val; + } + if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } +#endif + if (IS_STR_DATA_TYPE(colVal.type)) { + if (colVal.value.pData != NULL) { + char val[65535 + 2]; + memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); + varDataSetLen(val, colVal.value.nData); + if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } else { + colDataAppendNULL(pColData, i); + } + } else { + if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } + } + sourceIdx++; + targetIdx++; + } else { + ASSERT(0); + } + } + } else { + SArray* pRows = pSubmitTbData->aRowP; + + for (int32_t i = 0; i < numOfRows; i++) { + SRow* pRow = taosArrayGetP(pRows, i); + int32_t targetIdx = 0; + int32_t sourceIdx = 0; + + for (int32_t j = 0; j < colActual; j++) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, j); + while (1) { + ASSERT(sourceIdx < pTschema->numOfCols); + + SColVal colVal; + tRowGet(pRow, pTschema, sourceIdx, &colVal); + if (colVal.cid < pColData->info.colId) { + sourceIdx++; + continue; + } else if (colVal.cid == pColData->info.colId) { + if (IS_STR_DATA_TYPE(colVal.type)) { + if (colVal.value.pData != NULL) { + char val[65535 + 2]; + memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); + varDataSetLen(val, colVal.value.nData); + if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } else { + colDataAppendNULL(pColData, i); + } + /*val = colVal.value.pData;*/ + } else { + if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } + + sourceIdx++; + targetIdx++; + break; + } else { + ASSERT(0); + } + } + } + } + } + } + + return 0; + +FAIL: + blockDataFreeRes(pBlock); + return -1; +} + +#if 0 int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { // TODO: cache multiple schema int32_t sversion = htonl(pReader->pBlock->sversion); @@ -673,6 +1004,18 @@ FAIL: taosMemoryFree(assigned); return -1; } +#endif + +int32_t tqRetrieveTaosxBlock2(STqReader* pReader, SArray* blocks, SArray* schemas) { + SSDataBlock block = {0}; + if (tqRetrieveDataBlock2(&block, pReader) == 0) { + taosArrayPush(blocks, &block); + SSchemaWrapper* pSW = tCloneSSchemaWrapper(pReader->pSchemaWrapper); + taosArrayPush(schemas, &pSW); + return 0; + } + return -1; +} void tqReaderSetColIdList(STqReader* pReader, SArray* pColIdList) { pReader->pColIdList = pColIdList; } diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 078b50db680d05e1127f54d5bd94fe71459abd4c..f1103ad48a1be4101da041e40ddf1e7b4a6b181b 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -63,7 +63,7 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBl .startTs = startTs, .endTs = endTs, }; - strncpy(req.tbname, name, TSDB_TABLE_NAME_LEN); + strncpy(req.tbname, name, TSDB_TABLE_NAME_LEN - 1); taosMemoryFree(name); /*tqDebug("stream delete msg, active: vgId:%d, ts:%" PRId64 " name:%s", pVnode->config.vgId, ts, name);*/ taosArrayPush(deleteReq->deleteReqs, &req); @@ -71,233 +71,6 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBl return 0; } -SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pTSchema, - SSchemaWrapper* pTagSchemaWrapper, bool createTb, int64_t suid, const char* stbFullName, - SBatchDeleteReq* pDeleteReq) { - SSubmitReq* ret = NULL; - SArray* schemaReqs = NULL; - SArray* schemaReqSz = NULL; - SArray* tagArray = taosArrayInit(1, sizeof(STagVal)); - if (!tagArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - int32_t sz = taosArrayGetSize(pBlocks); - - if (createTb) { - schemaReqs = taosArrayInit(sz, sizeof(void*)); - schemaReqSz = taosArrayInit(sz, sizeof(int32_t)); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - if (pDataBlock->info.type == STREAM_DELETE_RESULT) { - int32_t padding1 = 0; - void* padding2 = NULL; - taosArrayPush(schemaReqSz, &padding1); - taosArrayPush(schemaReqs, &padding2); - continue; - } - - // STag* pTag = NULL; - // taosArrayClear(tagArray); - // SArray *tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); - // for(int j = 0; j < pTagSchemaWrapper->nCols; j++){ - // STagVal tagVal = { - // .cid = pTagSchemaWrapper->pSchema[j].colId, - // .type = pTagSchemaWrapper->pSchema[j].type, - // .i64 = (int64_t)pDataBlock->info.id.groupId, - // }; - // taosArrayPush(tagArray, &tagVal); - // taosArrayPush(tagName, pTagSchemaWrapper->pSchema[j].name); - // } - // - // tTagNew(tagArray, 1, false, &pTag); - // if (pTag == NULL) { - // terrno = TSDB_CODE_OUT_OF_MEMORY; - // taosArrayDestroy(tagArray); - // taosArrayDestroy(tagName); - // return NULL; - // } - - SVCreateTbReq createTbReq = {0}; - - // set const - createTbReq.flags = 0; - createTbReq.type = TSDB_CHILD_TABLE; - createTbReq.ctb.suid = suid; - - // set super table name - SName name = {0}; - tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - createTbReq.ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName); - - // set tag content - taosArrayClear(tagArray); - STagVal tagVal = { - .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, - .type = TSDB_DATA_TYPE_UBIGINT, - .i64 = (int64_t)pDataBlock->info.id.groupId, - }; - taosArrayPush(tagArray, &tagVal); - createTbReq.ctb.tagNum = taosArrayGetSize(tagArray); - - STag* pTag = NULL; - tTagNew(tagArray, 1, false, &pTag); - if (pTag == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosArrayDestroy(tagArray); - taosArrayDestroyP(schemaReqs, taosMemoryFree); - taosArrayDestroy(schemaReqSz); - return NULL; - } - createTbReq.ctb.pTag = (uint8_t*)pTag; - - // set tag name - SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); - char tagNameStr[TSDB_COL_NAME_LEN] = {0}; - strcpy(tagNameStr, "group_id"); - taosArrayPush(tagName, tagNameStr); - createTbReq.ctb.tagName = tagName; - - // set table name - if (pDataBlock->info.parTbName[0]) { - createTbReq.name = strdup(pDataBlock->info.parTbName); - } else { - createTbReq.name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId); - } - - // save schema len - int32_t code; - int32_t schemaLen; - tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); - if (code < 0) { - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); - taosArrayDestroyP(schemaReqs, taosMemoryFree); - taosArrayDestroy(schemaReqSz); - return NULL; - } - taosArrayPush(schemaReqSz, &schemaLen); - - // save schema str - void* schemaStr = taosMemoryMalloc(schemaLen); - if (schemaStr == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); - taosArrayDestroyP(schemaReqs, taosMemoryFree); - taosArrayDestroy(schemaReqSz); - return NULL; - } - taosArrayPush(schemaReqs, &schemaStr); - - SEncoder encoder = {0}; - tEncoderInit(&encoder, schemaStr, schemaLen); - code = tEncodeSVCreateTbReq(&encoder, &createTbReq); - if (code < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); - taosArrayDestroyP(schemaReqs, taosMemoryFree); - taosArrayDestroy(schemaReqSz); - tEncoderClear(&encoder); - return NULL; - } - tEncoderClear(&encoder); - tdDestroySVCreateTbReq(&createTbReq); - } - } - taosArrayDestroy(tagArray); - - // cal size - int32_t cap = sizeof(SSubmitReq); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - if (pDataBlock->info.type == STREAM_DELETE_RESULT) { - continue; - } - int32_t rows = pDataBlock->info.rows; - /*int32_t rowSize = pDataBlock->info.rowSize;*/ - int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); - - int32_t schemaLen = 0; - if (createTb) { - schemaLen = *(int32_t*)taosArrayGet(schemaReqSz, i); - } - cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen; - } - - // assign data - ret = rpcMallocCont(cap); - ret->header.vgId = pVnode->config.vgId; - ret->length = sizeof(SSubmitReq); - ret->numOfBlocks = htonl(sz); - - SSubmitBlk* blkHead = POINTER_SHIFT(ret, sizeof(SSubmitReq)); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - if (pDataBlock->info.type == STREAM_DELETE_RESULT) { - pDeleteReq->suid = suid; - pDeleteReq->deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); - tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, pDeleteReq); - continue; - } - - blkHead->numOfRows = htonl(pDataBlock->info.rows); - blkHead->sversion = htonl(pTSchema->version); - blkHead->suid = htobe64(suid); - // uid is assigned by vnode - blkHead->uid = 0; - - int32_t rows = pDataBlock->info.rows; - - tqDebug("tq sink, convert block1 %d, rows: %d", i, rows); - - int32_t dataLen = 0; - int32_t schemaLen = 0; - void* blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); - if (createTb) { - schemaLen = *(int32_t*)taosArrayGet(schemaReqSz, i); - void* schemaStr = taosArrayGetP(schemaReqs, i); - memcpy(blkSchema, schemaStr, schemaLen); - } - blkHead->schemaLen = htonl(schemaLen); - - STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - for (int32_t j = 0; j < rows; j++) { - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTSchema->version); - tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); - tdSRowResetBuf(&rb, rowData); - - for (int32_t k = 0; k < pTSchema->numOfCols; k++) { - const STColumn* pColumn = &pTSchema->columns[k]; - SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, k); - if (colDataIsNull_s(pColData, j)) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, pColumn->offset, k); - } else { - void* data = colDataGetData(pColData, j); - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); - } - } - tdSRowEnd(&rb); - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - dataLen += rowLen; - } - blkHead->dataLen = htonl(dataLen); - - ret->length += sizeof(SSubmitBlk) + schemaLen + dataLen; - blkHead = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk) + schemaLen + dataLen); - } - - ret->length = htonl(ret->length); - - taosArrayDestroyP(schemaReqs, taosMemoryFree); - taosArrayDestroy(schemaReqSz); - - return ret; -} void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { const SArray* pBlocks = (const SArray*)data; @@ -492,7 +265,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d blkHead->uid = 0; blkHead->schemaLen = 0; - tqDebug("tq sink, convert block2 %d, rows: %d", i, rows); + tqDebug("tq sink pipe1, convert block2 %d, rows: %d", i, rows); int32_t dataLen = 0; void* blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); @@ -521,7 +294,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d } else { void* colData = colDataGetData(pColData, j); if (k == 0) { - tqDebug("tq sink, row %d ts %" PRId64, j, *(int64_t*)colData); + tqDebug("tq sink pipe1, row %d ts %" PRId64, j, *(int64_t*)colData); } tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, pColumn->offset, k); } @@ -550,55 +323,409 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d taosArrayDestroy(tagArray); } -#if 0 -void tqSinkToTableMerge(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { - const SArray* pRes = (const SArray*)data; - SVnode* pVnode = (SVnode*)vnode; - SBatchDeleteReq deleteReq = {0}; - - tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, (int32_t)pRes->size); - - deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); - SSubmitReq* submitReq = tqBlockToSubmit(pVnode, pRes, pTask->tbSink.pTSchema, pTask->tbSink.pSchemaWrapper, true, - pTask->tbSink.stbUid, pTask->tbSink.stbFullName, &deleteReq); +static int32_t encodeCreateChildTableForRPC(SVCreateTbBatchReq* pReqs, int32_t vgId, void** pBuf, int32_t* contLen) { + int32_t ret = 0; - tqDebug("vgId:%d, task %d convert blocks over, put into write-queue", TD_VID(pVnode), pTask->taskId); - - if (taosArrayGetSize(deleteReq.deleteReqs) != 0) { - int32_t code; - int32_t len; - tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); - SEncoder encoder; - void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); - void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); - tEncoderInit(&encoder, abuf, len); - tEncodeSBatchDeleteReq(&encoder, &deleteReq); - tEncoderClear(&encoder); + tEncodeSize(tEncodeSVCreateTbBatchReq, pReqs, *contLen, ret); + if (ret < 0) { + ret = -1; + goto end; + } + *contLen += sizeof(SMsgHead); + *pBuf = rpcMallocCont(*contLen); + if (NULL == *pBuf) { + ret = -1; + goto end; + } + ((SMsgHead*)(*pBuf))->vgId = vgId; + ((SMsgHead*)(*pBuf))->contLen = htonl(*contLen); + SEncoder coder = {0}; + tEncoderInit(&coder, POINTER_SHIFT(*pBuf, sizeof(SMsgHead)), (*contLen) - sizeof(SMsgHead) ); + if (tEncodeSVCreateTbBatchReq(&coder, pReqs) < 0) { + rpcFreeCont(*pBuf); + *pBuf = NULL; + *contLen = 0; + tEncoderClear(&coder); + ret = -1; + goto end; + } + tEncoderClear(&coder); - ((SMsgHead*)serializedDeleteReq)->vgId = pVnode->config.vgId; +end: + return ret; +} - SRpcMsg msg = { - .msgType = TDMT_VND_BATCH_DEL, - .pCont = serializedDeleteReq, - .contLen = len + sizeof(SMsgHead), - }; - if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { - rpcFreeCont(serializedDeleteReq); - tqDebug("failed to put into write-queue since %s", terrstr()); - } - } - taosArrayDestroy(deleteReq.deleteReqs); +int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) { + void* buf = NULL; + int32_t tlen = 0; + encodeCreateChildTableForRPC(pReqs, TD_VID(pVnode), &buf, &tlen); - /*tPrintFixedSchemaSubmitReq(pReq, pTask->tbSink.pTSchema);*/ - // build write msg SRpcMsg msg = { - .msgType = TDMT_VND_SUBMIT, - .pCont = submitReq, - .contLen = ntohl(submitReq->length), + .msgType = TDMT_VND_CREATE_TABLE, + .pCont = buf, + .contLen = tlen, }; if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { - tqDebug("failed to put into write-queue since %s", terrstr()); + tqError("failed to put into write-queue since %s", terrstr()); } + + return TSDB_CODE_SUCCESS; + +_error: + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("failed to encode submit req since %s", terrstr()); + return TSDB_CODE_OUT_OF_MEMORY; +} + +void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { + const SArray* pBlocks = (const SArray*)data; + SVnode* pVnode = (SVnode*)vnode; + int64_t suid = pTask->tbSink.stbUid; + char* stbFullName = pTask->tbSink.stbFullName; + STSchema* pTSchema = pTask->tbSink.pTSchema; + /*SSchemaWrapper* pSchemaWrapper = pTask->tbSink.pSchemaWrapper;*/ + + int32_t blockSz = taosArrayGetSize(pBlocks); + + tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz); + + void* pBuf = NULL; + SArray* tagArray = NULL; + SArray* pVals = NULL; + SArray* crTblArray = NULL; + + for (int32_t i = 0; i < blockSz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + int32_t rows = pDataBlock->info.rows; + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + SBatchDeleteReq deleteReq = {0}; + deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); + deleteReq.suid = suid; + tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, &deleteReq); + if (taosArrayGetSize(deleteReq.deleteReqs) == 0) { + taosArrayDestroy(deleteReq.deleteReqs); + continue; + } + + int32_t len; + int32_t code; + tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); + if (code < 0) { + // + ASSERT(0); + } + SEncoder encoder; + void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); + void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); + tEncoderInit(&encoder, abuf, len); + tEncodeSBatchDeleteReq(&encoder, &deleteReq); + tEncoderClear(&encoder); + taosArrayDestroy(deleteReq.deleteReqs); + + ((SMsgHead*)serializedDeleteReq)->vgId = pVnode->config.vgId; + + SRpcMsg msg = { + .msgType = TDMT_VND_BATCH_DEL, + .pCont = serializedDeleteReq, + .contLen = len + sizeof(SMsgHead), + }; + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + tqDebug("failed to put delete req into write-queue since %s", terrstr()); + } + } else if (pDataBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + SVCreateTbBatchReq reqs = {0}; + crTblArray = reqs.pArray = taosArrayInit(1, sizeof(struct SVCreateTbReq)); + if (NULL == reqs.pArray) { + goto _end; + } + for (int32_t rowId = 0; rowId < rows; rowId++) { + SVCreateTbReq createTbReq = {0}; + SVCreateTbReq* pCreateTbReq = &createTbReq; + if (!pCreateTbReq) { + goto _end; + } + + // set const + pCreateTbReq->flags = 0; + pCreateTbReq->type = TSDB_CHILD_TABLE; + pCreateTbReq->ctb.suid = suid; + + // set super table name + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + pCreateTbReq->ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName); + + // set tag content + int32_t size = taosArrayGetSize(pDataBlock->pDataBlock); + if (size == 2) { + tagArray = taosArrayInit(1, sizeof(STagVal)); + if (!tagArray) { + goto _end; + } + STagVal tagVal = { + .cid = pTSchema->numOfCols + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.id.groupId, + }; + taosArrayPush(tagArray, &tagVal); + + // set tag name + SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = "group_id"; + taosArrayPush(tagName, tagNameStr); + pCreateTbReq->ctb.tagName = tagName; + } else { + tagArray = taosArrayInit(size - 1, sizeof(STagVal)); + if (!tagArray) { + goto _end; + } + for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) { + SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId); + STagVal tagVal = { + .cid = pTSchema->numOfCols + step, + .type = pTagData->info.type, + }; + void* pData = colDataGetData(pTagData, rowId); + if (colDataIsNull_s(pTagData, rowId)) { + tagVal.type = TSDB_DATA_TYPE_NULL; + tagVal.pData = NULL; + tagVal.nData = 0; + } else if (IS_VAR_DATA_TYPE(pTagData->info.type)) { + tagVal.nData = varDataLen(pData); + tagVal.pData = varDataVal(pData); + } else { + memcpy(&tagVal.i64, pData, pTagData->info.bytes); + } + taosArrayPush(tagArray, &tagVal); + } + } + pCreateTbReq->ctb.tagNum = taosArrayGetSize(tagArray); + + STag* pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + tagArray = taosArrayDestroy(tagArray); + if (pTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + pCreateTbReq->ctb.pTag = (uint8_t*)pTag; + + // set table name + SColumnInfoData* pTbColInfo = taosArrayGet(pDataBlock->pDataBlock, UD_TABLE_NAME_COLUMN_INDEX); + if (colDataIsNull_s(pTbColInfo, rowId)) { + SColumnInfoData* pGpIdColInfo = taosArrayGet(pDataBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX); + void* pGpIdData = colDataGetData(pGpIdColInfo, rowId); + pCreateTbReq->name = buildCtbNameByGroupId(stbFullName, *(uint64_t*)pGpIdData); + } else { + void* pTbData = colDataGetData(pTbColInfo, rowId); + pCreateTbReq->name = taosMemoryCalloc(1, varDataLen(pTbData) + 1); + memcpy(pCreateTbReq->name, varDataVal(pTbData), varDataLen(pTbData)); + } + taosArrayPush(reqs.pArray, pCreateTbReq); + } + reqs.nReqs = taosArrayGetSize(reqs.pArray); + if (tqPutReqToQueue(pVnode, &reqs) != TSDB_CODE_SUCCESS) { + goto _end; + } + tagArray = taosArrayDestroy(tagArray); + taosArrayDestroyEx(crTblArray, (FDelete)tdDestroySVCreateTbReq); + crTblArray = NULL; + } else { + SSubmitTbData tbData = {0}; + tqDebug("tq sink pipe2, convert block1 %d, rows: %d", i, rows); + + if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow*)))) { + goto _end; + } + + tbData.suid = suid; + tbData.uid = 0; // uid is assigned by vnode + tbData.sver = pTSchema->version; + + char* ctbName = NULL; + tqDebug("vgId:%d, stream write into %s, table auto created", TD_VID(pVnode), pDataBlock->info.parTbName); + if (pDataBlock->info.parTbName[0]) { + ctbName = strdup(pDataBlock->info.parTbName); + } else { + ctbName = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId); + } + + SMetaReader mr = {0}; + metaReaderInit(&mr, pVnode->pMeta, 0); + if (metaGetTableEntryByName(&mr, ctbName) < 0) { + metaReaderClear(&mr); + tqDebug("vgId:%d, stream write into %s, table auto created", TD_VID(pVnode), ctbName); + + SVCreateTbReq* pCreateTbReq = NULL; + + if (!(pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq)))) { + goto _end; + }; + + // set const + pCreateTbReq->flags = 0; + pCreateTbReq->type = TSDB_CHILD_TABLE; + pCreateTbReq->ctb.suid = suid; + + // set super table name + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + pCreateTbReq->ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName); + + // set tag content + tagArray = taosArrayInit(1, sizeof(STagVal)); + if (!tagArray) { + goto _end; + } + STagVal tagVal = { + .cid = pTSchema->numOfCols + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.id.groupId, + }; + taosArrayPush(tagArray, &tagVal); + pCreateTbReq->ctb.tagNum = taosArrayGetSize(tagArray); + + STag* pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + tagArray = taosArrayDestroy(tagArray); + if (pTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + pCreateTbReq->ctb.pTag = (uint8_t*)pTag; + + // set tag name + SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = {0}; + strcpy(tagNameStr, "group_id"); + taosArrayPush(tagName, tagNameStr); + pCreateTbReq->ctb.tagName = tagName; + + // set table name + pCreateTbReq->name = ctbName; + ctbName = NULL; + + tbData.pCreateTbReq = pCreateTbReq; + tbData.flags = SUBMIT_REQ_AUTO_CREATE_TABLE; + } else { + if (mr.me.type != TSDB_CHILD_TABLE) { + tqError("vgId:%d, failed to write into %s, since table type incorrect, type %d", TD_VID(pVnode), ctbName, + mr.me.type); + metaReaderClear(&mr); + taosMemoryFree(ctbName); + continue; + } + + if (mr.me.ctbEntry.suid != suid) { + tqError("vgId:%d, failed to write into %s, since suid mismatch, expect suid: %" PRId64 + ", actual suid %" PRId64 "", + TD_VID(pVnode), ctbName, suid, mr.me.ctbEntry.suid); + metaReaderClear(&mr); + taosMemoryFree(ctbName); + } + + tbData.uid = mr.me.uid; + metaReaderClear(&mr); + taosMemoryFreeClear(ctbName); + } + + // rows + if (!pVals && !(pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)))) { + taosArrayDestroy(tbData.aRowP); + goto _end; + } + + for (int32_t j = 0; j < rows; j++) { + taosArrayClear(pVals); + int32_t dataIndex = 0; + for (int32_t k = 0; k < pTSchema->numOfCols; k++) { + const STColumn* pCol = &pTSchema->columns[k]; + if (k == 0) { + SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, dataIndex); + void* colData = colDataGetData(pColData, j); + tqDebug("tq sink pipe2, row %d, col %d ts %" PRId64, j, k, *(int64_t*)colData); + } + if (IS_SET_NULL(pCol)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else{ + SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, dataIndex); + if (colDataIsNull_s(pColData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + dataIndex++; + } else { + void* colData = colDataGetData(pColData, j); + if (IS_STR_DATA_TYPE(pCol->type)) { + SValue sv = + (SValue){.nData = varDataLen(colData), .pData = varDataVal(colData)}; // address copy, no value + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } else { + SValue sv; + memcpy(&sv.val, colData, tDataTypes[pCol->type].bytes); + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } + dataIndex++; + } + } + } + SRow* pRow = NULL; + if ((terrno = tRowBuild(pVals, (STSchema*)pTSchema, &pRow)) < 0) { + tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + goto _end; + } + ASSERT(pRow); + taosArrayPush(tbData.aRowP, &pRow); + } + + SSubmitReq2 submitReq = {0}; + if (!(submitReq.aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { + goto _end; + } + + taosArrayPush(submitReq.aSubmitTbData, &tbData); + + // encode + int32_t len; + int32_t code; + tEncodeSize(tEncodeSSubmitReq2, &submitReq, len, code); + SEncoder encoder; + len += sizeof(SMsgHead); + pBuf = rpcMallocCont(len); + if (NULL == pBuf) { + goto _end; + } + ((SMsgHead*)pBuf)->vgId = TD_VID(pVnode); + ((SMsgHead*)pBuf)->contLen = htonl(len); + tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SMsgHead)), len - sizeof(SMsgHead)); + if (tEncodeSSubmitReq2(&encoder, &submitReq) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("failed to encode submit req since %s", terrstr()); + tEncoderClear(&encoder); + rpcFreeCont(pBuf); + continue; + } + tEncoderClear(&encoder); + tDestroySSubmitReq2(&submitReq, TSDB_MSG_FLG_ENCODE); + + SRpcMsg msg = { + .msgType = TDMT_VND_SUBMIT, + .pCont = pBuf, + .contLen = len, + }; + + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + tqDebug("failed to put into write-queue since %s", terrstr()); + } + } + } +_end: + taosArrayDestroy(tagArray); + taosArrayDestroy(pVals); + taosArrayDestroyEx(crTblArray, (FDelete)tdDestroySVCreateTbReq); + // TODO: change } -#endif diff --git a/source/dnode/vnode/src/tq/tqSnapshot.c b/source/dnode/vnode/src/tq/tqSnapshot.c index ab7093a701e0a5207b5f64072350ba5bc2bae703..5c0649c10985b05dc1f5677fa9bfe1812dd07808 100644 --- a/source/dnode/vnode/src/tq/tqSnapshot.c +++ b/source/dnode/vnode/src/tq/tqSnapshot.c @@ -173,6 +173,8 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { if (code) goto _err; } + int vgId = TD_VID(pWriter->pTq->pVnode); + taosMemoryFree(pWriter); *ppWriter = NULL; @@ -184,7 +186,7 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { return code; _err: - tqError("vgId:%d, tq snapshot writer close failed since %s", TD_VID(pWriter->pTq->pVnode), tstrerror(code)); + tqError("vgId:%d, tq snapshot writer close failed since %s", vgId, tstrerror(code)); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 6a8251706721cc4ac5dd178d258728c4d9f0b2d2..d18014c37b096f044a6cf4fbb7a933193e297d02 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -91,12 +91,10 @@ int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { } } + taosLRUCacheRelease(pCache, h, invalidate); if (invalidate) { - taosLRUCacheRelease(pCache, h, true); - } else { - taosLRUCacheRelease(pCache, h, false); + taosLRUCacheErase(pCache, key, keyLen); } - // void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen); } return code; @@ -124,12 +122,10 @@ int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { } } + taosLRUCacheRelease(pCache, h, invalidate); if (invalidate) { - taosLRUCacheRelease(pCache, h, true); - } else { - taosLRUCacheRelease(pCache, h, false); + taosLRUCacheErase(pCache, key, keyLen); } - // void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen); } return code; @@ -190,7 +186,7 @@ int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { return code; } -int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, STSRow *row, bool dup) { +int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, TSDBROW *row, bool dup) { int32_t code = 0; STSRow *cacheRow = NULL; char key[32] = {0}; @@ -201,13 +197,51 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); if (h) { STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); - TSKEY keyTs = row->ts; + TSKEY keyTs = TSDBROW_TS(row); bool invalidate = false; SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h); int16_t nCol = taosArrayGetSize(pLast); int16_t iCol = 0; + if (nCol <= 0) { + nCol = pTSchema->numOfCols; + + STColumn *pTColumn = &pTSchema->columns[0]; + SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = keyTs}); + if (taosArrayPush(pLast, &(SLastCol){.ts = keyTs, .colVal = tColVal}) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + + for (iCol = 1; iCol < nCol; ++iCol) { + SColVal colVal = {0}; + tsdbRowGetColVal(row, pTSchema, iCol, &colVal); + + SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; + if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { + SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); + taosMemoryFree(pLastCol->colVal.value.pData); + + lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); + if (lastCol.colVal.value.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); + } + + if (taosArrayPush(pLast, &lastCol) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + } + + goto _invalidate; + } + SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); if (keyTs > tTsVal->ts) { STColumn *pTColumn = &pTSchema->columns[0]; @@ -222,7 +256,7 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST SColVal *tColVal = &tTsVal1->colVal; SColVal colVal = {0}; - tTSRowGetVal(row, pTSchema, iCol, &colVal); + tsdbRowGetColVal(row, pTSchema, iCol, &colVal); if (!COL_VAL_IS_NONE(&colVal)) { if (keyTs == tTsVal1->ts && !COL_VAL_IS_NONE(tColVal)) { invalidate = true; @@ -261,7 +295,7 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST return code; } -int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb) { +int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, TSDBROW *row, STsdb *pTsdb) { int32_t code = 0; STSRow *cacheRow = NULL; char key[32] = {0}; @@ -272,13 +306,51 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); if (h) { STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); - TSKEY keyTs = row->ts; + TSKEY keyTs = TSDBROW_TS(row); bool invalidate = false; SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h); int16_t nCol = taosArrayGetSize(pLast); int16_t iCol = 0; + if (nCol <= 0) { + nCol = pTSchema->numOfCols; + + STColumn *pTColumn = &pTSchema->columns[0]; + SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = keyTs}); + if (taosArrayPush(pLast, &(SLastCol){.ts = keyTs, .colVal = tColVal}) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + + for (iCol = 1; iCol < nCol; ++iCol) { + SColVal colVal = {0}; + tsdbRowGetColVal(row, pTSchema, iCol, &colVal); + + SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; + if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { + SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); + taosMemoryFree(pLastCol->colVal.value.pData); + + lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); + if (lastCol.colVal.value.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); + } + + if (taosArrayPush(pLast, &lastCol) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + } + + goto _invalidate; + } + SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); if (keyTs > tTsVal->ts) { STColumn *pTColumn = &pTSchema->columns[0]; @@ -293,7 +365,7 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb SColVal *tColVal = &tTsVal1->colVal; SColVal colVal = {0}; - tTSRowGetVal(row, pTSchema, iCol, &colVal); + tsdbRowGetColVal(row, pTSchema, iCol, &colVal); if (COL_VAL_IS_VALUE(&colVal)) { if (keyTs == tTsVal1->ts && COL_VAL_IS_VALUE(tColVal)) { invalidate = true; @@ -617,7 +689,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { } else { // tBlockDataDestroy(&state->blockData, 1); if (state->pBlockData) { - tBlockDataDestroy(state->pBlockData, 1); + tBlockDataDestroy(state->pBlockData); state->pBlockData = NULL; } @@ -685,7 +757,8 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { tBlockDataReset(state->pBlockData); tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetDataBlk); - /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); */ + /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); + */ tBlockDataReset(state->pBlockData); TABLEID tid = {.suid = state->suid, .uid = state->uid}; code = tBlockDataInit(state->pBlockData, &tid, state->pTSchema, NULL, 0); @@ -739,7 +812,7 @@ _err: state->aBlockIdx = NULL; } if (state->pBlockData) { - tBlockDataDestroy(state->pBlockData, 1); + tBlockDataDestroy(state->pBlockData); state->pBlockData = NULL; } @@ -766,7 +839,7 @@ int32_t clearNextRowFromFS(void *iter) { } if (state->pBlockData) { // tBlockDataDestroy(&state->blockData, 1); - tBlockDataDestroy(state->pBlockData, 1); + tBlockDataDestroy(state->pBlockData); state->pBlockData = NULL; } @@ -1080,6 +1153,8 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow) { iMax[nMax] = i; max[nMax++] = pIter->input[i].pRow; + } else { + pIter->input[i].next = false; } } } @@ -1219,12 +1294,12 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo // build the result ts row here *dup = false; - if (taosArrayGetSize(pColArray) != nCol) { - *ppColArray = NULL; - taosArrayDestroy(pColArray); - } else { - *ppColArray = pColArray; - } + // if (taosArrayGetSize(pColArray) != nCol) { + //*ppColArray = NULL; + // taosArrayDestroy(pColArray); + //} else { + *ppColArray = pColArray; + //} nextRowIterClose(&iter); // taosMemoryFreeClear(pTSchema); @@ -1337,12 +1412,12 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach } } while (setNoneCol); - if (taosArrayGetSize(pColArray) <= 0) { - *ppLastArray = NULL; - taosArrayDestroy(pColArray); - } else { - *ppLastArray = pColArray; - } + // if (taosArrayGetSize(pColArray) <= 0) { + //*ppLastArray = NULL; + // taosArrayDestroy(pColArray); + //} else { + *ppLastArray = pColArray; + //} nextRowIterClose(&iter); // taosMemoryFreeClear(pTSchema); @@ -1373,8 +1448,8 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader * SArray *pArray = NULL; bool dup = false; // which is always false for now code = mergeLastRow(uid, pTsdb, &dup, &pArray, pr); - // if table's empty or error, return code of -1 - if (code < 0 || pArray == NULL) { + // if table's empty or error, set handle NULL and return + if (code < 0 /* || pArray == NULL*/) { if (!dup && pArray) { taosArrayDestroy(pArray); } @@ -1392,13 +1467,8 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader * if (status != TAOS_LRU_STATUS_OK) { code = -1; } - - // taosThreadMutexUnlock(&pTsdb->lruMutex); - - // h = taosLRUCacheLookup(pCache, key, keyLen); - } // else { + } taosThreadMutexUnlock(&pTsdb->lruMutex); - //} } *handle = h; @@ -1422,8 +1492,8 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, if (!h) { SArray *pLastArray = NULL; code = mergeLast(uid, pTsdb, &pLastArray, pr); - // if table's empty or error, return code of -1 - if (code < 0 || pLastArray == NULL) { + // if table's empty or error, set handle NULL and return + if (code < 0 /* || pLastArray == NULL*/) { taosThreadMutexUnlock(&pTsdb->lruMutex); *handle = NULL; @@ -1437,13 +1507,8 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, if (status != TAOS_LRU_STATUS_OK) { code = -1; } - - // taosThreadMutexUnlock(&pTsdb->lruMutex); - - // h = taosLRUCacheLookup(pCache, key, keyLen); - } // else { + } taosThreadMutexUnlock(&pTsdb->lruMutex); - //} } *handle = h; diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index f05f5d5c88f7ef19c8598410019326edc9b1c5c6..6a5acecfc3226753108863e3537f281028c8a34f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -109,6 +109,8 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, p->type = type; p->pVnode = pVnode; + p->pTsdb = p->pVnode->pTsdb; + p->verRange = (SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}; p->numOfCols = numOfCols; p->suid = suid; @@ -145,6 +147,7 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, } p->idstr = taosMemoryStrDup(idstr); + taosThreadMutexInit(&p->readerMutex, NULL); *pReader = p; return TSDB_CODE_SUCCESS; @@ -164,7 +167,9 @@ void* tsdbCacherowsReaderClose(void* pReader) { destroyLastBlockLoadInfo(p->pLoadInfo); - taosMemoryFree((void*) p->idstr); + taosMemoryFree((void*)p->idstr); + taosThreadMutexDestroy(&p->readerMutex); + taosMemoryFree(pReader); return NULL; } @@ -199,6 +204,27 @@ static void freeItem(void* pItem) { } } +static int32_t tsdbCacheQueryReseek(void* pQHandle) { + int32_t code = 0; + SCacheRowsReader* pReader = pQHandle; + + code = taosThreadMutexTryLock(&pReader->readerMutex); + if (code == 0) { + // pause current reader's state if not paused, save ts & version for resuming + // just wait for the big all tables' snapshot untaking for now + + code = TSDB_CODE_VND_QUERY_BUSY; + + taosThreadMutexUnlock(&pReader->readerMutex); + + return code; + } else if (code == EBUSY) { + return TSDB_CODE_VND_QUERY_BUSY; + } else { + return -1; + } +} + int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32_t* slotIds, SArray* pTableUidList) { if (pReader == NULL || pResBlock == NULL) { return TSDB_CODE_INVALID_PARA; @@ -241,7 +267,8 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 taosArrayPush(pLastCols, &p); } - tsdbTakeReadSnap(pr->pVnode->pTsdb, &pr->pReadSnap, "cache-l"); + taosThreadMutexLock(&pr->readerMutex); + tsdbTakeReadSnap((STsdbReader*)pr, tsdbCacheQueryReseek, &pr->pReadSnap); pr->pDataFReader = NULL; pr->pDataFReaderLast = NULL; @@ -258,6 +285,10 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 if (h == NULL) { continue; } + if (taosArrayGetSize(pRow) <= 0) { + tsdbCacheRelease(lruCache, h); + continue; + } { for (int32_t k = 0; k < pr->numOfCols; ++k) { @@ -327,6 +358,10 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 if (h == NULL) { continue; } + if (taosArrayGetSize(pRow) <= 0) { + tsdbCacheRelease(lruCache, h); + continue; + } saveOneRow(pRow, pResBlock, pr, slotIds, pRes, pr->idstr); // TODO reset the pRes @@ -347,8 +382,9 @@ _end: tsdbDataFReaderClose(&pr->pDataFReaderLast); tsdbDataFReaderClose(&pr->pDataFReader); - tsdbUntakeReadSnap(pr->pVnode->pTsdb, pr->pReadSnap, "cache-l"); resetLastBlockLoadInfo(pr->pLoadInfo); + tsdbUntakeReadSnap((STsdbReader*)pr, pr->pReadSnap, true); + taosThreadMutexUnlock(&pr->readerMutex); for (int32_t j = 0; j < pr->numOfCols; ++j) { taosMemoryFree(pRes[j]); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 8ec59ea95949a4bccda38c7cc2fda25e2f769fb4..d15f848cfdcde11c946a943d96b0397933324b14 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -175,7 +175,7 @@ int32_t tsdbCommit(STsdb *pTsdb, SCommitInfo *pInfo) { pTsdb->imem = NULL; taosThreadRwlockUnlock(&pTsdb->rwLock); - tsdbUnrefMemTable(pMemTable); + tsdbUnrefMemTable(pMemTable, NULL, true); goto _exit; } @@ -913,24 +913,24 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { // reader taosArrayDestroy(pCommitter->dReader.aBlockIdx); tMapDataClear(&pCommitter->dReader.mBlock); - tBlockDataDestroy(&pCommitter->dReader.bData, 1); + tBlockDataDestroy(&pCommitter->dReader.bData); // merger for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) { SDataIter *pIter = &pCommitter->aDataIter[iStt]; taosArrayDestroy(pIter->aSttBlk); - tBlockDataDestroy(&pIter->bData, 1); + tBlockDataDestroy(&pIter->bData); } // writer taosArrayDestroy(pCommitter->dWriter.aBlockIdx); taosArrayDestroy(pCommitter->dWriter.aSttBlk); tMapDataClear(&pCommitter->dWriter.mBlock); - tBlockDataDestroy(&pCommitter->dWriter.bData, 1); + tBlockDataDestroy(&pCommitter->dWriter.bData); #if USE_STREAM_COMPRESSION tDiskDataBuilderDestroy(pCommitter->dWriter.pBuilder); #else - tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1); + tBlockDataDestroy(&pCommitter->dWriter.bDatal); #endif tDestroyTSchema(pCommitter->skmTable.pTSchema); tDestroyTSchema(pCommitter->skmRow.pTSchema); @@ -1183,7 +1183,6 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) tBlockDataClear(pBlockData); while (pRowInfo) { - ASSERT(pRowInfo->row.type == 0); code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); TSDB_CHECK_CODE(code, lino, _exit); @@ -1251,7 +1250,6 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) pRow = NULL; } } else if (c > 0) { - ASSERT(pRowInfo->row.type == 0); code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); TSDB_CHECK_CODE(code, lino, _exit); @@ -1493,7 +1491,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { while (pRowInfo) { STSchema *pTSchema = NULL; - if (pRowInfo->row.type == 0) { + if (pRowInfo->row.type == TSDBROW_ROW_FMT) { code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); TSDB_CHECK_CODE(code, lino, _exit); pTSchema = pCommitter->skmRow.pTSchema; @@ -1536,7 +1534,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { while (pRowInfo) { STSchema *pTSchema = NULL; - if (pRowInfo->row.type == 0) { + if (pRowInfo->row.type == TSDBROW_ROW_FMT) { code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); TSDB_CHECK_CODE(code, lino, _exit); pTSchema = pCommitter->skmRow.pTSchema; @@ -1666,7 +1664,7 @@ int32_t tsdbFinishCommit(STsdb *pTsdb) { // unlock taosThreadRwlockUnlock(&pTsdb->rwLock); if (pMemTable) { - tsdbUnrefMemTable(pMemTable); + tsdbUnrefMemTable(pMemTable, NULL, true); } _exit: diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index b46a00363817666cdff7bc7756ff204483787c36..ae9af11f5ae8226c1fceefb57b4d83167800772a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -596,7 +596,7 @@ int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTS if (pBuilder->bi.maxKey < kRow.ts) pBuilder->bi.maxKey = kRow.ts; STSDBRowIter iter = {0}; - tsdbRowIterInit(&iter, pRow, pTSchema); + tsdbRowIterOpen(&iter, pRow, pTSchema); SColVal *pColVal = tsdbRowIterNext(&iter); for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) { diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 0a7f59e429142a8a8cdd52627a36ff6c8856acaf..d68afbed1f0e4a446e64b1aabe36f7b6bdb0a4fc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -19,17 +19,23 @@ #define SL_MAX_LEVEL 5 // sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l) * 2 -#define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + ((l) << 4)) -#define SL_NODE_FORWARD(n, l) ((n)->forwards[l]) -#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) +#define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + ((l) << 4)) +#define SL_NODE_FORWARD(n, l) ((n)->forwards[l]) +#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) +#define SL_GET_NODE_FORWARD(n, l) ((SMemSkipListNode *)atomic_load_ptr(&SL_NODE_FORWARD(n, l))) +#define SL_GET_NODE_BACKWARD(n, l) ((SMemSkipListNode *)atomic_load_ptr(&SL_NODE_BACKWARD(n, l))) +#define SL_SET_NODE_FORWARD(n, l, p) atomic_store_ptr(&SL_NODE_FORWARD(n, l), p) +#define SL_SET_NODE_BACKWARD(n, l, p) atomic_store_ptr(&SL_NODE_BACKWARD(n, l), p) #define SL_MOVE_BACKWARD 0x1 #define SL_MOVE_FROM_POS 0x2 static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags); static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData); -static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, int64_t version, - SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubmitBlkRsp *pRsp); +static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version, + SSubmitTbData *pSubmitTbData, int32_t *affectedRows); +static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version, + SSubmitTbData *pSubmitTbData, int32_t *affectedRows); int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { int32_t code = 0; @@ -44,6 +50,8 @@ int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { pMemTable->pTsdb = pTsdb; pMemTable->pPool = pTsdb->pVnode->inUse; pMemTable->nRef = 1; + pMemTable->minVer = VERSION_MAX; + pMemTable->maxVer = VERSION_MIN; pMemTable->minKey = TSKEY_MAX; pMemTable->maxKey = TSKEY_MIN; pMemTable->nRow = 0; @@ -56,6 +64,8 @@ int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { taosMemoryFree(pMemTable); goto _err; } + // pMemTable->qList.pNext = &pMemTable->qList; + // pMemTable->qList.ppNext = &pMemTable->qList.pNext; vnodeBufPoolRef(pMemTable->pPool); *ppMemTable = pMemTable; @@ -66,9 +76,9 @@ _err: return code; } -void tsdbMemTableDestroy(SMemTable *pMemTable) { +void tsdbMemTableDestroy(SMemTable *pMemTable, bool proactive) { if (pMemTable) { - vnodeBufPoolUnRef(pMemTable->pPool); + vnodeBufPoolUnRef(pMemTable->pPool, proactive); taosMemoryFree(pMemTable->aBucket); taosMemoryFree(pMemTable); } @@ -95,14 +105,14 @@ STbData *tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t return pTbData; } -int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, - SSubmitBlkRsp *pRsp) { +int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitTbData *pSubmitTbData, int32_t *affectedRows) { int32_t code = 0; SMemTable *pMemTable = pTsdb->mem; STbData *pTbData = NULL; - tb_uid_t suid = pMsgIter->suid; - tb_uid_t uid = pMsgIter->uid; + tb_uid_t suid = pSubmitTbData->suid; + tb_uid_t uid = pSubmitTbData->uid; +#if 0 SMetaInfo info; code = metaGetInfo(pTsdb->pVnode->pMeta, uid, &info, NULL); if (code) { @@ -116,14 +126,15 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI if (info.suid) { metaGetInfo(pTsdb->pVnode->pMeta, info.suid, &info, NULL); } - if (pMsgIter->sversion != info.skmVer) { + if (pSubmitTbData->sver != info.skmVer) { tsdbError("vgId:%d, req sver:%d, skmVer:%d suid:%" PRId64 " uid:%" PRId64, TD_VID(pTsdb->pVnode), - pMsgIter->sversion, info.skmVer, suid, uid); + pSubmitTbData->sver, info.skmVer, suid, uid); code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; goto _err; } - pRsp->sver = info.skmVer; + if (pRsp) pRsp->sver = info.skmVer; +#endif // create/get STbData to op code = tsdbGetOrCreateTbData(pMemTable, suid, uid, &pTbData); @@ -132,10 +143,16 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI } // do insert impl - code = tsdbInsertTableDataImpl(pMemTable, pTbData, version, pMsgIter, pBlock, pRsp); - if (code) { - goto _err; + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + code = tsdbInsertColDataToTable(pMemTable, pTbData, version, pSubmitTbData, affectedRows); + } else { + code = tsdbInsertRowDataToTable(pMemTable, pTbData, version, pSubmitTbData, affectedRows); } + if (code) goto _err; + + // update + pMemTable->minVer = TMIN(pMemTable->minVer, version); + pMemTable->maxVer = TMAX(pMemTable->maxVer, version); return code; @@ -188,6 +205,8 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid } pMemTable->nDel++; + pMemTable->minVer = TMIN(pMemTable->minVer, version); + pMemTable->maxVer = TMIN(pMemTable->maxVer, version); if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config) && tsdbKeyCmprFn(&lastKey, &pTbData->maxKey) >= 0) { tsdbCacheDeleteLastrow(pTsdb->lruCache, pTbData->uid, eKey); @@ -228,7 +247,6 @@ void *tsdbTbDataIterDestroy(STbDataIter *pIter) { if (pIter) { taosMemoryFree(pIter); } - return NULL; } @@ -242,22 +260,21 @@ void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDa pIter->pTbData = pTbData; pIter->backward = backward; pIter->pRow = NULL; - pIter->row.type = 0; if (pFrom == NULL) { // create from head or tail if (backward) { - pIter->pNode = SL_NODE_BACKWARD(pTbData->sl.pTail, 0); + pIter->pNode = SL_GET_NODE_BACKWARD(pTbData->sl.pTail, 0); } else { - pIter->pNode = SL_NODE_FORWARD(pTbData->sl.pHead, 0); + pIter->pNode = SL_GET_NODE_FORWARD(pTbData->sl.pHead, 0); } } else { // create from a key if (backward) { tbDataMovePosTo(pTbData, pos, pFrom, SL_MOVE_BACKWARD); - pIter->pNode = SL_NODE_BACKWARD(pos[0], 0); + pIter->pNode = SL_GET_NODE_BACKWARD(pos[0], 0); } else { tbDataMovePosTo(pTbData, pos, pFrom, 0); - pIter->pNode = SL_NODE_FORWARD(pos[0], 0); + pIter->pNode = SL_GET_NODE_FORWARD(pos[0], 0); } } } @@ -271,7 +288,7 @@ bool tsdbTbDataIterNext(STbDataIter *pIter) { return false; } - pIter->pNode = SL_NODE_BACKWARD(pIter->pNode, 0); + pIter->pNode = SL_GET_NODE_BACKWARD(pIter->pNode, 0); if (pIter->pNode == pIter->pTbData->sl.pHead) { return false; } @@ -282,7 +299,7 @@ bool tsdbTbDataIterNext(STbDataIter *pIter) { return false; } - pIter->pNode = SL_NODE_FORWARD(pIter->pNode, 0); + pIter->pNode = SL_GET_NODE_FORWARD(pIter->pNode, 0); if (pIter->pNode == pIter->pTbData->sl.pTail) { return false; } @@ -335,7 +352,7 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid int8_t maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel; ASSERT(pPool != NULL); - pTbData = vnodeBufPoolMalloc(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2); + pTbData = vnodeBufPoolMallocAligned(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2); if (pTbData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -408,17 +425,22 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p if (fromPos) px = pos[pTbData->sl.level - 1]; for (int8_t iLevel = pTbData->sl.level - 1; iLevel >= 0; iLevel--) { - pn = SL_NODE_BACKWARD(px, iLevel); + pn = SL_GET_NODE_BACKWARD(px, iLevel); while (pn != pTbData->sl.pHead) { - tKey.version = pn->version; - tKey.ts = pn->pTSRow->ts; + if (pn->flag == TSDBROW_ROW_FMT) { + tKey.version = pn->version; + tKey.ts = ((SRow *)pn->pData)->ts; + } else if (pn->flag == TSDBROW_COL_FMT) { + tKey.version = ((SBlockData *)pn->pData)->aVersion[pn->iRow]; + tKey.ts = ((SBlockData *)pn->pData)->aTSKEY[pn->iRow]; + } int32_t c = tsdbKeyCmprFn(&tKey, pKey); if (c <= 0) { break; } else { px = pn; - pn = SL_NODE_BACKWARD(px, iLevel); + pn = SL_GET_NODE_BACKWARD(px, iLevel); } } @@ -438,17 +460,22 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p if (fromPos) px = pos[pTbData->sl.level - 1]; for (int8_t iLevel = pTbData->sl.level - 1; iLevel >= 0; iLevel--) { - pn = SL_NODE_FORWARD(px, iLevel); + pn = SL_GET_NODE_FORWARD(px, iLevel); while (pn != pTbData->sl.pTail) { - tKey.version = pn->version; - tKey.ts = pn->pTSRow->ts; + if (pn->flag == TSDBROW_ROW_FMT) { + tKey.version = pn->version; + tKey.ts = ((SRow *)pn->pData)->ts; + } else if (pn->flag == TSDBROW_COL_FMT) { + tKey.version = ((SBlockData *)pn->pData)->aVersion[pn->iRow]; + tKey.ts = ((SBlockData *)pn->pData)->aTSKEY[pn->iRow]; + } int32_t c = tsdbKeyCmprFn(&tKey, pKey); if (c >= 0) { break; } else { px = pn; - pn = SL_NODE_FORWARD(px, iLevel); + pn = SL_GET_NODE_FORWARD(px, iLevel); } } @@ -468,138 +495,245 @@ static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) { return level; } -static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListNode **pos, int64_t version, - STSRow *pRow, int8_t forward) { +static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListNode **pos, TSDBROW *pRow, + int8_t forward) { int32_t code = 0; int8_t level; SMemSkipListNode *pNode; SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; + int64_t nSize; - // node + // create node level = tsdbMemSkipListRandLevel(&pTbData->sl); - ASSERT(pPool != NULL); - pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level)); + nSize = SL_NODE_SIZE(level); + if (pRow->type == TSDBROW_ROW_FMT) { + pNode = (SMemSkipListNode *)vnodeBufPoolMallocAligned(pPool, nSize + pRow->pTSRow->len); + } else if (pRow->type == TSDBROW_COL_FMT) { + pNode = (SMemSkipListNode *)vnodeBufPoolMallocAligned(pPool, nSize); + } else { + ASSERT(0); + } if (pNode == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } + pNode->level = level; - pNode->version = version; - pNode->pTSRow = vnodeBufPoolMalloc(pPool, pRow->len); - if (NULL == pNode->pTSRow) { + pNode->flag = pRow->type; + if (pRow->type == TSDBROW_ROW_FMT) { + pNode->version = pRow->version; + pNode->pData = (char *)pNode + nSize; + memcpy(pNode->pData, pRow->pTSRow, pRow->pTSRow->len); + } else if (pRow->type == TSDBROW_COL_FMT) { + pNode->iRow = pRow->iRow; + pNode->pData = pRow->pBlockData; + } else { + ASSERT(0); + } + + // set node + if (forward) { + for (int8_t iLevel = 0; iLevel < level; iLevel++) { + SL_NODE_FORWARD(pNode, iLevel) = SL_NODE_FORWARD(pos[iLevel], iLevel); + SL_NODE_BACKWARD(pNode, iLevel) = pos[iLevel]; + } + } else { + for (int8_t iLevel = 0; iLevel < level; iLevel++) { + SL_NODE_FORWARD(pNode, iLevel) = pos[iLevel]; + SL_NODE_BACKWARD(pNode, iLevel) = SL_NODE_BACKWARD(pos[iLevel], iLevel); + } + } + + // set forward and backward + if (forward) { + for (int8_t iLevel = level - 1; iLevel >= 0; iLevel--) { + SMemSkipListNode *pNext = pos[iLevel]->forwards[iLevel]; + + SL_SET_NODE_FORWARD(pos[iLevel], iLevel, pNode); + SL_SET_NODE_BACKWARD(pNext, iLevel, pNode); + + pos[iLevel] = pNode; + } + } else { + for (int8_t iLevel = level - 1; iLevel >= 0; iLevel--) { + SMemSkipListNode *pPrev = pos[iLevel]->forwards[pos[iLevel]->level + iLevel]; + + SL_SET_NODE_FORWARD(pPrev, iLevel, pNode); + SL_SET_NODE_BACKWARD(pos[iLevel], iLevel, pNode); + + pos[iLevel] = pNode; + } + } + + pTbData->sl.size++; + if (pTbData->sl.level < pNode->level) { + pTbData->sl.level = pNode->level; + } + +_exit: + return code; +} + +static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version, + SSubmitTbData *pSubmitTbData, int32_t *affectedRows) { + int32_t code = 0; + + SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; + int32_t nColData = TARRAY_SIZE(pSubmitTbData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pSubmitTbData->aCol); + + ASSERT(aColData[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); + ASSERT(aColData[0].type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(aColData[0].flag == HAS_VALUE); + + // copy and construct block data + SBlockData *pBlockData = vnodeBufPoolMalloc(pPool, sizeof(*pBlockData)); + if (pBlockData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - memcpy(pNode->pTSRow, pRow, pRow->len); - for (int8_t iLevel = level - 1; iLevel >= 0; iLevel--) { - SMemSkipListNode *pn = pos[iLevel]; - SMemSkipListNode *px; + pBlockData->suid = pTbData->suid; + pBlockData->uid = pTbData->uid; + pBlockData->nRow = aColData[0].nVal; + pBlockData->aUid = NULL; + pBlockData->aVersion = vnodeBufPoolMalloc(pPool, aColData[0].nData); + if (pBlockData->aVersion == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + for (int32_t i = 0; i < pBlockData->nRow; i++) { // todo: here can be optimized + pBlockData->aVersion[i] = version; + } - if (forward) { - px = SL_NODE_FORWARD(pn, iLevel); + pBlockData->aTSKEY = vnodeBufPoolMalloc(pPool, aColData[0].nData); + if (pBlockData->aTSKEY == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(pBlockData->aTSKEY, aColData[0].pData, aColData[0].nData); - SL_NODE_BACKWARD(pNode, iLevel) = pn; - SL_NODE_FORWARD(pNode, iLevel) = px; - } else { - px = SL_NODE_BACKWARD(pn, iLevel); + pBlockData->nColData = nColData - 1; + pBlockData->aColData = vnodeBufPoolMalloc(pPool, sizeof(SColData) * pBlockData->nColData); + if (pBlockData->aColData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + for (int32_t iColData = 0; iColData < pBlockData->nColData; ++iColData) { + code = tColDataCopy(&aColData[iColData + 1], &pBlockData->aColData[iColData], (xMallocFn)vnodeBufPoolMalloc, pPool); + if (code) goto _exit; + } + + // loop to add each row to the skiplist + SMemSkipListNode *pos[SL_MAX_LEVEL]; + TSDBROW tRow = tsdbRowFromBlockData(pBlockData, 0); + TSDBKEY key = {.version = version, .ts = pBlockData->aTSKEY[0]}; + TSDBROW lRow; // last row - SL_NODE_BACKWARD(pNode, iLevel) = px; - SL_NODE_FORWARD(pNode, iLevel) = pn; + // first row + tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD); + if ((code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 0))) goto _exit; + pTbData->minKey = TMIN(pTbData->minKey, key.ts); + lRow = tRow; + + // remain row + ++tRow.iRow; + if (tRow.iRow < pBlockData->nRow) { + for (int8_t iLevel = pos[0]->level; iLevel < pTbData->sl.maxLevel; iLevel++) { + pos[iLevel] = SL_NODE_BACKWARD(pos[iLevel], iLevel); } - } - for (int8_t iLevel = level - 1; iLevel >= 0; iLevel--) { - SMemSkipListNode *pn = pos[iLevel]; - SMemSkipListNode *px; + while (tRow.iRow < pBlockData->nRow) { + key.ts = pBlockData->aTSKEY[tRow.iRow]; - if (forward) { - px = SL_NODE_FORWARD(pn, iLevel); + if (SL_NODE_FORWARD(pos[0], 0) != pTbData->sl.pTail) { + tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_FROM_POS); + } - SL_NODE_FORWARD(pn, iLevel) = pNode; - SL_NODE_BACKWARD(px, iLevel) = pNode; - } else { - px = SL_NODE_BACKWARD(pn, iLevel); + if ((code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 1))) goto _exit; + lRow = tRow; - SL_NODE_FORWARD(px, iLevel) = pNode; - SL_NODE_BACKWARD(pn, iLevel) = pNode; + ++tRow.iRow; } + } - pos[iLevel] = pNode; + if (key.ts >= pTbData->maxKey) { + pTbData->maxKey = key.ts; + + if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config)) { + tsdbCacheInsertLastrow(pMemTable->pTsdb->lruCache, pMemTable->pTsdb, pTbData->uid, &lRow, true); + } } - pTbData->sl.size++; - if (pTbData->sl.level < pNode->level) { - pTbData->sl.level = pNode->level; + if (TSDB_CACHE_LAST(pMemTable->pTsdb->pVnode->config)) { + tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, &lRow, pMemTable->pTsdb); } + // SMemTable + pMemTable->minKey = TMIN(pMemTable->minKey, pTbData->minKey); + pMemTable->maxKey = TMAX(pMemTable->maxKey, pTbData->maxKey); + pMemTable->nRow += pBlockData->nRow; + + if (affectedRows) *affectedRows = pBlockData->nRow; + _exit: return code; } -static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, int64_t version, - SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubmitBlkRsp *pRsp) { - int32_t code = 0; - SSubmitBlkIter blkIter = {0}; +static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version, + SSubmitTbData *pSubmitTbData, int32_t *affectedRows) { + int32_t code = 0; + + int32_t nRow = TARRAY_SIZE(pSubmitTbData->aRowP); + SRow **aRow = (SRow **)TARRAY_DATA(pSubmitTbData->aRowP); TSDBKEY key = {.version = version}; SMemSkipListNode *pos[SL_MAX_LEVEL]; - TSDBROW row = tsdbRowFromTSRow(version, NULL); - int32_t nRow = 0; - STSRow *pLastRow = NULL; - - tInitSubmitBlkIter(pMsgIter, pBlock, &blkIter); + TSDBROW tRow = {.type = TSDBROW_ROW_FMT, .version = version}; + int32_t iRow = 0; + TSDBROW lRow; // backward put first data - row.pTSRow = tGetSubmitBlkNext(&blkIter); - if (row.pTSRow == NULL) return code; - - key.ts = row.pTSRow->ts; - nRow++; + tRow.pTSRow = aRow[iRow++]; + key.ts = tRow.pTSRow->ts; tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD); - code = tbDataDoPut(pMemTable, pTbData, pos, version, row.pTSRow, 0); - if (code) { - goto _err; - } + code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 0); + if (code) goto _exit; + lRow = tRow; pTbData->minKey = TMIN(pTbData->minKey, key.ts); - pLastRow = row.pTSRow; - // forward put rest data - row.pTSRow = tGetSubmitBlkNext(&blkIter); - if (row.pTSRow) { + if (iRow < nRow) { for (int8_t iLevel = pos[0]->level; iLevel < pTbData->sl.maxLevel; iLevel++) { pos[iLevel] = SL_NODE_BACKWARD(pos[iLevel], iLevel); } - do { - key.ts = row.pTSRow->ts; - nRow++; + + while (iRow < nRow) { + tRow.pTSRow = aRow[iRow]; + key.ts = tRow.pTSRow->ts; + if (SL_NODE_FORWARD(pos[0], 0) != pTbData->sl.pTail) { tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_FROM_POS); } - code = tbDataDoPut(pMemTable, pTbData, pos, version, row.pTSRow, 1); - if (code) { - goto _err; - } - pLastRow = row.pTSRow; + code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 1); + if (code) goto _exit; - row.pTSRow = tGetSubmitBlkNext(&blkIter); - } while (row.pTSRow); + lRow = tRow; + + iRow++; + } } if (key.ts >= pTbData->maxKey) { - if (key.ts > pTbData->maxKey) { - pTbData->maxKey = key.ts; - } + pTbData->maxKey = key.ts; - if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config) && pLastRow != NULL) { - tsdbCacheInsertLastrow(pMemTable->pTsdb->lruCache, pMemTable->pTsdb, pTbData->uid, pLastRow, true); + if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config)) { + tsdbCacheInsertLastrow(pMemTable->pTsdb->lruCache, pMemTable->pTsdb, pTbData->uid, &lRow, true); } } if (TSDB_CACHE_LAST(pMemTable->pTsdb->pVnode->config)) { - tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, pLastRow, pMemTable->pTsdb); + tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, &lRow, pMemTable->pTsdb); } // SMemTable @@ -607,27 +741,38 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i pMemTable->maxKey = TMAX(pMemTable->maxKey, pTbData->maxKey); pMemTable->nRow += nRow; - pRsp->numOfRows = nRow; - pRsp->affectedRows = nRow; + if (affectedRows) *affectedRows = nRow; - return code; - -_err: +_exit: return code; } int32_t tsdbGetNRowsInTbData(STbData *pTbData) { return pTbData->sl.size; } -void tsdbRefMemTable(SMemTable *pMemTable) { +int32_t tsdbRefMemTable(SMemTable *pMemTable, SQueryNode *pQNode) { + int32_t code = 0; + int32_t nRef = atomic_fetch_add_32(&pMemTable->nRef, 1); ASSERT(nRef > 0); + + vnodeBufPoolRegisterQuery(pMemTable->pPool, pQNode); + +_exit: + return code; } -void tsdbUnrefMemTable(SMemTable *pMemTable) { - int32_t nRef = atomic_sub_fetch_32(&pMemTable->nRef, 1); - if (nRef == 0) { - tsdbMemTableDestroy(pMemTable); +int32_t tsdbUnrefMemTable(SMemTable *pMemTable, SQueryNode *pNode, bool proactive) { + int32_t code = 0; + + if (pNode) { + vnodeBufPoolDeregisterQuery(pMemTable->pPool, pNode, proactive); + } + + if (atomic_sub_fetch_32(&pMemTable->nRef, 1) == 0) { + tsdbMemTableDestroy(pMemTable, proactive); } + + return code; } static FORCE_INLINE int32_t tbDataPCmprFn(const void *p1, const void *p2) { @@ -667,3 +812,29 @@ SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable) { _exit: return aTbDataP; } + +// int32_t tsdbRecycleMemTable(SMemTable *pMemTable) { +// int32_t code = 0; + +// SQueryNode *pNode = pMemTable->qList.pNext; +// while (1) { +// ASSERT(pNode != &pMemTable->qList); +// SQueryNode *pNextNode = pNode->pNext; + +// if (pNextNode == &pMemTable->qList) { +// code = (*pNode->reseek)(pNode->pQHandle); +// if (code) goto _exit; +// break; +// } else { +// code = (*pNode->reseek)(pNode->pQHandle); +// if (code) goto _exit; +// pNode = pMemTable->qList.pNext; +// ASSERT(pNode == pNextNode); +// } +// } + +// // NOTE: Take care here, pMemTable is destroyed + +// _exit: +// return code; +// } diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index a5ad18a22d3467a9232b35bb99cb76c10269117a..1f99444cf3cc655be656ce86a70ea22920c26fe8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -89,8 +89,8 @@ void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { pLoadInfo[i].blockIndex[0] = -1; pLoadInfo[i].blockIndex[1] = -1; - tBlockDataDestroy(&pLoadInfo[i].blockData[0], true); - tBlockDataDestroy(&pLoadInfo[i].blockData[1], true); + tBlockDataDestroy(&pLoadInfo[i].blockData[0]); + tBlockDataDestroy(&pLoadInfo[i].blockData[1]); taosArrayDestroy(pLoadInfo[i].aSttBlk); } @@ -567,7 +567,6 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead pMTree->pLoadInfo = pBlockLoadInfo; pMTree->destroyLoadInfo = destroyLoadInfo; - ASSERT(pMTree->pLoadInfo != NULL); for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file struct SLDataIter *pIter = NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index f71b5b6706ccb152f4dc221d01f4a1ea6e207c7a..8901f644598ec4ca5343f4a35a7b063bf39096fd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -88,7 +88,7 @@ _err: int tsdbClose(STsdb **pTsdb) { if (*pTsdb) { taosThreadRwlockWrlock(&(*pTsdb)->rwLock); - tsdbMemTableDestroy((*pTsdb)->mem); + tsdbMemTableDestroy((*pTsdb)->mem, true); (*pTsdb)->mem = NULL; taosThreadRwlockUnlock(&(*pTsdb)->rwLock); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 6634340d75a0061be6908b8927dee508efef44e8..d8c379f476943d64de5c93d9643b04ebfa832322 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -82,13 +82,13 @@ typedef struct SIOCostSummary { } SIOCostSummary; typedef struct SBlockLoadSuppInfo { - SArray* pColAgg; - SColumnDataAgg tsColAgg; - int16_t* colId; - int16_t* slotId; - int32_t numOfCols; - char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. - bool smaValid; // the sma on all queried columns are activated + SArray* pColAgg; + SColumnDataAgg tsColAgg; + int16_t* colId; + int16_t* slotId; + int32_t numOfCols; + char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. + bool smaValid; // the sma on all queried columns are activated } SBlockLoadSuppInfo; typedef struct SLastBlockReader { @@ -156,6 +156,9 @@ typedef struct SBlockInfoBuf { struct STsdbReader { STsdb* pTsdb; + SVersionRange verRange; + TdThreadMutex readerMutex; + bool suspended; uint64_t suid; int16_t order; bool freeBlock; @@ -168,15 +171,15 @@ struct STsdbReader { SBlockLoadSuppInfo suppInfo; STsdbReadSnap* pReadSnap; SIOCostSummary cost; - STSchema* pSchema; // the newest version schema - STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times - SDataFReader* pFileReader; // the file reader - SDelFReader* pDelFReader; // the del file reader - SArray* pDelIdx; // del file block index; - SVersionRange verRange; - SBlockInfoBuf blockInfoBuf; - int32_t step; - STsdbReader* innerReader[2]; + STSchema* pSchema; // the newest version schema + STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times + SDataFReader* pFileReader; // the file reader + SDelFReader* pDelFReader; // the del file reader + SArray* pDelIdx; // del file block index; + // SVersionRange verRange; + SBlockInfoBuf blockInfoBuf; + int32_t step; + STsdbReader* innerReader[2]; }; static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter); @@ -189,8 +192,8 @@ static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STabl SRowMerger* pMerger, SVersionRange* pVerRange); static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, STsdbReader* pReader); -static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, - STableBlockScanInfo* pScanInfo); +static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, + STableBlockScanInfo* pInfo); static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex); static void setComposedBlockFlag(STsdbReader* pReader, bool composed); @@ -198,9 +201,9 @@ static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* SVersionRange* pVerRange); static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, - STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow); + TSDBROW* pTSRow, STsdbReader* pReader, bool* freeTSRow); static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, - STsdbReader* pReader, STSRow** pTSRow); + STsdbReader* pReader, SRow** pTSRow); static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader); @@ -218,17 +221,18 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBl static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } -static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, int32_t numOfCols) { +static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, + int32_t numOfCols) { pSupInfo->smaValid = true; pSupInfo->numOfCols = numOfCols; - pSupInfo->colId = taosMemoryMalloc(numOfCols * (sizeof(int16_t)*2 + POINTER_BYTES)); + pSupInfo->colId = taosMemoryMalloc(numOfCols * (sizeof(int16_t) * 2 + POINTER_BYTES)); if (pSupInfo->colId == NULL) { taosMemoryFree(pSupInfo->colId); return TSDB_CODE_OUT_OF_MEMORY; } pSupInfo->slotId = (int16_t*)((char*)pSupInfo->colId + (sizeof(int16_t) * numOfCols)); - pSupInfo->buildBuf = (char**) ((char*)pSupInfo->slotId + (sizeof(int16_t) * numOfCols)); + pSupInfo->buildBuf = (char**)((char*)pSupInfo->slotId + (sizeof(int16_t) * numOfCols)); for (int32_t i = 0; i < numOfCols; ++i) { pSupInfo->colId[i] = pCols[i].colId; pSupInfo->slotId[i] = pSlotIdList[i]; @@ -243,15 +247,15 @@ static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pC return TSDB_CODE_SUCCESS; } -static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) { +static int32_t updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) { int32_t i = 0, j = 0; - while(i < pSchema->numOfCols && j < pSupInfo->numOfCols) { + while (i < pSchema->numOfCols && j < pSupInfo->numOfCols) { STColumn* pTCol = &pSchema->columns[i]; if (pTCol->colId == pSupInfo->colId[j]) { if (!IS_BSMA_ON(pTCol)) { pSupInfo->smaValid = false; - return; + return TSDB_CODE_SUCCESS; } i += 1; @@ -260,9 +264,11 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) // do nothing i += 1; } else { - ASSERT(0); + return TSDB_CODE_INVALID_PARA; } } + + return TSDB_CODE_SUCCESS; } static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) { @@ -309,7 +315,8 @@ static void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index) { } // NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model -static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, int32_t numOfTables) { +static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, + int32_t numOfTables) { // allocate buffer in order to load data blocks from file // todo use simple hash instead, optimize the memory consumption SHashObj* pTableMap = @@ -395,9 +402,7 @@ static void destroyAllBlockScanInfo(SHashObj* pTableMap) { taosHashCleanup(pTableMap); } -static bool isEmptyQueryTimeWindow(STimeWindow* pWindow) { - return pWindow->skey > pWindow->ekey; -} +static bool isEmptyQueryTimeWindow(STimeWindow* pWindow) { return pWindow->skey > pWindow->ekey; } // Update the query time window according to the data time to live(TTL) information, in order to avoid to return // the expired data to client, even it is queried already. @@ -579,7 +584,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd } if (VND_IS_TSMA(pVnode)) { - tsdbDebug("vgId:%d, tsma is selected to query", TD_VID(pVnode)); + tsdbDebug("vgId:%d, tsma is selected to query, %s", TD_VID(pVnode), idstr); } initReaderStatus(&pReader->status); @@ -594,7 +599,6 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->type = pCond->type; pReader->window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); pReader->blockInfoBuf.numPerBucket = 1000; // 1000 tables per bucket - ASSERT(pCond->numOfCols > 0); if (pReader->pResBlock == NULL) { pReader->freeBlock = true; @@ -605,6 +609,12 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd } } + if (pCond->numOfCols <= 0) { + tsdbError("vgId:%d, invalid column number %d in query cond, %s", TD_VID(pVnode), pCond->numOfCols, idstr); + code = TSDB_CODE_INVALID_PARA; + goto _end; + } + // todo refactor. limitOutputBufferSize(pCond, &pReader->capacity); @@ -626,6 +636,8 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd setColumnIdSlotList(&pReader->suppInfo, pCond->colList, pCond->pSlotList, pCond->numOfCols); + taosThreadMutexInit(&pReader->readerMutex, NULL); + *ppReader = pReader; return code; @@ -720,12 +732,25 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN sizeInDisk += pScanInfo->mapData.nData; + int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; + STimeWindow w = pReader->window; + if (ASCENDING_TRAVERSE(pReader->order)) { + w.skey = pScanInfo->lastKey + step; + } else { + w.ekey = pScanInfo->lastKey + step; + } + + if (isEmptyQueryTimeWindow(&w)) { + continue; + } + SDataBlk block = {0}; for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) { tGetDataBlk(pScanInfo->mapData.pData + pScanInfo->mapData.aOffset[j], &block); // 1. time range check - if (block.minKey.ts > pReader->window.ekey || block.maxKey.ts < pReader->window.skey) { + // if (block.minKey.ts > pReader->window.ekey || block.maxKey.ts < pReader->window.skey) { + if (block.minKey.ts > w.ekey || block.maxKey.ts < w.skey) { continue; } @@ -761,7 +786,6 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastFiles, sizeInDisk / 1000.0, el, pReader->idStr); - pReader->cost.numOfBlocks += total; pReader->cost.headFileLoadTime += el; @@ -794,8 +818,9 @@ static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_ } static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { - if (taosArrayGetSize(pBlockIter->blockList) == 0) { - ASSERT(pBlockIter->numOfBlocks == taosArrayGetSize(pBlockIter->blockList)); + size_t num = taosArrayGetSize(pBlockIter->blockList); + if (num == 0) { + ASSERT(pBlockIter->numOfBlocks == num); return NULL; } @@ -805,73 +830,6 @@ static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { static SDataBlk* getCurrentBlock(SDataBlockIter* pBlockIter) { return &pBlockIter->block; } -int32_t binarySearchForTs(char* pValue, int num, TSKEY key, int order) { - int32_t midPos = -1; - int32_t numOfRows; - - ASSERT(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - - TSKEY* keyList = (TSKEY*)pValue; - int32_t firstPos = 0; - int32_t lastPos = num - 1; - - if (order == TSDB_ORDER_DESC) { - // find the first position which is smaller than the key - while (1) { - if (key >= keyList[firstPos]) return firstPos; - if (key == keyList[lastPos]) return lastPos; - - if (key < keyList[lastPos]) { - lastPos += 1; - if (lastPos >= num) { - return -1; - } else { - return lastPos; - } - } - - numOfRows = lastPos - firstPos + 1; - midPos = (numOfRows >> 1) + firstPos; - - if (key < keyList[midPos]) { - firstPos = midPos + 1; - } else if (key > keyList[midPos]) { - lastPos = midPos - 1; - } else { - break; - } - } - - } else { - // find the first position which is bigger than the key - while (1) { - if (key <= keyList[firstPos]) return firstPos; - if (key == keyList[lastPos]) return lastPos; - - if (key > keyList[lastPos]) { - lastPos = lastPos + 1; - if (lastPos >= num) - return -1; - else - return lastPos; - } - - numOfRows = lastPos - firstPos + 1; - midPos = (numOfRows >> 1u) + firstPos; - - if (key < keyList[midPos]) { - lastPos = midPos - 1; - } else if (key > keyList[midPos]) { - firstPos = midPos + 1; - } else { - break; - } - } - } - - return midPos; -} - static int doBinarySearchKey(TSKEY* keyList, int num, int pos, TSKEY key, int order) { // start end position int s, e; @@ -961,7 +919,7 @@ static void copyPrimaryTsCol(const SBlockData* pBlockData, SFileBlockDumpInfo* p // a faster version of copy procedure. static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData, - int32_t dumpedRows, bool asc) { + int32_t dumpedRows, bool asc) { uint8_t* p = NULL; if (asc) { p = pData->pData + tDataTypes[pData->type].bytes * pDumpInfo->rowIndex; @@ -970,22 +928,21 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo p = pData->pData + tDataTypes[pData->type].bytes * startIndex; } - int32_t step = asc? 1:-1; + int32_t step = asc ? 1 : -1; - // make sure it is aligned to 8bit - ASSERT((((uint64_t)pColData->pData) & (0x8 - 1)) == 0); + // make sure it is aligned to 8bit, the allocated memory address is aligned to 256bit + // ASSERT((((uint64_t)pColData->pData) & (0x8 - 1)) == 0); // 1. copy data in a batch model memcpy(pColData->pData, p, dumpedRows * tDataTypes[pData->type].bytes); // 2. reverse the array list in case of descending order scan data block if (!asc) { - switch(pColData->info.type) { + switch (pColData->info.type) { case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - { + case TSDB_DATA_TYPE_UBIGINT: { int32_t mid = dumpedRows >> 1u; int64_t* pts = (int64_t*)pColData->pData; for (int32_t j = 0; j < mid; ++j) { @@ -999,7 +956,7 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_UTINYINT: { - int32_t mid = dumpedRows >> 1u; + int32_t mid = dumpedRows >> 1u; int8_t* pts = (int8_t*)pColData->pData; for (int32_t j = 0; j < mid; ++j) { int8_t t = pts[j]; @@ -1072,11 +1029,20 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn // pDumpInfo->rowIndex = 0; } else if (!asc && pReader->window.ekey >= pBlock->maxKey.ts) { // pDumpInfo->rowIndex = pBlock->nRow - 1; - } else { + } else { // find the appropriate the start position in current block, and set it to be the current rowIndex int32_t pos = asc ? pBlock->nRow - 1 : 0; int32_t order = asc ? TSDB_ORDER_DESC : TSDB_ORDER_ASC; int64_t key = asc ? pReader->window.skey : pReader->window.ekey; pDumpInfo->rowIndex = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, key, order); + + if (pDumpInfo->rowIndex < 0) { + tsdbError( + "%p failed to locate the start position in current block, global index:%d, table index:%d, brange:%" PRId64 + "-%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 " %s", + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->minVer, + pBlock->maxVer, pReader->idStr); + return TSDB_CODE_INVALID_PARA; + } } } @@ -1157,6 +1123,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn setBlockAllDumped(pDumpInfo, ts, pReader->order); } + pBlockScanInfo->lastKey = pDumpInfo->lastKey; + double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; pReader->cost.blockLoadTime += elapsedTime; @@ -1183,7 +1151,6 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - ASSERT(pBlockInfo != NULL); SDataBlk* pBlock = getCurrentBlock(pBlockIter); code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData); @@ -1221,8 +1188,6 @@ static void cleanupBlockOrderSupporter(SBlockOrderSupporter* pSup) { } static int32_t initBlockOrderSupporter(SBlockOrderSupporter* pSup, int32_t numOfTables) { - ASSERT(numOfTables >= 1); - pSup->numOfBlocksPerTable = taosMemoryCalloc(1, sizeof(int32_t) * numOfTables); pSup->indexPerTable = taosMemoryCalloc(1, sizeof(int32_t) * numOfTables); pSup->pDataBlockInfo = taosMemoryCalloc(1, POINTER_BYTES * numOfTables); @@ -1329,7 +1294,10 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte sup.numOfTables += 1; } - ASSERT(numOfBlocks == cnt); + if (numOfBlocks != cnt && sup.numOfTables != numOfTables) { + cleanupBlockOrderSupporter(&sup); + return TSDB_CODE_INVALID_PARA; + } // since there is only one table qualified, blocks are not sorted if (sup.numOfTables == 1) { @@ -1351,10 +1319,9 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables %s", pReader, cnt, sup.numOfTables, pReader->idStr); - ASSERT(cnt <= numOfBlocks && sup.numOfTables <= numOfTables); - SMultiwayMergeTreeInfo* pTree = NULL; - uint8_t ret = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, fileDataBlockOrderCompar); + + uint8_t ret = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, fileDataBlockOrderCompar); if (ret != TSDB_CODE_SUCCESS) { cleanupBlockOrderSupporter(&sup); return TSDB_CODE_OUT_OF_MEMORY; @@ -1432,8 +1399,6 @@ static bool getNeighborBlockOfSameTable(SFileDataBlockInfo* pBlockInfo, STableBl } static int32_t findFileBlockInfoIndex(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pFBlockInfo) { - ASSERT(pBlockIter != NULL && pFBlockInfo != NULL); - int32_t step = ASCENDING_TRAVERSE(pBlockIter->order) ? 1 : -1; int32_t index = pBlockIter->index; @@ -1741,7 +1706,7 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, SIterInfo* pIter, int64_t key, SLastBlockReader* pLastBlockReader) { SRowMerger merge = {0}; - STSRow* pTSRow = NULL; + SRow* pTSRow = NULL; SBlockData* pBlockData = &pReader->status.fileBlockData; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -1789,7 +1754,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (pReader->order == TSDB_ORDER_ASC) { if (minKey == key) { init = true; - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1799,10 +1764,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); } else { init = true; - int32_t code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1812,11 +1777,11 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { if (init) { - tRowMerge(&merge, pRow); + tsdbRowMerge(&merge, pRow); } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1830,7 +1795,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1844,10 +1809,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); } else { init = true; - int32_t code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1857,10 +1822,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { if (init) { - tRowMerge(&merge, &fRow); + tsdbRowMerge(&merge, &fRow); } else { init = true; - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1869,7 +1834,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } } - int32_t code = tRowMergerGetRow(&merge, &pTSRow); + int32_t code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1877,7 +1842,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); return TSDB_CODE_SUCCESS; } @@ -1887,7 +1852,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); - STSRow* pTSRow = NULL; + SRow* pTSRow = NULL; SRowMerger merge = {0}; TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", fRow.pBlockData, fRow.iRow, pLastBlockReader->uid, pReader->idStr); @@ -1898,16 +1863,16 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; } else { - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge, &pReader->verRange); - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1915,23 +1880,22 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); } } else { // not merge block data - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge, &pReader->verRange); - ASSERT(mergeBlockData); // merge with block data if ts == key if (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex]) { doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1939,7 +1903,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); } return TSDB_CODE_SUCCESS; @@ -1964,10 +1928,10 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader if (key < ts) { // imem, mem are all empty, file blocks (data blocks and last block) exist return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); } else if (key == ts) { - STSRow* pTSRow = NULL; + SRow* pTSRow = NULL; SRowMerger merge = {0}; - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1975,11 +1939,11 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge, &pReader->verRange); - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1987,10 +1951,9 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); return code; } else { - ASSERT(0); return TSDB_CODE_SUCCESS; } } else { // desc order @@ -2004,14 +1967,13 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { SRowMerger merge = {0}; - STSRow* pTSRow = NULL; + SRow* pTSRow = NULL; int32_t code = TSDB_CODE_SUCCESS; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SArray* pDelList = pBlockScanInfo->delSkyline; TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pDelList, pReader); TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader); - ASSERT(pRow != NULL && piRow != NULL); int64_t tsLast = INT64_MIN; if (hasDataInLastBlock(pLastBlockReader)) { @@ -2068,7 +2030,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { init = true; TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2079,10 +2041,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); } else { init = true; - code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2093,7 +2055,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == ik.ts) { if (init) { - tRowMerge(&merge, piRow); + tsdbRowMerge(&merge, piRow); } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); @@ -2101,7 +2063,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - code = tRowMergerInit(&merge, piRow, pSchema); + code = tsdbRowMergerInit(&merge, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2120,10 +2082,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - tRowMerge(&merge, pRow); + tsdbRowMerge(&merge, pRow); } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tRowMergerInit(&merge, pRow, pSchema); + code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2138,7 +2100,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tRowMergerInit(&merge, pRow, pSchema); + code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2152,11 +2114,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == ik.ts) { if (init) { - tRowMerge(&merge, piRow); + tsdbRowMerge(&merge, piRow); } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - code = tRowMergerInit(&merge, piRow, pSchema); + code = tsdbRowMergerInit(&merge, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2171,10 +2133,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); } else { init = true; - code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2185,7 +2147,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); if (!init) { - code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2193,7 +2155,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (merge.pTSchema == NULL) { return code; } - tRowMerge(&merge, &fRow); + tsdbRowMerge(&merge, &fRow); } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } @@ -2203,7 +2165,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2211,7 +2173,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); return code; } @@ -2224,9 +2186,11 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea TSDBKEY startKey = {0}; if (ASCENDING_TRAVERSE(pReader->order)) { - startKey = (TSDBKEY){.ts = pReader->window.skey, .version = pReader->verRange.minVer}; + // startKey = (TSDBKEY){.ts = pReader->window.skey, .version = pReader->verRange.minVer}; + startKey = (TSDBKEY){.ts = pBlockScanInfo->lastKey + 1, .version = pReader->verRange.minVer}; } else { - startKey = (TSDBKEY){.ts = pReader->window.ekey, .version = pReader->verRange.maxVer}; + // startKey = (TSDBKEY){.ts = pReader->window.ekey, .version = pReader->verRange.maxVer}; + startKey = (TSDBKEY){.ts = pBlockScanInfo->lastKey - 1, .version = pReader->verRange.maxVer}; } int32_t backward = (!ASCENDING_TRAVERSE(pReader->order)); @@ -2235,7 +2199,6 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea if (pReader->pReadSnap->pMem != NULL) { d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid); if (d != NULL) { - ASSERT(pBlockScanInfo->iter.iter == NULL); code = tsdbTbDataIterCreate(d, &startKey, backward, &pBlockScanInfo->iter.iter); if (code == TSDB_CODE_SUCCESS) { pBlockScanInfo->iter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iter.iter) != NULL); @@ -2349,10 +2312,9 @@ static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->mergeTree.pIter != NULL; } bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo) { - if (pBlockData->nRow > 0) { - ASSERT(pBlockData->nRow == pDumpInfo->totalRows); + if ((pBlockData->nRow > 0) && (pBlockData->nRow != pDumpInfo->totalRows)) { + return false; // this is an invalid result. } - return pBlockData->nRow > 0 && (!pDumpInfo->allDumped); } @@ -2365,16 +2327,16 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc } else { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - STSRow* pTSRow = NULL; + SRow* pTSRow = NULL; SRowMerger merge = {0}; - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2382,7 +2344,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); return TSDB_CODE_SUCCESS; } } @@ -2391,32 +2353,33 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + TSDBROW *pRow = NULL, *piRow = NULL; int64_t key = (pBlockData->nRow > 0 && (!pDumpInfo->allDumped)) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : INT64_MIN; - if (pBlockScanInfo->iter.hasVal && pBlockScanInfo->iiter.hasVal) { - return doMergeMultiLevelRows(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); - } else { - TSDBROW *pRow = NULL, *piRow = NULL; - if (pBlockScanInfo->iter.hasVal) { - pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); - } + if (pBlockScanInfo->iter.hasVal) { + pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); + } - if (pBlockScanInfo->iiter.hasVal) { - piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); - } + if (pBlockScanInfo->iiter.hasVal) { + piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); + } - // imem + file + last block - if (pBlockScanInfo->iiter.hasVal) { - return doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, key, pLastBlockReader); - } + // two levels of mem-table does contain the valid rows + if (pRow != NULL && piRow != NULL) { + return doMergeMultiLevelRows(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); + } - // mem + file + last block - if (pBlockScanInfo->iter.hasVal) { - return doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, key, pLastBlockReader); - } + // imem + file + last block + if (pBlockScanInfo->iiter.hasVal) { + return doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, key, pLastBlockReader); + } - // files data blocks + last block - return mergeFileBlockAndLastBlock(pReader, pLastBlockReader, key, pBlockScanInfo, pBlockData); + // mem + file + last block + if (pBlockScanInfo->iter.hasVal) { + return doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, key, pLastBlockReader); } + + // files data blocks + last block + return mergeFileBlockAndLastBlock(pReader, pLastBlockReader, key, pBlockScanInfo, pBlockData); } static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pBlockScanInfo, @@ -2459,6 +2422,19 @@ static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlock return code; } +static void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInfo* pBlockScanInfo) { + SSDataBlock* pResBlock = pReader->pResBlock; + + pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0; + pResBlock->info.dataLoad = 1; + blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]); + + setComposedBlockFlag(pReader, true); + + pReader->cost.composedBlocks += 1; + pReader->cost.buildComposedBlockTime += el; +} + static int32_t buildComposedDataBlock(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; @@ -2470,6 +2446,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { bool asc = ASCENDING_TRAVERSE(pReader->order); int64_t st = taosGetTimestampUs(); int32_t step = asc ? 1 : -1; + double el = 0; STableBlockScanInfo* pBlockScanInfo = NULL; if (pBlockInfo != NULL) { @@ -2487,7 +2464,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); // it is a clean block, load it directly - if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader) && + if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader) && pBlock->nRow <= pReader->capacity) { if (asc || ((!asc) && (!hasDataInLastBlock(pLastBlockReader)))) { copyBlockDataToSDataBlock(pReader, pBlockScanInfo); @@ -2507,7 +2484,8 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { while (1) { bool hasBlockData = false; { - while (pBlockData->nRow > 0) { // find the first qualified row in data block + while (pBlockData->nRow > 0 && + pBlockData->uid == pBlockScanInfo->uid) { // find the first qualified row in data block if (isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) { hasBlockData = true; break; @@ -2531,10 +2509,8 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } } - bool hasBlockLData = hasDataInLastBlock(pLastBlockReader); - // no data in last block and block, no need to proceed. - if ((hasBlockData == false) && (hasBlockLData == false)) { + if (hasBlockData == false) { break; } @@ -2553,15 +2529,8 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } _end: - pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0; - pResBlock->info.dataLoad = 1; - blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]); - - setComposedBlockFlag(pReader, true); - double el = (taosGetTimestampUs() - st) / 1000.0; - - pReader->cost.composedBlocks += 1; - pReader->cost.buildComposedBlockTime += el; + el = (taosGetTimestampUs() - st) / 1000.0; + updateComposedBlockInfo(pReader, el, pBlockScanInfo); if (pResBlock->info.rows > 0) { tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 @@ -2583,7 +2552,6 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* int32_t code = 0; SArray* pDelData = taosArrayInit(4, sizeof(SDelData)); - ASSERT(pReader->pReadSnap != NULL); SDelFile* pDelFile = pReader->pReadSnap->fs.pDelFile; if (pDelFile && taosArrayGetSize(pReader->pDelIdx) > 0) { @@ -2691,7 +2659,6 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { taosArrayDestroy(pIndexList); if (pReader->pReadSnap != NULL) { - SDelFile* pDelFile = pReader->pReadSnap->fs.pDelFile; if (pReader->pDelFReader == NULL && pDelFile != NULL) { int32_t code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pReader->pTsdb); @@ -2807,6 +2774,8 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { return code; } + SSDataBlock* pResBlock = pReader->pResBlock; + while (1) { // load the last data block of current table STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; @@ -2817,15 +2786,33 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { if (!hasNexTable) { return TSDB_CODE_SUCCESS; } + continue; } - code = doBuildDataBlock(pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; + int64_t st = taosGetTimestampUs(); + while (1) { + bool hasBlockLData = hasDataInLastBlock(pLastBlockReader); + + // no data in last block and block, no need to proceed. + if (hasBlockLData == false) { + break; + } + + buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader); + if (pResBlock->info.rows >= pReader->capacity) { + break; + } } - if (pReader->pResBlock->info.rows > 0) { + double el = (taosGetTimestampUs() - st) / 1000.0; + updateComposedBlockInfo(pReader, el, pScanInfo); + + if (pResBlock->info.rows > 0) { + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 + " rows:%d, elapsed time:%.2f ms %s", + pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, + pResBlock->info.rows, el, pReader->idStr); return TSDB_CODE_SUCCESS; } @@ -2868,8 +2855,37 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); if (pBlockInfo == NULL) { // build data block from last data file - ASSERT(pBlockIter->numOfBlocks == 0); - code = buildComposedDataBlock(pReader); + SBlockData* pBData = &pReader->status.fileBlockData; + tBlockDataReset(pBData); + + SSDataBlock* pResBlock = pReader->pResBlock; + tsdbDebug("load data in last block firstly, due to desc scan data, %s", pReader->idStr); + + int64_t st = taosGetTimestampUs(); + + while (1) { + bool hasBlockLData = hasDataInLastBlock(pLastBlockReader); + + // no data in last block and block, no need to proceed. + if (hasBlockLData == false) { + break; + } + + buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader); + if (pResBlock->info.rows >= pReader->capacity) { + break; + } + } + + double el = (taosGetTimestampUs() - st) / 1000.0; + updateComposedBlockInfo(pReader, el, pScanInfo); + + if (pResBlock->info.rows > 0) { + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 + " rows:%d, elapsed time:%.2f ms %s", + pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, + pResBlock->info.rows, el, pReader->idStr); + } } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) { code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { @@ -2888,10 +2904,38 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // only return the rows in last block int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); ASSERT(tsLast >= pBlock->maxKey.ts); - tBlockDataReset(&pReader->status.fileBlockData); + SBlockData* pBData = &pReader->status.fileBlockData; + tBlockDataReset(pBData); + + SSDataBlock* pResBlock = pReader->pResBlock; tsdbDebug("load data in last block firstly, due to desc scan data, %s", pReader->idStr); - code = buildComposedDataBlock(pReader); + + int64_t st = taosGetTimestampUs(); + + while (1) { + bool hasBlockLData = hasDataInLastBlock(pLastBlockReader); + + // no data in last block and block, no need to proceed. + if (hasBlockLData == false) { + break; + } + + buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader); + if (pResBlock->info.rows >= pReader->capacity) { + break; + } + } + + double el = (taosGetTimestampUs() - st) / 1000.0; + updateComposedBlockInfo(pReader, el, pScanInfo); + + if (pResBlock->info.rows > 0) { + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 + " rows:%d, elapsed time:%.2f ms %s", + pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, + pResBlock->info.rows, el, pReader->idStr); + } } else { // whole block is required, return it directly SDataBlockInfo* pInfo = &pReader->pResBlock->info; pInfo->rows = pBlock->nRow; @@ -2941,9 +2985,16 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { } // set the correct start position in case of the first/last file block, according to the query time window -void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { - SDataBlk* pBlock = getCurrentBlock(pBlockIter); - +static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { + int64_t lastKey = ASCENDING_TRAVERSE(pReader->order) ? INT64_MIN : INT64_MAX; + SDataBlk* pBlock = getCurrentBlock(pBlockIter); + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + if (pBlockInfo) { + STableBlockScanInfo* pScanInfo = taosHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + if (pScanInfo) { + lastKey = pScanInfo->lastKey; + } + } SReaderStatus* pStatus = &pReader->status; SFileBlockDumpInfo* pDumpInfo = &pStatus->fBlockDumpInfo; @@ -2951,6 +3002,7 @@ void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { pDumpInfo->totalRows = pBlock->nRow; pDumpInfo->allDumped = false; pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->order) ? 0 : pBlock->nRow - 1; + pDumpInfo->lastKey = lastKey; } static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBlockIter) { @@ -3042,7 +3094,12 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { } else { if (pReader->status.pCurrentFileset->nSttF > 0) { // data blocks in current file are exhausted, let's try the next file now - tBlockDataReset(&pReader->status.fileBlockData); + SBlockData* pBlockData = &pReader->status.fileBlockData; + if (pBlockData->uid != 0) { + tBlockDataClear(pBlockData); + } + + tBlockDataReset(pBlockData); resetDataBlockIterator(pBlockIter, pReader->order); goto _begin; } else { @@ -3150,7 +3207,8 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32 return false; } else if (pKey->ts == last->ts) { TSDBKEY* prev = taosArrayGet(pDelList, num - 2); - return (prev->version >= pKey->version && prev->version <= pVerRange->maxVer && prev->version >= pVerRange->minVer); + return (prev->version >= pKey->version && prev->version <= pVerRange->maxVer && + prev->version >= pVerRange->minVer); } } else { TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); @@ -3240,7 +3298,8 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p } TSDBROW* pRow = tsdbTbDataIterGet(pIter->iter); - TSDBKEY key = {.ts = pRow->pTSRow->ts, .version = pRow->version}; + TSDBKEY key = TSDBROW_KEY(pRow); + if (outOfTimeWindow(key.ts, &pReader->window)) { pIter->hasVal = false; return NULL; @@ -3293,12 +3352,16 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDe break; } - STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid); - if (pTSchema == NULL) { - return terrno; - } + if (pRow->type == TSDBROW_ROW_FMT) { + STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid); + if (pTSchema == NULL) { + return terrno; + } - tRowMergerAdd(pMerger, pRow, pTSchema); + tsdbRowMergerAdd(pMerger, pRow, pTSchema); + } else { // column format + tsdbRowMerge(pMerger, pRow); + } } return TSDB_CODE_SUCCESS; @@ -3313,7 +3376,7 @@ static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t rowInd } TSDBROW fRow = tsdbRowFromBlockData(pBlockData, rowIndex); - tRowMerge(pMerger, &fRow); + tsdbRowMerge(pMerger, &fRow); rowIndex += step; } @@ -3334,7 +3397,7 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn *state = CHECK_FILEBLOCK_QUIT; int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; - bool loadNeighbor = true; + bool loadNeighbor = true; int32_t code = loadNeighborIfOverlap(pFBlock, pScanInfo, pReader, &loadNeighbor); if (loadNeighbor && (code == TSDB_CODE_SUCCESS)) { @@ -3369,6 +3432,11 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc SFileDataBlockInfo* pFileBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); SDataBlk* pCurrentBlock = getCurrentBlock(&pReader->status.blockIter); + if (pFileBlockInfo == NULL) { + st = CHECK_FILEBLOCK_QUIT; + break; + } + checkForNeighborFileBlock(pReader, pScanInfo, pCurrentBlock, pFileBlockInfo, pMerger, key, &st); if (st == CHECK_FILEBLOCK_QUIT) { break; @@ -3385,7 +3453,7 @@ int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockSc int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 == ts) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tRowMerge(pMerger, &fRow1); + tsdbRowMerge(pMerger, &fRow1); } else { break; } @@ -3394,7 +3462,7 @@ int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockSc return TSDB_CODE_SUCCESS; } -int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, +int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, TSDBROW* pResRow, STsdbReader* pReader, bool* freeTSRow) { TSDBROW* pNextRow = NULL; TSDBROW current = *pRow; @@ -3403,19 +3471,19 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, pIter->hasVal = tsdbTbDataIterNext(pIter->iter); if (!pIter->hasVal) { - *pTSRow = current.pTSRow; + *pResRow = *pRow; *freeTSRow = false; return TSDB_CODE_SUCCESS; } else { // has next point in mem/imem pNextRow = getValidMemRow(pIter, pDelList, pReader); if (pNextRow == NULL) { - *pTSRow = current.pTSRow; + *pResRow = current; *freeTSRow = false; return TSDB_CODE_SUCCESS; } - if (current.pTSRow->ts != pNextRow->pTSRow->ts) { - *pTSRow = current.pTSRow; + if (TSDBROW_TS(¤t) != TSDBROW_TS(pNextRow)) { + *pResRow = current; *freeTSRow = false; return TSDB_CODE_SUCCESS; } @@ -3423,47 +3491,60 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, } SRowMerger merge = {0}; - - // get the correct schema for data in memory terrno = 0; - STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(¤t), pReader, uid); - if (pTSchema == NULL) { - return terrno; - } + int32_t code = 0; - if (pReader->pSchema == NULL) { - pReader->pSchema = pTSchema; - } + // start to merge duplicated rows + if (current.type == TSDBROW_ROW_FMT) { + // get the correct schema for data in memory + STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(¤t), pReader, uid); + if (pTSchema == NULL) { + return terrno; + } - int32_t code = tRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + if (pReader->pSchema == NULL) { + pReader->pSchema = pTSchema; + } - STSchema* pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid); - if (pTSchema1 == NULL) { - return terrno; - } + code = tsdbRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - tRowMergerAdd(&merge, pNextRow, pTSchema1); + STSchema* pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid); + if (pTSchema1 == NULL) { + return terrno; + } - code = doMergeRowsInBuf(pIter, uid, current.pTSRow->ts, pDelList, &merge, pReader); + tsdbRowMergerAdd(&merge, pNextRow, pTSchema1); + } else { // let's merge rows in file block + code = tsdbRowMergerInit(&merge, ¤t, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + tsdbRowMerge(&merge, pNextRow); + } + + code = doMergeRowsInBuf(pIter, uid, TSDBROW_TS(¤t), pDelList, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } - code = tRowMergerGetRow(&merge, pTSRow); + code = tsdbRowMergerGetRow(&merge, &pResRow->pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } - tRowMergerClear(&merge); + pResRow->type = TSDBROW_ROW_FMT; + tsdbRowMergerClear(&merge); *freeTSRow = true; + return TSDB_CODE_SUCCESS; } int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, - STSRow** pTSRow) { + SRow** pTSRow) { SRowMerger merge = {0}; TSDBKEY k = TSDBROW_KEY(pRow); @@ -3472,7 +3553,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, piRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3483,7 +3564,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p return code; } - tRowMerge(&merge, pRow); + tsdbRowMerge(&merge, pRow); code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { @@ -3493,7 +3574,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS || merge.pTSchema == NULL) { return code; } @@ -3504,7 +3585,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p return code; } - tRowMerge(&merge, piRow); + tsdbRowMerge(&merge, piRow); code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { @@ -3512,13 +3593,12 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p } } - int32_t code = tRowMergerGetRow(&merge, pTSRow); - tRowMergerClear(&merge); - + int32_t code = tsdbRowMergerGetRow(&merge, pTSRow); + tsdbRowMergerClear(&merge); return code; } -int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STSRow** pTSRow, int64_t endKey, +int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, TSDBROW* pResRow, int64_t endKey, bool* freeTSRow) { TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); @@ -3548,13 +3628,14 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR int32_t code = TSDB_CODE_SUCCESS; if (ik.ts != k.ts) { if (((ik.ts < k.ts) && asc) || ((ik.ts > k.ts) && (!asc))) { // ik.ts < k.ts - code = doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader, freeTSRow); + code = doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pResRow, pReader, freeTSRow); } else if (((k.ts < ik.ts) && asc) || ((k.ts > ik.ts) && (!asc))) { - code = doMergeMemTableMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, freeTSRow); + code = doMergeMemTableMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, pResRow, pReader, freeTSRow); } } else { // ik.ts == k.ts *freeTSRow = true; - code = doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, pTSRow); + pResRow->type = TSDBROW_ROW_FMT; + code = doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, &pResRow->pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3564,29 +3645,30 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR } if (pBlockScanInfo->iter.hasVal && pRow != NULL) { - return doMergeMemTableMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, + return doMergeMemTableMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pResRow, pReader, freeTSRow); } if (pBlockScanInfo->iiter.hasVal && piRow != NULL) { - return doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader, freeTSRow); + return doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pResRow, pReader, freeTSRow); } return TSDB_CODE_SUCCESS; } -int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, - STableBlockScanInfo* pScanInfo) { +int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, STableBlockScanInfo* pScanInfo) { int32_t outputRowIndex = pBlock->info.rows; int64_t uid = pScanInfo->uid; + int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); + SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; STSchema* pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid); SColVal colVal = {0}; int32_t i = 0, j = 0; - if (pSupInfo->colId[i]== PRIMARYKEY_TIMESTAMP_COL_ID) { + if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts; i += 1; @@ -3598,7 +3680,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* if (colId == pSchema->columns[j].colId) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); - tTSRowGetVal(pTSRow, pSchema, j, &colVal); + tRowGet(pTSRow, pSchema, j, &colVal); doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo); i += 1; j += 1; @@ -3631,7 +3713,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S int32_t outputRowIndex = pResBlock->info.rows; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - if (pReader->suppInfo.colId[i]== PRIMARYKEY_TIMESTAMP_COL_ID) { + if (pReader->suppInfo.colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; i += 1; @@ -3677,17 +3759,22 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e SSDataBlock* pBlock = pReader->pResBlock; do { - STSRow* pTSRow = NULL; + // SRow* pTSRow = NULL; + TSDBROW row = {.type = -1}; bool freeTSRow = false; - tsdbGetNextRowInMem(pBlockScanInfo, pReader, &pTSRow, endKey, &freeTSRow); - if (pTSRow == NULL) { + tsdbGetNextRowInMem(pBlockScanInfo, pReader, &row, endKey, &freeTSRow); + if (row.type == -1) { break; } - doAppendRowFromTSRow(pBlock, pReader, pTSRow, pBlockScanInfo); + if (row.type == TSDBROW_ROW_FMT) { + doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo); - if (freeTSRow) { - taosMemoryFree(pTSRow); + if (freeTSRow) { + taosMemoryFree(row.pTSRow); + } + } else { + doAppendRowFromFileBlock(pBlock, pReader, row.pBlockData, row.iRow); } // no data in buffer, return immediately @@ -3837,53 +3924,21 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL } if (pReader->pSchema != NULL) { - updateBlockSMAInfo(pReader->pSchema, &pReader->suppInfo); + code = updateBlockSMAInfo(pReader->pSchema, &pReader->suppInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } } STsdbReader* p = (pReader->innerReader[0] != NULL) ? pReader->innerReader[0] : pReader; pReader->status.pTableMap = createDataBlockScanInfo(p, &pReader->blockInfoBuf, pTableList, numOfTables); if (pReader->status.pTableMap == NULL) { - tsdbReaderClose(p); *ppReader = NULL; - code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - if (numOfTables > 0) { - code = tsdbTakeReadSnap(pReader->pTsdb, &pReader->pReadSnap, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } - - if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) { - code = doOpenReaderImpl(pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } else { - STsdbReader* pPrevReader = pReader->innerReader[0]; - STsdbReader* pNextReader = pReader->innerReader[1]; - - // we need only one row - pPrevReader->capacity = 1; - pPrevReader->status.pTableMap = pReader->status.pTableMap; - pPrevReader->pSchema = pReader->pSchema; - pPrevReader->pMemSchema = pReader->pMemSchema; - pPrevReader->pReadSnap = pReader->pReadSnap; - - pNextReader->capacity = 1; - pNextReader->status.pTableMap = pReader->status.pTableMap; - pNextReader->pSchema = pReader->pSchema; - pNextReader->pMemSchema = pReader->pMemSchema; - pNextReader->pReadSnap = pReader->pReadSnap; - - code = doOpenReaderImpl(pPrevReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - } + pReader->suspended = true; tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr); return code; @@ -3934,7 +3989,7 @@ void tsdbReaderClose(STsdbReader* pReader) { } taosMemoryFree(pSupInfo->colId); - tBlockDataDestroy(&pReader->status.fileBlockData, true); + tBlockDataDestroy(&pReader->status.fileBlockData); cleanupDataBlockIterator(&pReader->status.blockIter); size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); @@ -3956,7 +4011,10 @@ void tsdbReaderClose(STsdbReader* pReader) { pReader->pDelIdx = NULL; } - tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap, pReader->idStr); + qTrace("tsdb/reader: %p, untake snapshot", pReader); + tsdbUntakeReadSnap(pReader, pReader->pReadSnap, true); + + taosThreadMutexDestroy(&pReader->readerMutex); taosMemoryFree(pReader->status.uidCheckInfo.tableUidList); SIOCostSummary* pCost = &pReader->cost; @@ -3988,16 +4046,180 @@ void tsdbReaderClose(STsdbReader* pReader) { if (pReader->pMemSchema != pReader->pSchema) { taosMemoryFree(pReader->pMemSchema); } + taosMemoryFreeClear(pReader); } +int32_t tsdbReaderSuspend(STsdbReader* pReader) { + int32_t code = 0; + + // save reader's base state & reset top state to be reconstructed from base state + SReaderStatus* pStatus = &pReader->status; + STableBlockScanInfo* pBlockScanInfo = NULL; + + if (pStatus->loadFromFile) { + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + if (pBlockInfo != NULL) { + pBlockScanInfo = + *(STableBlockScanInfo**)taosHashGet(pStatus->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + if (pBlockScanInfo == NULL) { + code = TSDB_CODE_INVALID_PARA; + tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", pBlockInfo->uid, + taosHashGetSize(pReader->status.pTableMap), pReader->idStr); + goto _err; + } + } else { + pBlockScanInfo = *pStatus->pTableIter; + } + + tsdbDataFReaderClose(&pReader->pFileReader); + + // resetDataBlockScanInfo excluding lastKey + STableBlockScanInfo** p = NULL; + + while ((p = taosHashIterate(pStatus->pTableMap, p)) != NULL) { + STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; + + pInfo->iterInit = false; + pInfo->iter.hasVal = false; + pInfo->iiter.hasVal = false; + + if (pInfo->iter.iter != NULL) { + pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter); + } + + if (pInfo->iiter.iter != NULL) { + pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter); + } + + pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); + // pInfo->lastKey = ts; + } + } else { + pBlockScanInfo = pStatus->pTableIter == NULL ? NULL : *pStatus->pTableIter; + if (pBlockScanInfo) { + // save lastKey to restore memory iterator + STimeWindow w = pReader->pResBlock->info.window; + pBlockScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->order) ? w.ekey : w.skey; + + // reset current current table's data block scan info, + pBlockScanInfo->iterInit = false; + // pBlockScanInfo->iiter.hasVal = false; + if (pBlockScanInfo->iter.iter != NULL) { + pBlockScanInfo->iter.iter = tsdbTbDataIterDestroy(pBlockScanInfo->iter.iter); + } + + if (pBlockScanInfo->iiter.iter != NULL) { + pBlockScanInfo->iiter.iter = tsdbTbDataIterDestroy(pBlockScanInfo->iiter.iter); + } + + pBlockScanInfo->pBlockList = taosArrayDestroy(pBlockScanInfo->pBlockList); + tMapDataClear(&pBlockScanInfo->mapData); + // TODO: keep skyline for reuse + pBlockScanInfo->delSkyline = taosArrayDestroy(pBlockScanInfo->delSkyline); + } + } + + tsdbUntakeReadSnap(pReader, pReader->pReadSnap, false); + pReader->pReadSnap = NULL; + + pReader->suspended = true; + + tsdbDebug("reader: %p suspended uid %" PRIu64 " in this query %s", pReader, pBlockScanInfo ? pBlockScanInfo->uid : 0, + pReader->idStr); + return code; + +_err: + tsdbError("failed to suspend data reader, code:%s %s", tstrerror(code), pReader->idStr); + return code; +} + +static int32_t tsdbSetQueryReseek(void* pQHandle) { + int32_t code = 0; + STsdbReader* pReader = pQHandle; + + code = taosThreadMutexTryLock(&pReader->readerMutex); + if (code == 0) { + if (pReader->suspended) { + taosThreadMutexUnlock(&pReader->readerMutex); + return code; + } + + tsdbReaderSuspend(pReader); + + taosThreadMutexUnlock(&pReader->readerMutex); + + return code; + } else if (code == EBUSY) { + return TSDB_CODE_VND_QUERY_BUSY; + } else { + terrno = TAOS_SYSTEM_ERROR(code); + return TSDB_CODE_FAILED; + } +} + +int32_t tsdbReaderResume(STsdbReader* pReader) { + int32_t code = 0; + + STableBlockScanInfo** pBlockScanInfo = pReader->status.pTableIter; + + // restore reader's state + // task snapshot + int32_t numOfTables = taosHashGetSize(pReader->status.pTableMap); + if (numOfTables > 0) { + qTrace("tsdb/reader: %p, take snapshot", pReader); + code = tsdbTakeReadSnap(pReader, tsdbSetQueryReseek, &pReader->pReadSnap); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + + if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) { + code = doOpenReaderImpl(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else { + STsdbReader* pPrevReader = pReader->innerReader[0]; + STsdbReader* pNextReader = pReader->innerReader[1]; + + // we need only one row + pPrevReader->capacity = 1; + pPrevReader->status.pTableMap = pReader->status.pTableMap; + pPrevReader->pSchema = pReader->pSchema; + pPrevReader->pMemSchema = pReader->pMemSchema; + pPrevReader->pReadSnap = pReader->pReadSnap; + + pNextReader->capacity = 1; + pNextReader->status.pTableMap = pReader->status.pTableMap; + pNextReader->pSchema = pReader->pSchema; + pNextReader->pMemSchema = pReader->pMemSchema; + pNextReader->pReadSnap = pReader->pReadSnap; + + code = doOpenReaderImpl(pPrevReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + } + + pReader->suspended = false; + + tsdbDebug("reader: %p resumed uid %" PRIu64 ", numOfTable:%" PRId32 ", in this query %s", pReader, + pBlockScanInfo ? (*pBlockScanInfo)->uid : 0, numOfTables, pReader->idStr); + return code; + +_err: + tsdbError("failed to resume data reader, code:%s %s", tstrerror(code), pReader->idStr); + return code; +} + static bool doTsdbNextDataBlock(STsdbReader* pReader) { // cleanup the data that belongs to the previous data block SSDataBlock* pBlock = pReader->pResBlock; blockDataCleanup(pBlock); SReaderStatus* pStatus = &pReader->status; - if (taosHashGetSize(pStatus->pTableMap) == 0){ + if (taosHashGetSize(pStatus->pTableMap) == 0) { return false; } @@ -4024,10 +4246,24 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { return false; } + SReaderStatus* pStatus = &pReader->status; + + qTrace("tsdb/read: %p, take read mutex", pReader); + taosThreadMutexLock(&pReader->readerMutex); + if (pReader->suspended) { + tsdbReaderResume(pReader); + } + if (pReader->innerReader[0] != NULL && pReader->step == 0) { bool ret = doTsdbNextDataBlock(pReader->innerReader[0]); pReader->step = EXTERNAL_ROWS_PREV; if (ret) { + pStatus = &pReader->innerReader[0]->status; + if (pStatus->composedDataBlock) { + qTrace("tsdb/read: %p, unlock read mutex", pReader); + taosThreadMutexUnlock(&pReader->readerMutex); + } + return ret; } } @@ -4046,6 +4282,11 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { bool ret = doTsdbNextDataBlock(pReader); if (ret) { + if (pStatus->composedDataBlock) { + qTrace("tsdb/read: %p, unlock read mutex", pReader); + taosThreadMutexUnlock(&pReader->readerMutex); + } + return ret; } @@ -4060,10 +4301,19 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { bool ret1 = doTsdbNextDataBlock(pReader->innerReader[1]); pReader->step = EXTERNAL_ROWS_NEXT; if (ret1) { + pStatus = &pReader->innerReader[1]->status; + if (pStatus->composedDataBlock) { + qTrace("tsdb/read: %p, unlock read mutex", pReader); + taosThreadMutexUnlock(&pReader->readerMutex); + } + return ret1; } } + qTrace("tsdb/read: %p, unlock read mutex", pReader); + taosThreadMutexUnlock(&pReader->readerMutex); + return false; } @@ -4087,12 +4337,10 @@ void tsdbRetrieveDataBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64 } } - -static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, - SColumnDataAgg* pTsAgg) { +static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) { // do fill all null column value SMA info int32_t i = 0, j = 0; - int32_t size = (int32_t) taosArrayGetSize(pSup->pColAgg); + int32_t size = (int32_t)taosArrayGetSize(pSup->pColAgg); taosArrayInsert(pSup->pColAgg, 0, pTsAgg); while (j < numOfCols && i < size) { @@ -4105,7 +4353,7 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ } else if (pSup->colId[j] < pAgg->colId) { if (pSup->colId[j] != PRIMARYKEY_TIMESTAMP_COL_ID) { SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows}; - taosArrayInsert(pSup->pColAgg, i ,&nullColAgg); + taosArrayInsert(pSup->pColAgg, i, &nullColAgg); } j += 1; } @@ -4113,25 +4361,27 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ } int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave) { + SColumnDataAgg*** pBlockSMA = &pDataBlock->pBlockAgg; + int32_t code = 0; - SColumnDataAgg ***pBlockSMA = &pDataBlock->pBlockAgg; *allHave = false; + *pBlockSMA = NULL; if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { - *pBlockSMA = NULL; return TSDB_CODE_SUCCESS; } // there is no statistics data for composed block if (pReader->status.composedDataBlock || (!pReader->suppInfo.smaValid)) { - *pBlockSMA = NULL; return TSDB_CODE_SUCCESS; } SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter); SBlockLoadSuppInfo* pSup = &pReader->suppInfo; - ASSERT(pReader->pResBlock->info.id.uid == pFBlock->uid); + if (pReader->pResBlock->info.id.uid != pFBlock->uid) { + return TSDB_CODE_SUCCESS; + } SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); if (tDataBlkHasSma(pBlock)) { @@ -4159,13 +4409,10 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, // update the number of NULL data rows size_t numOfCols = pSup->numOfCols; - int32_t i = 0, j = 0; - size_t size = taosArrayGetSize(pSup->pColAgg); - // ensure capacity - if(pDataBlock->pDataBlock) { - size_t colsNum = taosArrayGetSize(pDataBlock->pDataBlock); - taosArrayEnsureCap(pSup->pColAgg, colsNum); + if (pDataBlock->pDataBlock) { + size_t colsNum = taosArrayGetSize(pDataBlock->pDataBlock); + taosArrayEnsureCap(pSup->pColAgg, colsNum); } SSDataBlock* pResBlock = pReader->pResBlock; @@ -4176,8 +4423,9 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, // do fill all null column value SMA info doFillNullColSMA(pSup, pBlock->nRow, numOfCols, pTsAgg); + size_t size = taosArrayGetSize(pSup->pColAgg); - i = 0, j = 0; + int32_t i = 0, j = 0; while (j < numOfCols && i < size) { SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i); if (pAgg->colId == pSup->colId[j]) { @@ -4187,7 +4435,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, } else if (pAgg->colId < pSup->colId[j]) { i += 1; } else if (pSup->colId[j] < pAgg->colId) { - ASSERT(pSup->colId[j] == PRIMARYKEY_TIMESTAMP_COL_ID); + // ASSERT(pSup->colId[j] == PRIMARYKEY_TIMESTAMP_COL_ID); pResBlock->pBlockAgg[pSup->slotId[j]] = &pSup->tsColAgg; j += 1; } @@ -4201,12 +4449,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, } static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { - SReaderStatus* pStatus = &pReader->status; - - if (pStatus->composedDataBlock) { - return pReader->pResBlock; - } - + SReaderStatus* pStatus = &pReader->status; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter); STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)taosHashGet(pStatus->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); @@ -4219,7 +4462,7 @@ static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { - tBlockDataDestroy(&pStatus->fileBlockData, 1); + tBlockDataDestroy(&pStatus->fileBlockData); terrno = code; return NULL; } @@ -4228,20 +4471,51 @@ static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { return pReader->pResBlock; } +void tsdbReleaseDataBlock(STsdbReader* pReader) { + // SReaderStatus* pStatus = &pReader->status; + // if (!pStatus->composedDataBlock) { + taosThreadMutexUnlock(&pReader->readerMutex); + //} +} + SSDataBlock* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { + STsdbReader* pTReader = pReader; if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { if (pReader->step == EXTERNAL_ROWS_PREV) { - return doRetrieveDataBlock(pReader->innerReader[0]); + pTReader = pReader->innerReader[0]; } else if (pReader->step == EXTERNAL_ROWS_NEXT) { - return doRetrieveDataBlock(pReader->innerReader[1]); + pTReader = pReader->innerReader[1]; } } - return doRetrieveDataBlock(pReader); + SReaderStatus* pStatus = &pTReader->status; + if (pStatus->composedDataBlock) { + return pTReader->pResBlock; + } + + SSDataBlock* ret = doRetrieveDataBlock(pTReader); + + qTrace("tsdb/read-retrieve: %p, unlock read mutex", pReader); + taosThreadMutexUnlock(&pReader->readerMutex); + + return ret; } int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { + SReaderStatus* pStatus = &pReader->status; + + qTrace("tsdb/reader-reset: %p, take read mutex", pReader); + taosThreadMutexLock(&pReader->readerMutex); + + if (pReader->suspended) { + tsdbReaderResume(pReader); + } + if (isEmptyQueryTimeWindow(&pReader->window) || pReader->pReadSnap == NULL) { + tsdbDebug("tsdb reader reset return %p", pReader->pReadSnap); + + taosThreadMutexUnlock(&pReader->readerMutex); + return TSDB_CODE_SUCCESS; } @@ -4277,6 +4551,9 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { if (code != TSDB_CODE_SUCCESS) { tsdbError("%p reset reader failed, numOfTables:%d, query range:%" PRId64 " - %" PRId64 " in query %s", pReader, numOfTables, pReader->window.skey, pReader->window.ekey, pReader->idStr); + + taosThreadMutexUnlock(&pReader->readerMutex); + return code; } } @@ -4286,6 +4563,8 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { pReader, pReader->suid, numOfTables, pCond->twindows.skey, pReader->window.skey, pReader->window.ekey, pReader->idStr); + taosThreadMutexUnlock(&pReader->readerMutex); + return code; } @@ -4297,6 +4576,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa int32_t code = TSDB_CODE_SUCCESS; pTableBlockInfo->totalSize = 0; pTableBlockInfo->totalRows = 0; + pTableBlockInfo->numOfVgroups = 1; // find the start data block in file SReaderStatus* pStatus = &pReader->status; @@ -4368,13 +4648,18 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { int64_t rows = 0; SReaderStatus* pStatus = &pReader->status; + taosThreadMutexLock(&pReader->readerMutex); + if (pReader->suspended) { + tsdbReaderResume(pReader); + } + pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, NULL); while (pStatus->pTableIter != NULL) { STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; STbData* d = NULL; - if (pReader->pTsdb->mem != NULL) { + if (pReader->pReadSnap->pMem != NULL) { d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid); if (d != NULL) { rows += tsdbGetNRowsInTbData(d); @@ -4382,7 +4667,7 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { } STbData* di = NULL; - if (pReader->pTsdb->imem != NULL) { + if (pReader->pReadSnap->pIMem != NULL) { di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->suid, pBlockScanInfo->uid); if (di != NULL) { rows += tsdbGetNRowsInTbData(di); @@ -4393,6 +4678,8 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); } + taosThreadMutexUnlock(&pReader->readerMutex); + return rows; } @@ -4420,9 +4707,12 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 return terrno; } sversion = mr.me.stbEntry.schemaRow.version; - } else { - ASSERT(mr.me.type == TSDB_NORMAL_TABLE); + } else if (mr.me.type == TSDB_NORMAL_TABLE) { sversion = mr.me.ntbEntry.schemaRow.version; + } else { + terrno = TSDB_CODE_INVALID_PARA; + metaReaderClear(&mr); + return terrno; } metaReaderClear(&mr); @@ -4431,66 +4721,92 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 return TSDB_CODE_SUCCESS; } -int32_t tsdbTakeReadSnap(STsdb* pTsdb, STsdbReadSnap** ppSnap, const char* idStr) { - int32_t code = 0; +int32_t tsdbTakeReadSnap(STsdbReader* pReader, _query_reseek_func_t reseek, STsdbReadSnap** ppSnap) { + int32_t code = 0; + STsdb* pTsdb = pReader->pTsdb; + SVersionRange* pRange = &pReader->verRange; // alloc - *ppSnap = (STsdbReadSnap*)taosMemoryCalloc(1, sizeof(STsdbReadSnap)); - if (*ppSnap == NULL) { + STsdbReadSnap* pSnap = (STsdbReadSnap*)taosMemoryCalloc(1, sizeof(*pSnap)); + if (pSnap == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } // lock - code = taosThreadRwlockRdlock(&pTsdb->rwLock); - if (code) { - code = TAOS_SYSTEM_ERROR(code); - goto _exit; - } + taosThreadRwlockRdlock(&pTsdb->rwLock); // take snapshot - (*ppSnap)->pMem = pTsdb->mem; - (*ppSnap)->pIMem = pTsdb->imem; + if (pTsdb->mem && (pRange->minVer <= pTsdb->mem->maxVer && pRange->maxVer >= pTsdb->mem->minVer)) { + pSnap->pMem = pTsdb->mem; + pSnap->pNode = taosMemoryMalloc(sizeof(*pSnap->pNode)); + if (pSnap->pNode == NULL) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + pSnap->pNode->pQHandle = pReader; + pSnap->pNode->reseek = reseek; - if ((*ppSnap)->pMem) { - tsdbRefMemTable((*ppSnap)->pMem); + tsdbRefMemTable(pTsdb->mem, pSnap->pNode); } - if ((*ppSnap)->pIMem) { - tsdbRefMemTable((*ppSnap)->pIMem); + if (pTsdb->imem && (pRange->minVer <= pTsdb->imem->maxVer && pRange->maxVer >= pTsdb->imem->minVer)) { + pSnap->pIMem = pTsdb->imem; + pSnap->pINode = taosMemoryMalloc(sizeof(*pSnap->pINode)); + if (pSnap->pINode == NULL) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + pSnap->pINode->pQHandle = pReader; + pSnap->pINode->reseek = reseek; + + tsdbRefMemTable(pTsdb->imem, pSnap->pINode); } // fs - code = tsdbFSRef(pTsdb, &(*ppSnap)->fs); + code = tsdbFSRef(pTsdb, &pSnap->fs); if (code) { taosThreadRwlockUnlock(&pTsdb->rwLock); goto _exit; } // unlock - code = taosThreadRwlockUnlock(&pTsdb->rwLock); - if (code) { - code = TAOS_SYSTEM_ERROR(code); - goto _exit; - } + taosThreadRwlockUnlock(&pTsdb->rwLock); + + tsdbTrace("vgId:%d, take read snapshot", TD_VID(pTsdb->pVnode)); - tsdbTrace("vgId:%d, take read snapshot, %s", TD_VID(pTsdb->pVnode), idStr); _exit: + if (code) { + *ppSnap = NULL; + if (pSnap) { + if (pSnap->pNode) taosMemoryFree(pSnap->pNode); + if (pSnap->pINode) taosMemoryFree(pSnap->pINode); + taosMemoryFree(pSnap); + } + } else { + *ppSnap = pSnap; + } return code; } -void tsdbUntakeReadSnap(STsdb* pTsdb, STsdbReadSnap* pSnap, const char* idStr) { +void tsdbUntakeReadSnap(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proactive) { + STsdb* pTsdb = pReader->pTsdb; + if (pSnap) { if (pSnap->pMem) { - tsdbUnrefMemTable(pSnap->pMem); + tsdbUnrefMemTable(pSnap->pMem, pSnap->pNode, proactive); } if (pSnap->pIMem) { - tsdbUnrefMemTable(pSnap->pIMem); + tsdbUnrefMemTable(pSnap->pIMem, pSnap->pINode, proactive); } tsdbFSUnref(pTsdb, &pSnap->fs); + if (pSnap->pNode) taosMemoryFree(pSnap->pNode); + if (pSnap->pINode) taosMemoryFree(pSnap->pINode); taosMemoryFree(pSnap); } - tsdbTrace("vgId:%d, untake read snapshot, %s", TD_VID(pTsdb->pVnode), idStr); + tsdbTrace("vgId:%d, untake read snapshot", TD_VID(pTsdb->pVnode)); } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 99acc249926f001fb795f5bc0302a1d1cb3e1139..5c7bfee5a7bc07bdf1e91015e4d3d36e5873d7ec 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -76,6 +76,7 @@ static int32_t tFDataIterCmprFn(const SRBTreeNode* pNode1, const SRBTreeNode* pN static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { int32_t code = 0; + int32_t lino = 0; SDFileSet dFileSet = {.fid = pReader->fid}; SDFileSet* pSet = taosArraySearch(pReader->fs.aDFileSet, &dFileSet, tDFileSetCmprFn, TD_GT); @@ -83,7 +84,7 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { pReader->fid = pSet->fid; code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, pSet); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pReader->pIter = NULL; tRBTreeCreate(&pReader->rbt, tFDataIterCmprFn); @@ -93,13 +94,13 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { pIter->type = SNAP_DATA_FILE_ITER; code = tsdbReadBlockIdx(pReader->pDataFReader, pIter->aBlockIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); for (pIter->iBlockIdx = 0; pIter->iBlockIdx < taosArrayGetSize(pIter->aBlockIdx); pIter->iBlockIdx++) { pIter->pBlockIdx = (SBlockIdx*)taosArrayGet(pIter->aBlockIdx, pIter->iBlockIdx); code = tsdbReadDataBlk(pReader->pDataFReader, pIter->pBlockIdx, &pIter->mBlock); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); for (pIter->iBlock = 0; pIter->iBlock < pIter->mBlock.nItem; pIter->iBlock++) { SDataBlk dataBlk; @@ -108,7 +109,7 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { if (dataBlk.minVer > pReader->ever || dataBlk.maxVer < pReader->sver) continue; code = tsdbReadDataBlockEx(pReader->pDataFReader, &dataBlk, &pIter->bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); ASSERT(pIter->pBlockIdx->suid == pIter->bData.suid); ASSERT(pIter->pBlockIdx->uid == pIter->bData.uid); @@ -139,7 +140,7 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { pIter->iStt = iStt; code = tsdbReadSttBlk(pReader->pDataFReader, iStt, pIter->aSttBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); for (pIter->iSttBlk = 0; pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk); pIter->iSttBlk++) { SSttBlk* pSttBlk = (SSttBlk*)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk); @@ -148,7 +149,7 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { if (pSttBlk->maxVer < pReader->sver) continue; code = tsdbReadSttBlockEx(pReader->pDataFReader, iStt, pSttBlk, &pIter->bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); for (pIter->iRow = 0; pIter->iRow < pIter->bData.nRow; pIter->iRow++) { int64_t rowVer = pIter->bData.aVersion[pIter->iRow]; @@ -169,13 +170,13 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { pIter++; } - tsdbInfo("vgId:%d, vnode snapshot tsdb open data file to read for %s, fid:%d", TD_VID(pReader->pTsdb->pVnode), - pReader->pTsdb->path, pReader->fid); - return code; - -_err: - tsdbError("vgId:%d, vnode snapshot tsdb snap read open file failed since %s", TD_VID(pReader->pTsdb->pVnode), - tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d, %s failed since %s", TD_VID(pReader->pTsdb->pVnode), __func__, tstrerror(code)); + } else { + tsdbInfo("vgId:%d, %s done, path:%s, fid:%d", TD_VID(pReader->pTsdb->pVnode), __func__, pReader->pTsdb->path, + pReader->fid); + } return code; } @@ -191,6 +192,7 @@ static int32_t tsdbSnapNextRow(STsdbSnapReader* pReader) { int64_t rowVer = pIter->bData.aVersion[pIter->iRow]; if (rowVer >= pReader->sver && rowVer <= pReader->ever) { + pIter->rInfo.suid = pIter->bData.suid; pIter->rInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow]; pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); goto _out; @@ -318,12 +320,14 @@ _exit: static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; - STsdb* pTsdb = pReader->pTsdb; + int32_t lino = 0; + + STsdb* pTsdb = pReader->pTsdb; while (true) { if (pReader->pDataFReader == NULL) { code = tsdbSnapReadOpenFile(pReader); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } if (pReader->pDataFReader == NULL) break; @@ -338,17 +342,17 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { SBlockData* pBlockData = &pReader->bData; code = tsdbUpdateTableSchema(pTsdb->pVnode->pMeta, id.suid, id.uid, &pReader->skmTable); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataInit(pBlockData, &id, pReader->skmTable.pTSchema, NULL, 0); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); while (pRowInfo->suid == id.suid && pRowInfo->uid == id.uid) { code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL, pRowInfo->uid); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbSnapNextRow(pReader); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pRowInfo = tsdbSnapGetRow(pReader); if (pRowInfo == NULL) { @@ -360,21 +364,22 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { } code = tsdbSnapCmprData(pReader, ppData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); break; } - return code; - -_err: - tsdbError("vgId:%d, vnode snapshot tsdb read data for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, - tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d, %s failed since %s, path:%s", TD_VID(pTsdb->pVnode), __func__, tstrerror(code), pTsdb->path); + } return code; } static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { - int32_t code = 0; + int32_t code = 0; + int32_t lino = 0; + STsdb* pTsdb = pReader->pTsdb; SDelFile* pDelFile = pReader->fs.pDelFile; @@ -385,11 +390,11 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { // open code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pTsdb); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // read index code = tsdbReadDelIdx(pReader->pDelFReader, pReader->aDelIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pReader->iDelIdx = 0; } @@ -405,7 +410,7 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { pReader->iDelIdx++; code = tsdbReadDelData(pReader->pDelFReader, pDelIdx, pReader->aDelData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); int32_t size = 0; for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); iDelData++) { @@ -422,7 +427,7 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size); if (*ppData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData); @@ -449,11 +454,9 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { } _exit: - return code; - -_err: - tsdbError("vgId:%d, vnode snapshot tsdb read del for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, - tstrerror(code)); + if (code) { + tsdbError("vgId:%d, %s failed since %s, path:%s", TD_VID(pTsdb->pVnode), __func__, tstrerror(code), pTsdb->path); + } return code; } @@ -538,7 +541,7 @@ _exit: if (pReader) { taosArrayDestroy(pReader->aDelData); taosArrayDestroy(pReader->aDelIdx); - tBlockDataDestroy(&pReader->bData, 1); + tBlockDataDestroy(&pReader->bData); tsdbFSDestroy(&pReader->fs); taosMemoryFree(pReader); } @@ -565,10 +568,10 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { taosArrayDestroy(pIter->aSttBlk); } - tBlockDataDestroy(&pIter->bData, 1); + tBlockDataDestroy(&pIter->bData); } - tBlockDataDestroy(&pReader->bData, 1); + tBlockDataDestroy(&pReader->bData); tDestroyTSchema(pReader->skmTable.pTSchema); // del @@ -591,44 +594,39 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; + int32_t lino = 0; *ppData = NULL; // read data file if (!pReader->dataDone) { code = tsdbSnapReadData(pReader, ppData); - if (code) { - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); + if (*ppData) { + goto _exit; } else { - if (*ppData) { - goto _exit; - } else { - pReader->dataDone = 1; - } + pReader->dataDone = 1; } } // read del file if (!pReader->delDone) { code = tsdbSnapReadDel(pReader, ppData); - if (code) { - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); + if (*ppData) { + goto _exit; } else { - if (*ppData) { - goto _exit; - } else { - pReader->delDone = 1; - } + pReader->delDone = 1; } } _exit: - tsdbDebug("vgId:%d, vnode snapshot tsdb read for %s", TD_VID(pReader->pTsdb->pVnode), pReader->pTsdb->path); - return code; - -_err: - tsdbError("vgId:%d, vnode snapshot tsdb read for %s failed since %s", TD_VID(pReader->pTsdb->pVnode), - pReader->pTsdb->path, tstrerror(code)); + if (code) { + tsdbError("vgId:%d, %s failed since %s, path:%s", TD_VID(pReader->pTsdb->pVnode), __func__, tstrerror(code), + pReader->pTsdb->path); + } else { + tsdbDebug("vgId:%d, %s done, path:%s", TD_VID(pReader->pTsdb->pVnode), __func__, pReader->pTsdb->path); + } return code; } @@ -1359,13 +1357,13 @@ _exit: if (pWriter->aDelIdxW) taosArrayDestroy(pWriter->aDelIdxW); if (pWriter->aDelData) taosArrayDestroy(pWriter->aDelData); if (pWriter->aDelIdxR) taosArrayDestroy(pWriter->aDelIdxR); - tBlockDataDestroy(&pWriter->dWriter.sData, 1); - tBlockDataDestroy(&pWriter->dWriter.bData, 1); + tBlockDataDestroy(&pWriter->dWriter.sData); + tBlockDataDestroy(&pWriter->dWriter.bData); if (pWriter->dWriter.aSttBlk) taosArrayDestroy(pWriter->dWriter.aSttBlk); if (pWriter->dWriter.aBlockIdx) taosArrayDestroy(pWriter->dWriter.aBlockIdx); - tBlockDataDestroy(&pWriter->dReader.bData, 1); + tBlockDataDestroy(&pWriter->dReader.bData); if (pWriter->dReader.aBlockIdx) taosArrayDestroy(pWriter->dReader.aBlockIdx); - tBlockDataDestroy(&pWriter->bData, 1); + tBlockDataDestroy(&pWriter->bData); tsdbFSDestroy(&pWriter->fs); taosMemoryFree(pWriter); } @@ -1425,18 +1423,18 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { // SNAP_DATA_TSDB // Writer - tBlockDataDestroy(&pWriter->dWriter.sData, 1); - tBlockDataDestroy(&pWriter->dWriter.bData, 1); + tBlockDataDestroy(&pWriter->dWriter.sData); + tBlockDataDestroy(&pWriter->dWriter.bData); taosArrayDestroy(pWriter->dWriter.aSttBlk); tMapDataClear(&pWriter->dWriter.mDataBlk); taosArrayDestroy(pWriter->dWriter.aBlockIdx); // Reader - tBlockDataDestroy(&pWriter->dReader.bData, 1); + tBlockDataDestroy(&pWriter->dReader.bData); tMapDataClear(&pWriter->dReader.mDataBlk); taosArrayDestroy(pWriter->dReader.aBlockIdx); - tBlockDataDestroy(&pWriter->bData, 1); + tBlockDataDestroy(&pWriter->bData); tDestroyTSchema(pWriter->skmTable.pTSchema); for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) { diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 55703002b80a518d3d572dab6dc5368910dcfbc2..f4ac4b7a975b9639d865ef7bd1a9631edff7fa37 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -23,7 +23,7 @@ void tMapDataReset(SMapData *pMapData) { } void tMapDataClear(SMapData *pMapData) { - tFree((uint8_t *)pMapData->aOffset); + tFree(pMapData->aOffset); tFree(pMapData->pData); pMapData->pData = NULL; pMapData->aOffset = NULL; @@ -572,9 +572,9 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal * ASSERT(iCol > 0); - if (pRow->type == 0) { - tTSRowGetVal(pRow->pTSRow, pTSchema, iCol, pColVal); - } else if (pRow->type == 1) { + if (pRow->type == TSDBROW_ROW_FMT) { + tRowGet(pRow->pTSRow, pTSchema, iCol, pColVal); + } else if (pRow->type == TSDBROW_COL_FMT) { SColData *pColData; tBlockDataGetColData(pRow->pBlockData, pTColumn->colId, &pColData); @@ -589,60 +589,61 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal * } } -// int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow) { -// int32_t n = 0; - -// n += tPutI64(p, pRow->version); -// if (p) memcpy(p + n, pRow->pTSRow, pRow->pTSRow->len); -// n += pRow->pTSRow->len; - -// return n; -// } - int32_t tsdbRowCmprFn(const void *p1, const void *p2) { return tsdbKeyCmprFn(&TSDBROW_KEY((TSDBROW *)p1), &TSDBROW_KEY((TSDBROW *)p2)); } // STSDBRowIter ====================================================== -void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { + int32_t code = 0; + pIter->pRow = pRow; - if (pRow->type == 0) { - ASSERT(pTSchema); - pIter->pTSchema = pTSchema; - pIter->i = 1; - } else if (pRow->type == 1) { - pIter->pTSchema = NULL; - pIter->i = 0; + if (pRow->type == TSDBROW_ROW_FMT) { + code = tRowIterOpen(pRow->pTSRow, pTSchema, &pIter->pIter); + if (code) goto _exit; + } else if (pRow->type == TSDBROW_COL_FMT) { + pIter->iColData = 0; } else { ASSERT(0); } + +_exit: + return code; } -SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { - if (pIter->pRow->type == 0) { - if (pIter->i < pIter->pTSchema->numOfCols) { - tTSRowGetVal(pIter->pRow->pTSRow, pIter->pTSchema, pIter->i, &pIter->colVal); - pIter->i++; +void tsdbRowClose(STSDBRowIter *pIter) { + if (pIter->pRow->type == TSDBROW_ROW_FMT) { + tRowIterClose(&pIter->pIter); + } +} - return &pIter->colVal; +SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { + if (pIter->pRow->type == TSDBROW_ROW_FMT) { + return tRowIterNext(pIter->pIter); + } else if (pIter->pRow->type == TSDBROW_COL_FMT) { + if (pIter->iColData == 0) { + pIter->cv = COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, + (SValue){.val = pIter->pRow->pBlockData->aTSKEY[pIter->pRow->iRow]}); + ++pIter->iColData; + return &pIter->cv; } - } else { - if (pIter->i < pIter->pRow->pBlockData->nColData) { - SColData *pColData = tBlockDataGetColDataByIdx(pIter->pRow->pBlockData, pIter->i); - - tColDataGetValue(pColData, pIter->pRow->iRow, &pIter->colVal); - pIter->i++; - return &pIter->colVal; + if (pIter->iColData < pIter->pRow->pBlockData->nColData) { + tColDataGetValue(&pIter->pRow->pBlockData->aColData[pIter->iColData], pIter->pRow->iRow, &pIter->cv); + ++pIter->iColData; + return &pIter->cv; + } else { + return NULL; } + } else { + ASSERT(0); + return NULL; // suppress error report by compiler } - - return NULL; } // SRowMerger ====================================================== -int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; @@ -682,6 +683,18 @@ int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRo } tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal); + if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { + uint8_t *pVal = pColVal->value.pData; + + pColVal->value.pData = NULL; + code = tRealloc(&pColVal->value.pData, pColVal->value.nData); + if (code) goto _exit; + + if (pColVal->value.nData) { + memcpy(pColVal->value.pData, pVal, pColVal->value.nData); + } + } + if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -697,7 +710,7 @@ _exit: return code; } -int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; @@ -720,12 +733,35 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { if (key.version > pMerger->version) { if (!COL_VAL_IS_NONE(pColVal)) { - taosArraySet(pMerger->pArray, iCol, pColVal); + if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { + SColVal *tColVal = taosArrayGet(pMerger->pArray, iCol); + code = tRealloc(&tColVal->value.pData, pColVal->value.nData); + if (code) return code; + + tColVal->value.nData = pColVal->value.nData; + if (pColVal->value.nData) { + memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData); + } + tColVal->flag = 0; + } else { + taosArraySet(pMerger->pArray, iCol, pColVal); + } } } else if (key.version < pMerger->version) { SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol); if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) { - taosArraySet(pMerger->pArray, iCol, pColVal); + if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { + code = tRealloc(&tColVal->value.pData, pColVal->value.nData); + if (code) return code; + + tColVal->value.nData = pColVal->value.nData; + if (pColVal->value.nData) { + memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData); + } + tColVal->flag = 0; + } else { + taosArraySet(pMerger->pArray, iCol, pColVal); + } } } else { ASSERT(0 && "dup versions not allowed"); @@ -736,7 +772,7 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { return code; } -int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; @@ -765,6 +801,18 @@ int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { // other for (int16_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) { tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal); + if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { + uint8_t *pVal = pColVal->value.pData; + + pColVal->value.pData = NULL; + code = tRealloc(&pColVal->value.pData, pColVal->value.nData); + if (code) goto _exit; + + if (pColVal->value.nData) { + memcpy(pColVal->value.pData, pVal, pColVal->value.nData); + } + } + if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -775,9 +823,18 @@ _exit: return code; } -void tRowMergerClear(SRowMerger *pMerger) { taosArrayDestroy(pMerger->pArray); } +void tsdbRowMergerClear(SRowMerger *pMerger) { + for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { + SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); + if (IS_VAR_DATA_TYPE(pTColVal->type)) { + tFree(pTColVal->value.pData); + } + } -int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { + taosArrayDestroy(pMerger->pArray); +} + +int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; @@ -789,12 +846,47 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { if (key.version > pMerger->version) { if (!COL_VAL_IS_NONE(pColVal)) { - taosArraySet(pMerger->pArray, iCol, pColVal); + if (IS_VAR_DATA_TYPE(pColVal->type)) { + SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); + if (!COL_VAL_IS_NULL(pColVal)) { + code = tRealloc(&pTColVal->value.pData, pColVal->value.nData); + if (code) goto _exit; + + pTColVal->value.nData = pColVal->value.nData; + if (pTColVal->value.nData) { + memcpy(pTColVal->value.pData, pColVal->value.pData, pTColVal->value.nData); + } + pTColVal->flag = 0; + } else { + tFree(pTColVal->value.pData); + pTColVal->value.pData = NULL; + taosArraySet(pMerger->pArray, iCol, pColVal); + } + } else { + taosArraySet(pMerger->pArray, iCol, pColVal); + } } } else if (key.version < pMerger->version) { SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol); if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) { - taosArraySet(pMerger->pArray, iCol, pColVal); + if (IS_VAR_DATA_TYPE(pColVal->type)) { + if (!COL_VAL_IS_NULL(pColVal)) { + code = tRealloc(&tColVal->value.pData, pColVal->value.nData); + if (code) goto _exit; + + tColVal->value.nData = pColVal->value.nData; + if (tColVal->value.nData) { + memcpy(tColVal->value.pData, pColVal->value.pData, tColVal->value.nData); + } + tColVal->flag = 0; + } else { + tFree(tColVal->value.pData); + tColVal->value.pData = NULL; + taosArraySet(pMerger->pArray, iCol, pColVal); + } + } else { + taosArraySet(pMerger->pArray, iCol, pColVal); + } } } else { ASSERT(0); @@ -807,12 +899,8 @@ _exit: return code; } -int32_t tRowMergerGetRow(SRowMerger *pMerger, STSRow **ppRow) { - int32_t code = 0; - - code = tdSTSRowNew(pMerger->pArray, pMerger->pTSchema, ppRow); - - return code; +int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow) { + return tRowBuild(pMerger->pArray, pMerger->pTSchema, ppRow); } // delete skyline ====================================================== @@ -931,27 +1019,49 @@ int32_t tBlockDataCreate(SBlockData *pBlockData) { pBlockData->aVersion = NULL; pBlockData->aTSKEY = NULL; pBlockData->nColData = 0; - pBlockData->aColData = taosArrayInit(0, sizeof(SColData)); - if (pBlockData->aColData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } + pBlockData->aColData = NULL; _exit: return code; } -void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear) { - tFree((uint8_t *)pBlockData->aUid); - tFree((uint8_t *)pBlockData->aVersion); - tFree((uint8_t *)pBlockData->aTSKEY); - taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataDestroy : NULL); - pBlockData->aUid = NULL; - pBlockData->aVersion = NULL; - pBlockData->aTSKEY = NULL; - pBlockData->aColData = NULL; +void tBlockDataDestroy(SBlockData *pBlockData) { + tFree(pBlockData->aUid); + tFree(pBlockData->aVersion); + tFree(pBlockData->aTSKEY); + + for (int32_t i = 0; i < pBlockData->nColData; i++) { + tColDataDestroy(&pBlockData->aColData[i]); + } + + if (pBlockData->aColData) { + taosMemoryFree(pBlockData->aColData); + pBlockData->aColData = NULL; + } } +static int32_t tBlockDataAdjustColData(SBlockData *pBlockData, int32_t nColData) { + int32_t code = 0; + + if (pBlockData->nColData > nColData) { + for (int32_t i = nColData; i < pBlockData->nColData; i++) { + tColDataDestroy(&pBlockData->aColData[i]); + } + } else if (pBlockData->nColData < nColData) { + SColData *aColData = taosMemoryRealloc(pBlockData->aColData, sizeof(SBlockData) * nColData); + if (aColData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + pBlockData->aColData = aColData; + memset(&pBlockData->aColData[pBlockData->nColData], 0, sizeof(SBlockData) * (nColData - pBlockData->nColData)); + } + pBlockData->nColData = nColData; + +_exit: + return code; +} int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid) { int32_t code = 0; @@ -961,37 +1071,35 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, pBlockData->uid = pId->uid; pBlockData->nRow = 0; - pBlockData->nColData = 0; if (aCid) { + code = tBlockDataAdjustColData(pBlockData, nCid); + if (code) goto _exit; + int32_t iColumn = 1; STColumn *pTColumn = &pTSchema->columns[iColumn]; for (int32_t iCid = 0; iCid < nCid; iCid++) { - while (pTColumn && pTColumn->colId < aCid[iCid]) { + ASSERT(pTColumn); + while (pTColumn->colId < aCid[iCid]) { iColumn++; - pTColumn = (iColumn < pTSchema->numOfCols) ? &pTSchema->columns[iColumn] : NULL; + ASSERT(iColumn < pTSchema->numOfCols); + pTColumn = &pTSchema->columns[iColumn]; } - if (pTColumn == NULL) { - break; - } else if (pTColumn->colId == aCid[iCid]) { - SColData *pColData; - code = tBlockDataAddColData(pBlockData, &pColData); - if (code) goto _exit; - tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) ? 1 : 0); + ASSERT(pTColumn->colId == aCid[iCid]); + tColDataInit(&pBlockData->aColData[iCid], pTColumn->colId, pTColumn->type, + (pTColumn->flags & COL_SMA_ON) ? 1 : 0); - iColumn++; - pTColumn = (iColumn < pTSchema->numOfCols) ? &pTSchema->columns[iColumn] : NULL; - } + iColumn++; + pTColumn = (iColumn < pTSchema->numOfCols) ? &pTSchema->columns[iColumn] : NULL; } } else { - for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { - STColumn *pTColumn = &pTSchema->columns[iColumn]; - - SColData *pColData; - code = tBlockDataAddColData(pBlockData, &pColData); - if (code) goto _exit; + code = tBlockDataAdjustColData(pBlockData, pTSchema->numOfCols - 1); + if (code) goto _exit; - tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) ? 1 : 0); + for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { + STColumn *pTColumn = &pTSchema->columns[iColData + 1]; + tColDataInit(&pBlockData->aColData[iColData], pTColumn->colId, pTColumn->type, + (pTColumn->flags & COL_SMA_ON) ? 1 : 0); } } @@ -1002,8 +1110,6 @@ _exit: void tBlockDataReset(SBlockData *pBlockData) { pBlockData->suid = 0; pBlockData->uid = 0; - pBlockData->nRow = 0; - pBlockData->nColData = 0; } void tBlockDataClear(SBlockData *pBlockData) { @@ -1011,49 +1117,22 @@ void tBlockDataClear(SBlockData *pBlockData) { pBlockData->nRow = 0; for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - tColDataClear(pColData); + tColDataClear(tBlockDataGetColDataByIdx(pBlockData, iColData)); } } -int32_t tBlockDataAddColData(SBlockData *pBlockData, SColData **ppColData) { - int32_t code = 0; - SColData *pColData = NULL; - - if (pBlockData->nColData >= taosArrayGetSize(pBlockData->aColData)) { - if (taosArrayPush(pBlockData->aColData, &((SColData){0})) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - } - pColData = (SColData *)taosArrayGet(pBlockData->aColData, pBlockData->nColData); - - pBlockData->nColData++; - - *ppColData = pColData; - return code; - -_err: - *ppColData = NULL; - return code; -} - static int32_t tBlockDataAppendBlockRow(SBlockData *pBlockData, SBlockData *pBlockDataFrom, int32_t iRow) { int32_t code = 0; SColVal cv = {0}; int32_t iColDataFrom = 0; - SColData *pColDataFrom = - (iColDataFrom < pBlockDataFrom->nColData) ? &((SColData *)pBlockDataFrom->aColData->pData)[iColDataFrom] : NULL; + SColData *pColDataFrom = (iColDataFrom < pBlockDataFrom->nColData) ? &pBlockDataFrom->aColData[iColDataFrom] : NULL; for (int32_t iColDataTo = 0; iColDataTo < pBlockData->nColData; iColDataTo++) { - SColData *pColDataTo = &((SColData *)pBlockData->aColData->pData)[iColDataTo]; + SColData *pColDataTo = &pBlockData->aColData[iColDataTo]; while (pColDataFrom && pColDataFrom->cid < pColDataTo->cid) { - iColDataFrom++; - pColDataFrom = (iColDataFrom < pBlockDataFrom->nColData) - ? &((SColData *)pBlockDataFrom->aColData->pData)[iColDataFrom] - : NULL; + pColDataFrom = (++iColDataFrom < pBlockDataFrom->nColData) ? &pBlockDataFrom->aColData[iColDataFrom] : NULL; } if (pColDataFrom == NULL || pColDataFrom->cid > pColDataTo->cid) { @@ -1065,157 +1144,7 @@ static int32_t tBlockDataAppendBlockRow(SBlockData *pBlockData, SBlockData *pBlo code = tColDataAppendValue(pColDataTo, &cv); if (code) goto _exit; - iColDataFrom++; - pColDataFrom = (iColDataFrom < pBlockDataFrom->nColData) - ? &((SColData *)pBlockDataFrom->aColData->pData)[iColDataFrom] - : NULL; - } - } - -_exit: - return code; -} - -static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSchema *pTSchema) { - int32_t code = 0; - - int32_t iTColumn = 1; - STColumn *pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - void *pBitmap = pRow->statis ? tdGetBitmapAddrTp(pRow, pTSchema->flen) : NULL; - - for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = &((SColData *)pBlockData->aColData->pData)[iColData]; - - while (pTColumn && pTColumn->colId < pColData->cid) { - iTColumn++; - pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - } - - if (pTColumn == NULL || pTColumn->colId > pColData->cid) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } else { - ASSERT(pTColumn->type == pColData->type); - - SColVal cv = {.cid = pTColumn->colId, .type = pTColumn->type}; - - if (pRow->statis) { - TDRowValT vt = TD_VTYPE_MAX; - tdGetBitmapValTypeII(pBitmap, iTColumn - 1, &vt); - - if (vt == TD_VTYPE_NORM) { - cv.flag = CV_FLAG_VALUE; - - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); - cv.value.nData = varDataLen(pData); - cv.value.pData = varDataVal(pData); - } else { - memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); - } - - code = tColDataAppendValue(pColData, &cv); - if (code) goto _exit; - } else if (vt == TD_VTYPE_NONE) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } else if (vt == TD_VTYPE_NULL) { - code = tColDataAppendValue(pColData, &COL_VAL_NULL(pColData->cid, pColData->type)); - if (code) goto _exit; - } else { - ASSERT(0); - } - } else { - cv.flag = CV_FLAG_VALUE; - - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); - cv.value.nData = varDataLen(pData); - cv.value.pData = varDataVal(pData); - } else { - memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); - } - - code = tColDataAppendValue(pColData, &cv); - if (code) goto _exit; - } - - iTColumn++; - pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - } - } - -_exit: - return code; -} - -static int32_t tBlockDataAppendKVRow(SBlockData *pBlockData, STSRow *pRow, STSchema *pTSchema) { - int32_t code = 0; - - col_id_t kvIter = 0; - col_id_t nKvCols = tdRowGetNCols(pRow) - 1; - void *pColIdx = TD_ROW_COL_IDX(pRow); - void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow)); - int32_t iTColumn = 1; - STColumn *pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - - for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = &((SColData *)pBlockData->aColData->pData)[iColData]; - - while (pTColumn && pTColumn->colId < pColData->cid) { - iTColumn++; - pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - } - - if (pTColumn == NULL || pTColumn->colId > pColData->cid) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } else { - ASSERT(pTColumn->type == pColData->type); - - SColVal cv = {.cid = pTColumn->colId, .type = pTColumn->type}; - TDRowValT vt = TD_VTYPE_NONE; // default is NONE - SKvRowIdx *pKvIdx = NULL; - - while (kvIter < nKvCols) { - pKvIdx = (SKvRowIdx *)POINTER_SHIFT(pColIdx, kvIter * sizeof(SKvRowIdx)); - if (pKvIdx->colId == pTColumn->colId) { - tdGetBitmapValTypeII(pBitmap, kvIter, &vt); - ++kvIter; - break; - } else if (pKvIdx->colId > pTColumn->colId) { - vt = TD_VTYPE_NONE; - break; - } else { - ++kvIter; - } - } - - if (vt == TD_VTYPE_NORM) { - cv.flag = CV_FLAG_VALUE; - - void *pData = POINTER_SHIFT(pRow, pKvIdx->offset); - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - cv.value.nData = varDataLen(pData); - cv.value.pData = varDataVal(pData); - } else { - memcpy(&cv.value.val, pData, pTColumn->bytes); - } - - code = tColDataAppendValue(pColData, &cv); - if (code) goto _exit; - } else if (vt == TD_VTYPE_NONE) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } else if (vt == TD_VTYPE_NULL) { - code = tColDataAppendValue(pColData, &COL_VAL_NULL(pColData->cid, pColData->type)); - if (code) goto _exit; - } else { - ASSERT(0); - } - - iTColumn++; - pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + pColDataFrom = (++iColDataFrom < pBlockDataFrom->nColData) ? &pBlockDataFrom->aColData[iColDataFrom] : NULL; } } @@ -1245,19 +1174,14 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS pBlockData->aTSKEY[pBlockData->nRow] = TSDBROW_TS(pRow); SColVal cv = {0}; - if (pRow->type == 0) { - if (TD_IS_TP_ROW(pRow->pTSRow)) { - code = tBlockDataAppendTPRow(pBlockData, pRow->pTSRow, pTSchema); - if (code) goto _err; - } else if (TD_IS_KV_ROW(pRow->pTSRow)) { - code = tBlockDataAppendKVRow(pBlockData, pRow->pTSRow, pTSchema); - if (code) goto _err; - } else { - ASSERT(0); - } - } else { + if (pRow->type == TSDBROW_ROW_FMT) { + code = tRowAppendToColData(pRow->pTSRow, pTSchema, pBlockData->aColData, pBlockData->nColData); + if (code) goto _err; + } else if (pRow->type == TSDBROW_COL_FMT) { code = tBlockDataAppendBlockRow(pBlockData, pRow->pBlockData, pRow->iRow); if (code) goto _err; + } else { + ASSERT(0); } pBlockData->nRow++; @@ -1267,133 +1191,13 @@ _err: return code; } -int32_t tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFrom) { - int32_t code = 0; - - int32_t iColData = 0; - for (int32_t iColDataFrom = 0; iColDataFrom < pBlockDataFrom->nColData; iColDataFrom++) { - SColData *pColDataFrom = tBlockDataGetColDataByIdx(pBlockDataFrom, iColDataFrom); - - while (true) { - SColData *pColData; - if (iColData < pBlockData->nColData) { - pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - } else { - pColData = NULL; - } - - if (pColData == NULL || pColData->cid > pColDataFrom->cid) { - code = tBlockDataAddColData(pBlockData, &pColData); - if (code) goto _exit; - - tColDataInit(pColData, pColDataFrom->cid, pColDataFrom->type, pColDataFrom->smaOn); - for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } - - iColData++; - break; - } else if (pColData->cid == pColDataFrom->cid) { - iColData++; - break; - } else { - iColData++; - } - } - } - -_exit: - return code; -} - -int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData) { - int32_t code = 0; - - ASSERT(pBlockData->suid == pBlockData1->suid); - ASSERT(pBlockData->uid == pBlockData1->uid); - ASSERT(pBlockData1->nRow > 0); - ASSERT(pBlockData2->nRow > 0); - - tBlockDataClear(pBlockData); - - TSDBROW row1 = tsdbRowFromBlockData(pBlockData1, 0); - TSDBROW row2 = tsdbRowFromBlockData(pBlockData2, 0); - TSDBROW *pRow1 = &row1; - TSDBROW *pRow2 = &row2; - - while (pRow1 && pRow2) { - int32_t c = tsdbRowCmprFn(pRow1, pRow2); - - if (c < 0) { - code = tBlockDataAppendRow(pBlockData, pRow1, NULL, - pBlockData1->uid ? pBlockData1->uid : pBlockData1->aUid[pRow1->iRow]); - if (code) goto _exit; - - pRow1->iRow++; - if (pRow1->iRow < pBlockData1->nRow) { - *pRow1 = tsdbRowFromBlockData(pBlockData1, pRow1->iRow); - } else { - pRow1 = NULL; - } - } else if (c > 0) { - code = tBlockDataAppendRow(pBlockData, pRow2, NULL, - pBlockData2->uid ? pBlockData2->uid : pBlockData2->aUid[pRow2->iRow]); - if (code) goto _exit; - - pRow2->iRow++; - if (pRow2->iRow < pBlockData2->nRow) { - *pRow2 = tsdbRowFromBlockData(pBlockData2, pRow2->iRow); - } else { - pRow2 = NULL; - } - } else { - ASSERT(0); - } - } - - while (pRow1) { - code = tBlockDataAppendRow(pBlockData, pRow1, NULL, - pBlockData1->uid ? pBlockData1->uid : pBlockData1->aUid[pRow1->iRow]); - if (code) goto _exit; - - pRow1->iRow++; - if (pRow1->iRow < pBlockData1->nRow) { - *pRow1 = tsdbRowFromBlockData(pBlockData1, pRow1->iRow); - } else { - pRow1 = NULL; - } - } - - while (pRow2) { - code = tBlockDataAppendRow(pBlockData, pRow2, NULL, - pBlockData2->uid ? pBlockData2->uid : pBlockData2->aUid[pRow2->iRow]); - if (code) goto _exit; - - pRow2->iRow++; - if (pRow2->iRow < pBlockData2->nRow) { - *pRow2 = tsdbRowFromBlockData(pBlockData2, pRow2->iRow); - } else { - pRow2 = NULL; - } - } - -_exit: - return code; -} - -SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx) { - ASSERT(idx >= 0 && idx < pBlockData->nColData); - return (SColData *)taosArrayGet(pBlockData->aColData, idx); -} - void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData) { ASSERT(cid != PRIMARYKEY_TIMESTAMP_COL_ID); int32_t lidx = 0; int32_t ridx = pBlockData->nColData - 1; while (lidx <= ridx) { - int32_t midx = (lidx + ridx) / 2; + int32_t midx = (lidx + ridx) >> 1; SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, midx); int32_t c = (pColData->cid == cid) ? 0 : ((pColData->cid > cid) ? 1 : -1); @@ -1540,15 +1344,25 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin // loop to decode each column data if (hdr.szBlkCol == 0) goto _exit; + int32_t nColData = 0; int32_t nt = 0; while (nt < hdr.szBlkCol) { SBlockCol blockCol = {0}; nt += tGetBlockCol(pIn + n + nt, &blockCol); - ASSERT(nt <= hdr.szBlkCol); + ++nColData; + } + ASSERT(nt == hdr.szBlkCol); - SColData *pColData; - code = tBlockDataAddColData(pBlockData, &pColData); - if (code) goto _exit; + code = tBlockDataAdjustColData(pBlockData, nColData); + if (code) goto _exit; + + nt = 0; + int32_t iColData = 0; + while (nt < hdr.szBlkCol) { + SBlockCol blockCol = {0}; + nt += tGetBlockCol(pIn + n + nt, &blockCol); + + SColData *pColData = &pBlockData->aColData[iColData++]; tColDataInit(pColData, blockCol.cid, blockCol.type, blockCol.smaOn); if (blockCol.flag == HAS_NULL) { diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 49d5eaac43c6dbd2356bf95ca4cfb9c70cb350c7..02d076113f6cdab7066740ce071426ad06efb7d4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -26,14 +26,17 @@ static int64_t tsMaxKeyByPrecision[] = {31556995200000L, 31556995200000000L, 921 // static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); -int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *pRsp) { - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - int32_t affectedrows = 0; - int32_t numOfRows = 0; +int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2 *pRsp) { + int32_t arrSize = 0; + int32_t affectedrows = 0; + int32_t numOfRows = 0; ASSERT(pTsdb->mem != NULL); + if (pMsg) { + arrSize = taosArrayGetSize(pMsg->aSubmitTbData); + } + // scan and convert if (tsdbScanAndConvertSubmitMsg(pTsdb, pMsg) < 0) { if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) { @@ -43,18 +46,10 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp * } // loop to insert - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { - return -1; - } - while (true) { - SSubmitBlkRsp r = {0}; - tGetSubmitMsgNext(&msgIter, &pBlock); - if (pBlock == NULL) break; - if ((terrno = tsdbInsertTableData(pTsdb, version, &msgIter, pBlock, &r)) < 0) { + for (int32_t i = 0; i < arrSize; ++i) { + if ((terrno = tsdbInsertTableData(pTsdb, version, taosArrayGet(pMsg->aSubmitTbData, i), &affectedrows)) < 0) { return -1; } - - numOfRows += msgIter.numOfRows; } if (pRsp != NULL) { @@ -82,20 +77,19 @@ static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, STable *pTable, STSRow * } #endif -static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, STSRow *row, TSKEY minKey, TSKEY maxKey, +static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, TSKEY rowKey, TSKEY minKey, TSKEY maxKey, TSKEY now) { - TSKEY rowKey = TD_ROW_KEY(row); if (rowKey < minKey || rowKey > maxKey) { tsdbError("vgId:%d, table uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 " maxKey %" PRId64 " row key %" PRId64, TD_VID(pTsdb->pVnode), uid, now, minKey, maxKey, rowKey); - terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; - return -1; + return TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; } return 0; } +#if 0 int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { ASSERT(pMsg != NULL); // STsdbMeta * pMeta = pTsdb->tsdbMeta; @@ -165,4 +159,42 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { if (terrno != TSDB_CODE_SUCCESS) return -1; return 0; +} +#endif + +int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) { + int32_t code = 0; + STsdbKeepCfg *pCfg = &pTsdb->keepCfg; + TSKEY now = taosGetTimestamp(pCfg->precision); + TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2; + TSKEY maxKey = tsMaxKeyByPrecision[pCfg->precision]; + int32_t size = taosArrayGetSize(pMsg->aSubmitTbData); + + for (int32_t i = 0; i < size; ++i) { + SSubmitTbData *pData = TARRAY_GET_ELEM(pMsg->aSubmitTbData, i); + if (pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + uint64_t nColData = TARRAY_SIZE(pData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pData->aCol); + if (nColData > 0) { + int32_t nRows = aColData[0].nVal; + TSKEY *aKey = (TSKEY *)aColData[0].pData; + for (int32_t r = 0; r < nRows; ++r) { + if ((code = tsdbCheckRowRange(pTsdb, pData->uid, aKey[r], minKey, maxKey, now)) < 0) { + goto _exit; + } + } + } + } else { + int32_t nRows = taosArrayGetSize(pData->aRowP); + for (int32_t r = 0; r < nRows; ++r) { + SRow *pRow = (SRow *)taosArrayGetP(pData->aRowP, r); + if ((code = tsdbCheckRowRange(pTsdb, pData->uid, pRow->ts, minKey, maxKey, now)) < 0) { + goto _exit; + } + } + } + } + +_exit: + return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 83a414dae08bd48e41e12682922daf7055e1f5a0..80a422ca3b647ea2642a87e960e8f01ca24073c5 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -16,9 +16,7 @@ #include "vnd.h" /* ------------------------ STRUCTURES ------------------------ */ -#define VNODE_BUFPOOL_SEGMENTS 3 - -static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool) { +static int vnodeBufPoolCreate(SVnode *pVnode, int32_t id, int64_t size, SVBufPool **ppPool) { SVBufPool *pPool; pPool = taosMemoryMalloc(sizeof(SVBufPool) + size); @@ -26,6 +24,21 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool) terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + memset(pPool, 0, sizeof(SVBufPool)); + + // query handle list + taosThreadMutexInit(&pPool->mutex, NULL); + pPool->nQuery = 0; + pPool->qList.pNext = &pPool->qList; + pPool->qList.ppNext = &pPool->qList.pNext; + + pPool->pVnode = pVnode; + pPool->id = id; + pPool->ptr = pPool->node.data; + pPool->pTail = &pPool->node; + pPool->node.prev = NULL; + pPool->node.pnext = &pPool->pTail; + pPool->node.size = size; if (VND_IS_RSMA(pVnode)) { pPool->lock = taosMemoryMalloc(sizeof(TdThreadSpinlock)); @@ -44,16 +57,6 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool) pPool->lock = NULL; } - pPool->next = NULL; - pPool->pVnode = pVnode; - pPool->nRef = 0; - pPool->size = 0; - pPool->ptr = pPool->node.data; - pPool->pTail = &pPool->node; - pPool->node.prev = NULL; - pPool->node.pnext = &pPool->pTail; - pPool->node.size = size; - *ppPool = pPool; return 0; } @@ -64,27 +67,25 @@ static int vnodeBufPoolDestroy(SVBufPool *pPool) { taosThreadSpinDestroy(pPool->lock); taosMemoryFree((void *)pPool->lock); } + taosThreadMutexDestroy(&pPool->mutex); taosMemoryFree(pPool); return 0; } int vnodeOpenBufPool(SVnode *pVnode) { - SVBufPool *pPool = NULL; - int64_t size = pVnode->config.szBuf / VNODE_BUFPOOL_SEGMENTS; + int64_t size = pVnode->config.szBuf / VNODE_BUFPOOL_SEGMENTS; - ASSERT(pVnode->pPool == NULL); - - for (int i = 0; i < 3; i++) { + for (int i = 0; i < VNODE_BUFPOOL_SEGMENTS; i++) { // create pool - if (vnodeBufPoolCreate(pVnode, size, &pPool)) { + if (vnodeBufPoolCreate(pVnode, i, size, &pVnode->aBufPool[i])) { vError("vgId:%d, failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno)); vnodeCloseBufPool(pVnode); return -1; } - // add pool to vnode - pPool->next = pVnode->pPool; - pVnode->pPool = pPool; + // add to free list + pVnode->aBufPool[i]->freeNext = pVnode->freeList; + pVnode->freeList = pVnode->aBufPool[i]; } vDebug("vgId:%d, vnode buffer pool is opened, size:%" PRId64, TD_VID(pVnode), size); @@ -92,23 +93,19 @@ int vnodeOpenBufPool(SVnode *pVnode) { } int vnodeCloseBufPool(SVnode *pVnode) { - SVBufPool *pPool; - - for (pPool = pVnode->pPool; pPool; pPool = pVnode->pPool) { - pVnode->pPool = pPool->next; - vnodeBufPoolDestroy(pPool); + for (int32_t i = 0; i < VNODE_BUFPOOL_SEGMENTS; i++) { + if (pVnode->aBufPool[i]) { + vnodeBufPoolDestroy(pVnode->aBufPool[i]); + pVnode->aBufPool[i] = NULL; + } } - if (pVnode->inUse) { - vnodeBufPoolDestroy(pVnode->inUse); - pVnode->inUse = NULL; - } vDebug("vgId:%d, vnode buffer pool is closed", TD_VID(pVnode)); - return 0; } void vnodeBufPoolReset(SVBufPool *pPool) { + ASSERT(pPool->nQuery == 0); for (SVBufPoolNode *pNode = pPool->pTail; pNode->prev; pNode = pPool->pTail) { ASSERT(pNode->pnext == &pPool->pTail); pNode->prev->pnext = &pPool->pTail; @@ -215,35 +212,121 @@ void vnodeBufPoolRef(SVBufPool *pPool) { ASSERT(nRef > 0); } -void vnodeBufPoolUnRef(SVBufPool *pPool) { - if (pPool == NULL) { - return; - } - int32_t nRef = atomic_sub_fetch_32(&pPool->nRef, 1); - if (nRef == 0) { - SVnode *pVnode = pPool->pVnode; - - vnodeBufPoolReset(pPool); - - taosThreadMutexLock(&pVnode->mutex); - - int64_t size = pVnode->config.szBuf / VNODE_BUFPOOL_SEGMENTS; - if (pPool->node.size != size) { - SVBufPool *pPoolT = NULL; - if (vnodeBufPoolCreate(pVnode, size, &pPoolT) < 0) { - vWarn("vgId:%d, try to change buf pools size from %" PRId64 " to %" PRId64 " since %s", TD_VID(pVnode), - pPool->node.size, size, tstrerror(errno)); - } else { - vnodeBufPoolDestroy(pPool); - pPool = pPoolT; - vDebug("vgId:%d, change buf pools size from %" PRId64 " to %" PRId64, TD_VID(pVnode), pPool->node.size, size); - } +void vnodeBufPoolAddToFreeList(SVBufPool *pPool) { + SVnode *pVnode = pPool->pVnode; + + int64_t size = pVnode->config.szBuf / VNODE_BUFPOOL_SEGMENTS; + if (pPool->node.size != size) { + SVBufPool *pNewPool = NULL; + if (vnodeBufPoolCreate(pVnode, pPool->id, size, &pNewPool) < 0) { + vWarn("vgId:%d failed to change buffer pool of id %d size from %" PRId64 " to %" PRId64 " since %s", + TD_VID(pVnode), pPool->id, pPool->node.size, size, tstrerror(errno)); + } else { + vInfo("vgId:%d buffer pool of id %d size changed from %" PRId64 " to %" PRId64, TD_VID(pVnode), pPool->id, + pPool->node.size, size); + + vnodeBufPoolDestroy(pPool); + pPool = pNewPool; + pVnode->aBufPool[pPool->id] = pPool; } + } + + // add to free list + vDebug("vgId:%d buffer pool %p of id %d is added to free list", TD_VID(pVnode), pPool, pPool->id); + vnodeBufPoolReset(pPool); + pPool->freeNext = pVnode->freeList; + pVnode->freeList = pPool; + taosThreadCondSignal(&pVnode->poolNotEmpty); +} + +void vnodeBufPoolUnRef(SVBufPool *pPool, bool proactive) { + if (pPool == NULL) return; - pPool->next = pVnode->pPool; - pVnode->pPool = pPool; - taosThreadCondSignal(&pVnode->poolNotEmpty); + SVnode *pVnode = pPool->pVnode; - taosThreadMutexUnlock(&pVnode->mutex); + if (proactive) taosThreadMutexLock(&pVnode->mutex); + + if (atomic_sub_fetch_32(&pPool->nRef, 1) > 0) goto _exit; + + // remove from recycle queue or on-recycle position + if (pVnode->onRecycle == pPool) { + pVnode->onRecycle = NULL; + } else { + ASSERT(proactive); + + if (pPool->recyclePrev) { + pPool->recyclePrev->recycleNext = pPool->recycleNext; + } else { + pVnode->recycleHead = pPool->recycleNext; + } + + if (pPool->recycleNext) { + pPool->recycleNext->recyclePrev = pPool->recyclePrev; + } else { + pVnode->recycleTail = pPool->recyclePrev; + } + pPool->recyclePrev = pPool->recycleNext = NULL; } + + vnodeBufPoolAddToFreeList(pPool); + +_exit: + if (proactive) taosThreadMutexUnlock(&pVnode->mutex); + return; +} + +int32_t vnodeBufPoolRegisterQuery(SVBufPool *pPool, SQueryNode *pQNode) { + int32_t code = 0; + + taosThreadMutexLock(&pPool->mutex); + + pQNode->pNext = pPool->qList.pNext; + pQNode->ppNext = &pPool->qList.pNext; + pPool->qList.pNext->ppNext = &pQNode->pNext; + pPool->qList.pNext = pQNode; + pPool->nQuery++; + + taosThreadMutexUnlock(&pPool->mutex); + +_exit: + return code; } + +void vnodeBufPoolDeregisterQuery(SVBufPool *pPool, SQueryNode *pQNode, bool proactive) { + int32_t code = 0; + + if (proactive) taosThreadMutexLock(&pPool->mutex); + + pQNode->pNext->ppNext = pQNode->ppNext; + *pQNode->ppNext = pQNode->pNext; + pPool->nQuery--; + + if (proactive) taosThreadMutexUnlock(&pPool->mutex); +} + +int32_t vnodeBufPoolRecycle(SVBufPool *pPool) { + int32_t code = 0; + + SVnode *pVnode = pPool->pVnode; + + vDebug("vgId:%d recycle buffer pool %p of id %d", TD_VID(pVnode), pPool, pPool->id); + + taosThreadMutexLock(&pPool->mutex); + + SQueryNode *pNode = pPool->qList.pNext; + while (pNode != &pPool->qList) { + SQueryNode *pTNode = pNode->pNext; + + int32_t rc = pNode->reseek(pNode->pQHandle); + if (rc == 0 || rc == TSDB_CODE_VND_QUERY_BUSY) { + pNode = pTNode; + } else { + code = rc; + goto _exit; + } + } + +_exit: + taosThreadMutexUnlock(&pPool->mutex); + return code; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 782cc69d30709c16a328cd056a8b52f6b42a5a6f..c326c8bfacbd897fc15527edaef5cd8e77c40f5f 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -125,13 +125,22 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "vndStats.timeseries", pCfg->vndStats.numOfTimeSeries) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "vndStats.ntimeseries", pCfg->vndStats.numOfNTimeSeries) < 0) return -1; - SJson *pNodeInfoArr = tjsonCreateArray(); - tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", pNodeInfoArr); + SJson *nodeInfo = tjsonCreateArray(); + if (nodeInfo == NULL) return -1; + if (tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", nodeInfo) < 0) return -1; + vDebug("vgId:%d, encode config, replicas:%d selfIndex:%d", pCfg->vgId, pCfg->syncCfg.replicaNum, + pCfg->syncCfg.myIndex); for (int i = 0; i < pCfg->syncCfg.replicaNum; ++i) { - SJson *pNodeInfo = tjsonCreateObject(); - tjsonAddIntegerToObject(pNodeInfo, "nodePort", (pCfg->syncCfg.nodeInfo)[i].nodePort); - tjsonAddStringToObject(pNodeInfo, "nodeFqdn", (pCfg->syncCfg.nodeInfo)[i].nodeFqdn); - tjsonAddItemToArray(pNodeInfoArr, pNodeInfo); + SJson *info = tjsonCreateObject(); + SNodeInfo *pNode = (SNodeInfo *)&pCfg->syncCfg.nodeInfo[i]; + if (info == NULL) return -1; + if (tjsonAddIntegerToObject(info, "nodePort", pNode->nodePort) < 0) return -1; + if (tjsonAddStringToObject(info, "nodeFqdn", pNode->nodeFqdn) < 0) return -1; + if (tjsonAddIntegerToObject(info, "nodeId", pNode->nodeId) < 0) return -1; + if (tjsonAddIntegerToObject(info, "clusterId", pNode->clusterId) < 0) return -1; + if (tjsonAddItemToArray(nodeInfo, info) < 0) return -1; + vDebug("vgId:%d, encode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort, + pNode->nodeId); } return 0; @@ -240,15 +249,25 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { tjsonGetNumberValue(pJson, "vndStats.ntimeseries", pCfg->vndStats.numOfNTimeSeries, code); if (code < 0) return -1; - SJson *pNodeInfoArr = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo"); - int arraySize = tjsonGetArraySize(pNodeInfoArr); - assert(arraySize == pCfg->syncCfg.replicaNum); + SJson *nodeInfo = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo"); + int arraySize = tjsonGetArraySize(nodeInfo); + if (arraySize != pCfg->syncCfg.replicaNum) return -1; + vDebug("vgId:%d, decode config, replicas:%d selfIndex:%d", pCfg->vgId, pCfg->syncCfg.replicaNum, + pCfg->syncCfg.myIndex); for (int i = 0; i < arraySize; ++i) { - SJson *pNodeInfo = tjsonGetArrayItem(pNodeInfoArr, i); - assert(pNodeInfo != NULL); - tjsonGetNumberValue(pNodeInfo, "nodePort", (pCfg->syncCfg.nodeInfo)[i].nodePort, code); - tjsonGetStringValue(pNodeInfo, "nodeFqdn", (pCfg->syncCfg.nodeInfo)[i].nodeFqdn); + SJson *info = tjsonGetArrayItem(nodeInfo, i); + SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i]; + if (info == NULL) return -1; + tjsonGetNumberValue(info, "nodePort", pNode->nodePort, code); + if (code < 0) return -1; + tjsonGetStringValue(info, "nodeFqdn", pNode->nodeFqdn); + tjsonGetNumberValue(info, "nodeId", pNode->nodeId, code); + if (code < 0) return -1; + tjsonGetNumberValue(info, "clusterId", pNode->clusterId, code); + if (code < 0) return -1; + vDebug("vgId:%d, decode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort, + pNode->nodeId); } tjsonGetNumberValue(pJson, "tsdbPageSize", pCfg->tsdbPageSize, code); diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 2e0370a81cd128f5ba42122a417d7faa96525544..6a6fbcff4dedd15599166bd1078468602bae4489 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -21,44 +21,149 @@ static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData); static int vnodeCommitImpl(SCommitInfo *pInfo); -int vnodeBegin(SVnode *pVnode) { - // alloc buffer pool - taosThreadMutexLock(&pVnode->mutex); +#define WAIT_TIME_MILI_SEC 10 // miliseconds + +static int32_t vnodeTryRecycleBufPool(SVnode *pVnode) { + int32_t code = 0; + + if (pVnode->onRecycle == NULL) { + if (pVnode->recycleHead == NULL) { + vDebug("vgId:%d no recyclable buffer pool", TD_VID(pVnode)); + goto _exit; + } else { + vDebug("vgId:%d buffer pool %p of id %d on recycle queue, try to recycle", TD_VID(pVnode), pVnode->recycleHead, + pVnode->recycleHead->id); + + pVnode->onRecycle = pVnode->recycleHead; + if (pVnode->recycleHead == pVnode->recycleTail) { + pVnode->recycleHead = pVnode->recycleTail = NULL; + } else { + pVnode->recycleHead = pVnode->recycleHead->recycleNext; + pVnode->recycleHead->recyclePrev = NULL; + } + pVnode->onRecycle->recycleNext = pVnode->onRecycle->recyclePrev = NULL; + } + } + + code = vnodeBufPoolRecycle(pVnode->onRecycle); + if (code) goto _exit; - while (pVnode->pPool == NULL) { - taosThreadCondWait(&pVnode->poolNotEmpty, &pVnode->mutex); +_exit: + if (code) { + vError("vgId:%d %s failed since %s", TD_VID(pVnode), __func__, tstrerror(code)); } + return code; +} +static int32_t vnodeGetBufPoolToUse(SVnode *pVnode) { + int32_t code = 0; + int32_t lino = 0; + + taosThreadMutexLock(&pVnode->mutex); - pVnode->inUse = pVnode->pPool; - pVnode->inUse->nRef = 1; - pVnode->pPool = pVnode->inUse->next; - pVnode->inUse->next = NULL; + int32_t nTry = 0; + for (;;) { + ++nTry; + + if (pVnode->freeList) { + vDebug("vgId:%d allocate free buffer pool on %d try, pPool:%p id:%d", TD_VID(pVnode), nTry, pVnode->freeList, + pVnode->freeList->id); + + pVnode->inUse = pVnode->freeList; + pVnode->inUse->nRef = 1; + pVnode->freeList = pVnode->inUse->freeNext; + pVnode->inUse->freeNext = NULL; + break; + } else { + vDebug("vgId:%d no free buffer pool on %d try, try to recycle...", TD_VID(pVnode), nTry); + /* + code = vnodeTryRecycleBufPool(pVnode); + TSDB_CHECK_CODE(code, lino, _exit); + */ + if (pVnode->freeList == NULL) { + vDebug("vgId:%d no free buffer pool on %d try, wait %d ms...", TD_VID(pVnode), nTry, WAIT_TIME_MILI_SEC); + + struct timeval tv; + struct timespec ts; + taosGetTimeOfDay(&tv); + ts.tv_nsec = tv.tv_usec * 1000 + WAIT_TIME_MILI_SEC * 1000000; + if (ts.tv_nsec > 999999999l) { + ts.tv_sec = tv.tv_sec + 1; + ts.tv_nsec -= 1000000000l; + } else { + ts.tv_sec = tv.tv_sec; + } + + int32_t rc = taosThreadCondTimedWait(&pVnode->poolNotEmpty, &pVnode->mutex, &ts); + if (rc && rc != ETIMEDOUT) { + code = TAOS_SYSTEM_ERROR(rc); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + } + } +_exit: taosThreadMutexUnlock(&pVnode->mutex); + if (code) { + vError("vgId:%d %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } + return code; +} +int vnodeBegin(SVnode *pVnode) { + int32_t code = 0; + int32_t lino = 0; pVnode->state.commitID++; + + // alloc buffer pool + code = vnodeGetBufPoolToUse(pVnode); + TSDB_CHECK_CODE(code, lino, _exit); + // begin meta if (metaBegin(pVnode->pMeta, META_BEGIN_HEAP_BUFFERPOOL) < 0) { - vError("vgId:%d, failed to begin meta since %s", TD_VID(pVnode), tstrerror(terrno)); - return -1; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } // begin tsdb if (tsdbBegin(pVnode->pTsdb) < 0) { - vError("vgId:%d, failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); - return -1; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } // begin sma if (VND_IS_RSMA(pVnode) && smaBegin(pVnode->pSma) < 0) { - vError("vgId:%d, failed to begin sma since %s", TD_VID(pVnode), tstrerror(terrno)); - return -1; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } - return 0; +_exit: + if (code) { + terrno = code; + vError("vgId:%d %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } + return code; +} + +void vnodeUpdCommitSched(SVnode *pVnode) { + int64_t randNum = taosRand(); + pVnode->commitSched.commitMs = taosGetMonoTimestampMs(); + pVnode->commitSched.maxWaitMs = tsVndCommitMaxIntervalMs + (randNum % tsVndCommitMaxIntervalMs); } int vnodeShouldCommit(SVnode *pVnode) { + if (!pVnode->inUse || !osDataSpaceAvailable()) { + return false; + } + + SVCommitSched *pSched = &pVnode->commitSched; + int64_t nowMs = taosGetMonoTimestampMs(); + + return (((pVnode->inUse->size > pVnode->inUse->node.size) && (pSched->commitMs + SYNC_VND_COMMIT_MIN_MS < nowMs)) || + (pVnode->inUse->size > 0 && pSched->commitMs + pSched->maxWaitMs < nowMs)); +} + +int vnodeShouldCommitOld(SVnode *pVnode) { if (pVnode->inUse) { return osDataSpaceAvailable() && (pVnode->inUse->size > pVnode->inUse->node.size); } @@ -105,8 +210,8 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { // free info binary taosMemoryFree(data); - vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d", pInfo->config.vgId, fname, - pInfo->config.syncCfg.replicaNum); + vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d selfIndex:%d", pInfo->config.vgId, fname, + pInfo->config.syncCfg.replicaNum, pInfo->config.syncCfg.myIndex); return 0; @@ -184,66 +289,133 @@ _err: return -1; } -static int32_t vnodePrepareCommit(SVnode *pVnode) { +static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) { int32_t code = 0; + int32_t lino = 0; + char dir[TSDB_FILENAME_LEN] = {0}; + tsem_wait(&pVnode->canCommit); + taosThreadMutexLock(&pVnode->mutex); + ASSERT(pVnode->onCommit == NULL); + pVnode->onCommit = pVnode->inUse; + pVnode->inUse = NULL; + taosThreadMutexUnlock(&pVnode->mutex); + + pVnode->state.commitTerm = pVnode->state.applyTerm; + + pInfo->info.config = pVnode->config; + pInfo->info.state.committed = pVnode->state.applied; + pInfo->info.state.commitTerm = pVnode->state.applyTerm; + pInfo->info.state.commitID = pVnode->state.commitID; + pInfo->pVnode = pVnode; + pInfo->txn = metaGetTxn(pVnode->pMeta); + + // save info + if (pVnode->pTfs) { + snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); + } else { + snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); + } + + vDebug("vgId:%d, save config while prepare commit", TD_VID(pVnode)); + if (vnodeSaveInfo(dir, &pInfo->info) < 0) { + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + } + tsdbPrepareCommit(pVnode->pTsdb); + metaPrepareAsyncCommit(pVnode->pMeta); + code = smaPrepareAsyncCommit(pVnode->pSma); if (code) goto _exit; _exit: - vnodeBufPoolUnRef(pVnode->inUse); - pVnode->inUse = NULL; + if (code) { + vError("vgId:%d, %s failed at line %d since %s, commit id:%" PRId64, TD_VID(pVnode), __func__, lino, + tstrerror(code), pVnode->state.commitID); + } else { + vDebug("vgId:%d, %s done", TD_VID(pVnode), __func__); + } + return code; } +static void vnodeReturnBufPool(SVnode *pVnode) { + taosThreadMutexLock(&pVnode->mutex); + SVBufPool *pPool = pVnode->onCommit; + int32_t nRef = atomic_sub_fetch_32(&pPool->nRef, 1); + + pVnode->onCommit = NULL; + if (nRef == 0) { + vnodeBufPoolAddToFreeList(pPool); + } else if (nRef > 0) { + vDebug("vgId:%d buffer pool %p of id %d is added to recycle queue", TD_VID(pVnode), pPool, pPool->id); + + if (pVnode->recycleTail == NULL) { + pPool->recyclePrev = pPool->recycleNext = NULL; + pVnode->recycleHead = pVnode->recycleTail = pPool; + } else { + pPool->recyclePrev = pVnode->recycleTail; + pPool->recycleNext = NULL; + pVnode->recycleTail->recycleNext = pPool; + pVnode->recycleTail = pPool; + } + } else { + ASSERT(0); + } + + taosThreadMutexUnlock(&pVnode->mutex); +} static int32_t vnodeCommitTask(void *arg) { int32_t code = 0; SCommitInfo *pInfo = (SCommitInfo *)arg; + SVnode *pVnode = pInfo->pVnode; // commit code = vnodeCommitImpl(pInfo); if (code) goto _exit; + vnodeReturnBufPool(pVnode); + _exit: // end commit - tsem_post(&pInfo->pVnode->canCommit); + tsem_post(&pVnode->canCommit); taosMemoryFree(pInfo); return code; } + int vnodeAsyncCommit(SVnode *pVnode) { int32_t code = 0; - // prepare to commit - code = vnodePrepareCommit(pVnode); - if (code) goto _exit; - - // schedule the task - pVnode->state.commitTerm = pVnode->state.applyTerm; - SCommitInfo *pInfo = (SCommitInfo *)taosMemoryCalloc(1, sizeof(*pInfo)); if (NULL == pInfo) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - pInfo->info.config = pVnode->config; - pInfo->info.state.committed = pVnode->state.applied; - pInfo->info.state.commitTerm = pVnode->state.applyTerm; - pInfo->info.state.commitID = pVnode->state.commitID; - pInfo->pVnode = pVnode; - pInfo->txn = metaGetTxn(pVnode->pMeta); + + // prepare to commit + code = vnodePrepareCommit(pVnode, pInfo); + if (TSDB_CODE_SUCCESS != code) { + goto _exit; + } + + // schedule the task code = vnodeScheduleTask(vnodeCommitTask, pInfo); _exit: if (code) { + if (NULL != pInfo) { + taosMemoryFree(pInfo); + } tsem_post(&pVnode->canCommit); vError("vgId:%d, %s failed since %s, commit id:%" PRId64, TD_VID(pVnode), __func__, tstrerror(code), pVnode->state.commitID); } else { - vDebug("vgId:%d, %s done", TD_VID(pVnode), __func__); + vInfo("vgId:%d, vnode async commit done, commitId:%" PRId64 " term:%" PRId64 " applied:%" PRId64, TD_VID(pVnode), + pVnode->state.commitID, pVnode->state.applyTerm, pVnode->state.applied); } return code; } @@ -262,8 +434,10 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) { char dir[TSDB_FILENAME_LEN] = {0}; SVnode *pVnode = pInfo->pVnode; - vInfo("vgId:%d, start to commit, commit ID:%" PRId64 " version:%" PRId64 " term: %" PRId64, TD_VID(pVnode), - pVnode->state.commitID, pVnode->state.applied, pVnode->state.applyTerm); + vInfo("vgId:%d, start to commit, commitId:%" PRId64 " version:%" PRId64 " term: %" PRId64, TD_VID(pVnode), + pInfo->info.state.commitID, pInfo->info.state.committed, pInfo->info.state.commitTerm); + + vnodeUpdCommitSched(pVnode); // persist wal before starting if (walPersist(pVnode->pWal) < 0) { @@ -271,19 +445,13 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) { return -1; } - // save info if (pVnode->pTfs) { snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); } else { snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); } - if (vnodeSaveInfo(dir, &pInfo->info) < 0) { - code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); - } - // walBeginSnapshot(pVnode->pWal, pVnode->state.applied); - syncBeginSnapshot(pVnode->sync, pVnode->state.applied); + syncBeginSnapshot(pVnode->sync, pInfo->info.state.committed); // commit each sub-system code = tsdbCommit(pVnode->pTsdb, pInfo); @@ -325,7 +493,6 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) { return -1; } - // walEndSnapshot(pVnode->pWal); syncEndSnapshot(pVnode->sync); _exit: diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index e09fafb75690f13dd160763126d139dd21ae8c1b..743ae17f2b87940cdd381f3ac2ebdb443f4e2e46 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -48,6 +48,7 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { info.state.applied = -1; info.state.commitID = 0; + vInfo("vgId:%d, save config while create", pCfg->vgId); if (vnodeSaveInfo(dir, &info) < 0 || vnodeCommitInfo(dir, &info) < 0) { vError("vgId:%d, failed to save vnode config since %s", pCfg ? pCfg->vgId : 0, tstrerror(terrno)); return -1; @@ -79,12 +80,14 @@ int32_t vnodeAlter(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs) { pCfg->replicaNum = pReq->replica; memset(&pCfg->nodeInfo, 0, sizeof(pCfg->nodeInfo)); - vInfo("vgId:%d, save config, replicas:%d selfIndex:%d", pReq->vgId, pCfg->replicaNum, pCfg->myIndex); + vInfo("vgId:%d, save config while alter, replicas:%d selfIndex:%d", pReq->vgId, pCfg->replicaNum, pCfg->myIndex); for (int i = 0; i < pReq->replica; ++i) { SNodeInfo *pNode = &pCfg->nodeInfo[i]; + pNode->nodeId = pReq->replicas[i].id; pNode->nodePort = pReq->replicas[i].port; tstrncpy(pNode->nodeFqdn, pReq->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); - vInfo("vgId:%d, save config, replica:%d ep:%s:%u", pReq->vgId, i, pNode->nodeFqdn, pNode->nodePort); + tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + vInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", pReq->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); } info.config.syncCfg = *pCfg; @@ -157,6 +160,8 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { taosThreadMutexInit(&pVnode->mutex, NULL); taosThreadCondInit(&pVnode->poolNotEmpty, NULL); + vnodeUpdCommitSched(pVnode); + int8_t rollback = vnodeShouldRollback(pVnode); // open buffer pool @@ -235,7 +240,7 @@ _err: if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb); if (pVnode->pSma) smaClose(pVnode->pSma); if (pVnode->pMeta) metaClose(pVnode->pMeta); - if (pVnode->pPool) vnodeCloseBufPool(pVnode); + if (pVnode->freeList) vnodeCloseBufPool(pVnode); tsem_destroy(&(pVnode->canCommit)); taosMemoryFree(pVnode); @@ -247,9 +252,11 @@ void vnodePreClose(SVnode *pVnode) { vnodeSyncPreClose(pVnode); } +void vnodePostClose(SVnode *pVnode) { vnodeSyncPostClose(pVnode); } + void vnodeClose(SVnode *pVnode) { if (pVnode) { - vnodeSyncCommit(pVnode); + tsem_wait(&pVnode->canCommit); vnodeSyncClose(pVnode); vnodeQueryClose(pVnode); walClose(pVnode->pWal); @@ -258,6 +265,8 @@ void vnodeClose(SVnode *pVnode) { smaClose(pVnode->pSma); metaClose(pVnode->pMeta); vnodeCloseBufPool(pVnode); + tsem_post(&pVnode->canCommit); + // destroy handle tsem_destroy(&(pVnode->canCommit)); tsem_destroy(&pVnode->syncSem); diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index fcfacd1ca9d6a369e0290863b86c96c925b4174c..e75dc24329c03ade45e12242fd70a62b963f74ff 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -257,8 +257,8 @@ _exit: pReader->index++; *nData = sizeof(SSnapDataHdr) + pHdr->size; pHdr->index = pReader->index; - vInfo("vgId:%d, vnode snapshot read data,index:%" PRId64 " type:%d nData:%d ", TD_VID(pReader->pVnode), - pReader->index, pHdr->type, *nData); + vDebug("vgId:%d, vnode snapshot read data, index:%" PRId64 " type:%d blockLen:%d ", TD_VID(pReader->pVnode), + pReader->index, pHdr->type, *nData); } else { vInfo("vgId:%d, vnode snapshot read data end, index:%" PRId64, TD_VID(pReader->pVnode), pReader->index); } @@ -405,6 +405,12 @@ static int32_t vnodeSnapWriteInfo(SVSnapWriter *pWriter, uint8_t *pData, uint32_ } else { snprintf(dir, TSDB_FILENAME_LEN, "%s", pWriter->pVnode->path); } + + SVnodeStats vndStats = pWriter->info.config.vndStats; + SVnode *pVnode = pWriter->pVnode; + pWriter->info.config = pVnode->config; + pWriter->info.config.vndStats = vndStats; + vDebug("vgId:%d, save config while write snapshot", pWriter->pVnode->config.vgId); if (vnodeSaveInfo(dir, &pWriter->info) < 0) { code = terrno; goto _exit; @@ -423,8 +429,8 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { ASSERT(pHdr->index == pWriter->index + 1); pWriter->index = pHdr->index; - vInfo("vgId:%d, vnode snapshot write data, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode), pHdr->index, - pHdr->type, nData); + vDebug("vgId:%d, vnode snapshot write data, index:%" PRId64 " type:%d blockLen:%d", TD_VID(pVnode), pHdr->index, + pHdr->type, nData); switch (pHdr->type) { case SNAP_DATA_CFG: { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index d719a349671c8ba0d32f12e00ff068b113e2add5..6abc144b911720244e33b3553249b98b48550af7 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -77,52 +77,72 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { tDecoderClear(&dc); } break; case TDMT_VND_SUBMIT: { - SSubmitMsgIter msgIter = {0}; - SSubmitReq *pSubmitReq = (SSubmitReq *)pMsg->pCont; - SSubmitBlk *pBlock = NULL; - int64_t ctime = taosGetTimestampMs(); - tb_uid_t uid; - - if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { - code = terrno; + int64_t ctime = taosGetTimestampMs(); + + tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); + tStartDecode(&dc); + + uint64_t nSubmitTbData; + if (tDecodeU64v(&dc, &nSubmitTbData) < 0) { + code = TSDB_CODE_INVALID_MSG; goto _err; } - for (;;) { - tGetSubmitMsgNext(&msgIter, &pBlock); - if (pBlock == NULL) break; + for (int32_t i = 0; i < nSubmitTbData; i++) { + if (tStartDecode(&dc) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _err; + } - if (msgIter.schemaLen > 0) { - char *name = NULL; + int32_t flags; + if (tDecodeI32v(&dc, &flags) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _err; + } - tDecoderInit(&dc, pBlock->data, msgIter.schemaLen); + if (flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { + // SVCreateTbReq if (tStartDecode(&dc) < 0) { code = TSDB_CODE_INVALID_MSG; - return code; + goto _err; } if (tDecodeI32v(&dc, NULL) < 0) { code = TSDB_CODE_INVALID_MSG; - return code; + goto _err; } + + char *name = NULL; if (tDecodeCStr(&dc, &name) < 0) { code = TSDB_CODE_INVALID_MSG; - return code; + goto _err; } - uid = metaGetTableEntryUidByName(pVnode->pMeta, name); + int64_t uid = metaGetTableEntryUidByName(pVnode->pMeta, name); if (uid == 0) { uid = tGenIdPI64(); } + *(int64_t *)(dc.data + dc.pos) = uid; *(int64_t *)(dc.data + dc.pos + 8) = ctime; - pBlock->uid = htobe64(uid); tEndDecode(&dc); - tDecoderClear(&dc); + + // SSubmitTbData + int64_t suid; + if (tDecodeI64(&dc, &suid) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _err; + } + + *(int64_t *)(dc.data + dc.pos) = uid; } + + tEndDecode(&dc); } + tEndDecode(&dc); + tDecoderClear(&dc); } break; case TDMT_VND_DELETE: { int32_t size; @@ -171,18 +191,17 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp void *pReq; int32_t len; int32_t ret; - + /* if (!pVnode->inUse) { terrno = TSDB_CODE_VND_NO_AVAIL_BUFPOOL; vError("vgId:%d, not ready to write since %s", TD_VID(pVnode), terrstr()); return -1; } - + */ if (version <= pVnode->state.applied) { vError("vgId:%d, duplicate write request. version: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), version, pVnode->state.applied); terrno = TSDB_CODE_VND_DUP_REQUEST; - pRsp->info.handle = NULL; return -1; } @@ -197,13 +216,14 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp if (!syncUtilUserCommit(pMsg->msgType)) goto _exit; - if (pMsg->msgType == TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE) { + if (pMsg->msgType == TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE || pMsg->msgType == TDMT_STREAM_TASK_CHECK_RSP) { if (tqCheckLogInWal(pVnode->pTq, version)) return 0; } // skip header pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); len = pMsg->contLen - sizeof(SMsgHead); + bool needCommit = false; switch (pMsg->msgType) { /* META */ @@ -236,7 +256,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp break; /* TSDB */ case TDMT_VND_SUBMIT: - if (vnodeProcessSubmitReq(pVnode, version, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err; + if (vnodeProcessSubmitReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DELETE: if (vnodeProcessDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; @@ -300,9 +320,8 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp vnodeProcessAlterConfigReq(pVnode, version, pReq, len, pRsp); break; case TDMT_VND_COMMIT: - vnodeSyncCommit(pVnode); - vnodeBegin(pVnode); - goto _exit; + needCommit = true; + break; default: vError("vgId:%d, unprocessed msg, %d", TD_VID(pVnode), pMsg->msgType); return -1; @@ -319,7 +338,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp } // commit if need - if (vnodeShouldCommit(pVnode)) { + if (needCommit) { vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); if (vnodeAsyncCommit(pVnode) < 0) { vError("vgId:%d, failed to vnode async commit since %s.", TD_VID(pVnode), tstrerror(terrno)); @@ -855,57 +874,199 @@ static int32_t vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, return TSDB_CODE_SUCCESS; } -static int32_t vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char *tags) { - ASSERT(pMsg != NULL); - SSubmitMsgIter msgIter = {0}; - SMeta *pMeta = pVnode->pMeta; - SSubmitBlk *pBlock = NULL; +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +#if 1 + int32_t code = 0; + terrno = 0; - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; - while (true) { - if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; - if (pBlock == NULL) break; + SSubmitReq2 *pSubmitReq = &(SSubmitReq2){0}; + SSubmitRsp2 *pSubmitRsp = &(SSubmitRsp2){0}; + SArray *newTbUids = NULL; + int32_t ret; + SEncoder ec = {0}; + + pRsp->code = TSDB_CODE_SUCCESS; + + // decode + SDecoder dc = {0}; + tDecoderInit(&dc, pReq, len); + if (tDecodeSSubmitReq2(&dc, pSubmitReq) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + tDecoderClear(&dc); - vnodeDebugPrintSingleSubmitMsg(pMeta, pBlock, &msgIter, tags); + // check + code = tsdbScanAndConvertSubmitMsg(pVnode->pTsdb, pSubmitReq); + if (code) { + goto _exit; } - return 0; -} + for (int32_t i = 0; i < TARRAY_SIZE(pSubmitReq->aSubmitTbData); ++i) { + SSubmitTbData *pSubmitTbData = taosArrayGet(pSubmitReq->aSubmitTbData, i); -static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; - SSubmitRsp submitRsp = {0}; - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SVCreateTbReq createTbReq = {0}; - SDecoder decoder = {0}; - int32_t nRows = 0; - int32_t tsize, ret; - SEncoder encoder = {0}; - SArray *newTbUids = NULL; - SVStatis statis = {0}; - bool tbCreated = false; + if (pSubmitTbData->pCreateTbReq) { + pSubmitTbData->uid = pSubmitTbData->pCreateTbReq->uid; + } else { + SMetaInfo info = {0}; + + code = metaGetInfo(pVnode->pMeta, pSubmitTbData->uid, &info, NULL); + if (code) { + code = TSDB_CODE_TDB_TABLE_NOT_EXIST; + vWarn("vgId:%d, table uid:%" PRId64 " not exists", TD_VID(pVnode), pSubmitTbData->uid); + goto _exit; + } + + if (info.suid != pSubmitTbData->suid) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (info.suid) { + metaGetInfo(pVnode->pMeta, info.suid, &info, NULL); + } + + if (pSubmitTbData->sver != info.skmVer) { + code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; + goto _exit; + } + } + + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + int32_t nColData = TARRAY_SIZE(pSubmitTbData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pSubmitTbData->aCol); + + if (nColData <= 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (aColData[0].cid != PRIMARYKEY_TIMESTAMP_COL_ID || aColData[0].type != TSDB_DATA_TYPE_TIMESTAMP || + aColData[0].nVal <= 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + for (int32_t i = 1; i < nColData; i++) { + if (aColData[i].nVal != aColData[0].nVal) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + } + } + } + + vDebug("vgId:%d, submit block size %d", TD_VID(pVnode), (int32_t)taosArrayGetSize(pSubmitReq->aSubmitTbData)); + + // loop to handle + for (int32_t i = 0; i < TARRAY_SIZE(pSubmitReq->aSubmitTbData); ++i) { + SSubmitTbData *pSubmitTbData = taosArrayGet(pSubmitReq->aSubmitTbData, i); + + // create table + if (pSubmitTbData->pCreateTbReq) { + // check (TODO: move check to create table) + code = grantCheck(TSDB_GRANT_TIMESERIES); + if (code) goto _exit; + + code = grantCheck(TSDB_GRANT_TABLE); + if (code) goto _exit; + + // alloc if need + if (pSubmitRsp->aCreateTbRsp == NULL && + (pSubmitRsp->aCreateTbRsp = taosArrayInit(TARRAY_SIZE(pSubmitReq->aSubmitTbData), sizeof(SVCreateTbRsp))) == + NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + SVCreateTbRsp *pCreateTbRsp = taosArrayReserve(pSubmitRsp->aCreateTbRsp, 1); + + // create table + if (metaCreateTable(pVnode->pMeta, version, pSubmitTbData->pCreateTbReq, &pCreateTbRsp->pMeta) == + 0) { // create table success + + if (newTbUids == NULL && + (newTbUids = taosArrayInit(TARRAY_SIZE(pSubmitReq->aSubmitTbData), sizeof(int64_t))) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + taosArrayPush(newTbUids, &pSubmitTbData->uid); + + if (pCreateTbRsp->pMeta) { + vnodeUpdateMetaRsp(pVnode, pCreateTbRsp->pMeta); + } + } else { // create table failed + if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { + code = terrno; + goto _exit; + } + pSubmitTbData->uid = pSubmitTbData->pCreateTbReq->uid; // update uid if table exist for using below + } + } + + // insert data + int32_t affectedRows; + code = tsdbInsertTableData(pVnode->pTsdb, version, pSubmitTbData, &affectedRows); + if (code) goto _exit; + + pSubmitRsp->affectedRows += affectedRows; + } + + // update table uid list + if (taosArrayGetSize(newTbUids) > 0) { + vDebug("vgId:%d, add %d table into query table list in handling submit", TD_VID(pVnode), + (int32_t)taosArrayGetSize(newTbUids)); + tqUpdateTbUidList(pVnode->pTq, newTbUids, true); + } + +_exit: + // message + pRsp->code = code; + tEncodeSize(tEncodeSSubmitRsp2, pSubmitRsp, pRsp->contLen, ret); + pRsp->pCont = rpcMallocCont(pRsp->contLen); + tEncoderInit(&ec, pRsp->pCont, pRsp->contLen); + tEncodeSSubmitRsp2(&ec, pSubmitRsp); + tEncoderClear(&ec); + + // update statistics + atomic_add_fetch_64(&pVnode->statis.nInsert, pSubmitRsp->affectedRows); + atomic_add_fetch_64(&pVnode->statis.nInsertSuccess, pSubmitRsp->affectedRows); + atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1); + if (code == 0) { + atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1); + tdProcessRSmaSubmit(pVnode->pSma, version, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT); + } + + // clear + taosArrayDestroy(newTbUids); + tDestroySSubmitReq2(pSubmitReq, TSDB_MSG_FLG_DECODE); + tDestroySSubmitRsp2(pSubmitRsp, TSDB_MSG_FLG_ENCODE); + + if (code) terrno = code; + + return code; + +#else + SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; + SSubmitRsp submitRsp = {0}; + int32_t nRows = 0; + int32_t tsize, ret; + SEncoder encoder = {0}; + SArray *newTbUids = NULL; + SVStatis statis = {0}; + bool tbCreated = false; terrno = TSDB_CODE_SUCCESS; pRsp->code = 0; pSubmitReq->version = version; statis.nBatchInsert = 1; -#ifdef TD_DEBUG_PRINT_ROW - vnodeDebugPrintSubmitMsg(pVnode, pReq, __func__); -#endif - if (tsdbScanAndConvertSubmitMsg(pVnode->pTsdb, pSubmitReq) < 0) { pRsp->code = terrno; goto _exit; } - // handle the request - if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { - pRsp->code = TSDB_CODE_INVALID_MSG; - goto _exit; - } - submitRsp.pArray = taosArrayInit(msgIter.numOfBlocks, sizeof(SSubmitBlkRsp)); newTbUids = taosArrayInit(msgIter.numOfBlocks, sizeof(int64_t)); if (!submitRsp.pArray || !newTbUids) { @@ -922,42 +1083,42 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq // create table for auto create table mode if (msgIter.schemaLen > 0) { - tDecoderInit(&decoder, pBlock->data, msgIter.schemaLen); - if (tDecodeSVCreateTbReq(&decoder, &createTbReq) < 0) { - pRsp->code = TSDB_CODE_INVALID_MSG; - tDecoderClear(&decoder); - taosArrayDestroy(createTbReq.ctb.tagName); - goto _exit; - } - - if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0) { - pRsp->code = terrno; - tDecoderClear(&decoder); - taosArrayDestroy(createTbReq.ctb.tagName); - goto _exit; - } - - if ((terrno = grantCheck(TSDB_GRANT_TABLE)) < 0) { - pRsp->code = terrno; - tDecoderClear(&decoder); - taosArrayDestroy(createTbReq.ctb.tagName); - goto _exit; - } + // tDecoderInit(&decoder, pBlock->data, msgIter.schemaLen); + // if (tDecodeSVCreateTbReq(&decoder, &createTbReq) < 0) { + // pRsp->code = TSDB_CODE_INVALID_MSG; + // tDecoderClear(&decoder); + // taosArrayDestroy(createTbReq.ctb.tagName); + // goto _exit; + // } + + // if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0) { + // pRsp->code = terrno; + // tDecoderClear(&decoder); + // taosArrayDestroy(createTbReq.ctb.tagName); + // goto _exit; + // } + + // if ((terrno = grantCheck(TSDB_GRANT_TABLE)) < 0) { + // pRsp->code = terrno; + // tDecoderClear(&decoder); + // taosArrayDestroy(createTbReq.ctb.tagName); + // goto _exit; + // } if (metaCreateTable(pVnode->pMeta, version, &createTbReq, &submitBlkRsp.pMeta) < 0) { - if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { - submitBlkRsp.code = terrno; - pRsp->code = terrno; - tDecoderClear(&decoder); - taosArrayDestroy(createTbReq.ctb.tagName); - goto _exit; - } + // if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { + // submitBlkRsp.code = terrno; + // pRsp->code = terrno; + // tDecoderClear(&decoder); + // taosArrayDestroy(createTbReq.ctb.tagName); + // goto _exit; + // } } else { if (NULL != submitBlkRsp.pMeta) { vnodeUpdateMetaRsp(pVnode, submitBlkRsp.pMeta); } - taosArrayPush(newTbUids, &createTbReq.uid); + // taosArrayPush(newTbUids, &createTbReq.uid); submitBlkRsp.uid = createTbReq.uid; submitBlkRsp.tblFName = taosMemoryMalloc(strlen(pVnode->config.dbname) + strlen(createTbReq.name) + 2); @@ -965,18 +1126,15 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq tbCreated = true; } - msgIter.uid = createTbReq.uid; - if (createTbReq.type == TSDB_CHILD_TABLE) { - msgIter.suid = createTbReq.ctb.suid; - } else { - msgIter.suid = 0; - } + // msgIter.uid = createTbReq.uid; + // if (createTbReq.type == TSDB_CHILD_TABLE) { + // msgIter.suid = createTbReq.ctb.suid; + // } else { + // msgIter.suid = 0; + // } -#ifdef TD_DEBUG_PRINT_ROW - vnodeDebugPrintSingleSubmitMsg(pVnode->pMeta, pBlock, &msgIter, "real uid"); -#endif - tDecoderClear(&decoder); - taosArrayDestroy(createTbReq.ctb.tagName); + // tDecoderClear(&decoder); + // taosArrayDestroy(createTbReq.ctb.tagName); } if (tsdbInsertTableData(pVnode->pTsdb, version, &msgIter, pBlock, &submitBlkRsp) < 0) { @@ -990,21 +1148,21 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq } } - if (taosArrayGetSize(newTbUids) > 0) { - vDebug("vgId:%d, add %d table into query table list in handling submit", TD_VID(pVnode), - (int32_t)taosArrayGetSize(newTbUids)); - } + // if (taosArrayGetSize(newTbUids) > 0) { + // vDebug("vgId:%d, add %d table into query table list in handling submit", TD_VID(pVnode), + // (int32_t)taosArrayGetSize(newTbUids)); + // } - tqUpdateTbUidList(pVnode->pTq, newTbUids, true); + // tqUpdateTbUidList(pVnode->pTq, newTbUids, true); _exit: taosArrayDestroy(newTbUids); - tEncodeSize(tEncodeSSubmitRsp, &submitRsp, tsize, ret); - pRsp->pCont = rpcMallocCont(tsize); - pRsp->contLen = tsize; - tEncoderInit(&encoder, pRsp->pCont, tsize); - tEncodeSSubmitRsp(&encoder, &submitRsp); - tEncoderClear(&encoder); + // tEncodeSize(tEncodeSSubmitRsp, &submitRsp, tsize, ret); + // pRsp->pCont = rpcMallocCont(tsize); + // pRsp->contLen = tsize; + // tEncoderInit(&encoder, pRsp->pCont, tsize); + // tEncodeSSubmitRsp(&encoder, &submitRsp); + // tEncoderClear(&encoder); taosArrayDestroyEx(submitRsp.pArray, tFreeSSubmitBlkRsp); @@ -1025,6 +1183,8 @@ _exit: vDebug("vgId:%d, submit success, index:%" PRId64, pVnode->config.vgId, version); return 0; +#endif + return 0; } static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 5caaae502f14f58b6cb464a4c503ce526fc1c8cb..6f3788616a5f45e440d07f23b89e3e493d07fa4d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -101,6 +101,64 @@ static void vnodeHandleProposeError(SVnode *pVnode, SRpcMsg *pMsg, int32_t code) } } +static int32_t inline vnodeProposeMsg(SVnode *pVnode, SRpcMsg *pMsg, bool isWeak) { + int64_t seq = 0; + + taosThreadMutexLock(&pVnode->lock); + int32_t code = syncPropose(pVnode->sync, pMsg, isWeak, &seq); + bool wait = (code == 0 && vnodeIsMsgBlock(pMsg->msgType)); + if (wait) { + ASSERT(!pVnode->blocked); + pVnode->blocked = true; + pVnode->blockSec = taosGetTimestampSec(); + pVnode->blockSeq = seq; +#if 0 + pVnode->blockInfo = pMsg->info; +#endif + } + taosThreadMutexUnlock(&pVnode->lock); + + if (code > 0) { + vnodeHandleWriteMsg(pVnode, pMsg); + } else if (code < 0) { + if (terrno != 0) code = terrno; + vnodeHandleProposeError(pVnode, pMsg, code); + } + + if (wait) vnodeWaitBlockMsg(pVnode, pMsg); + return code; +} + +void vnodeProposeCommitOnNeed(SVnode *pVnode) { + if (!vnodeShouldCommit(pVnode)) { + return; + } + + int32_t contLen = sizeof(SMsgHead); + SMsgHead *pHead = rpcMallocCont(contLen); + pHead->contLen = contLen; + pHead->vgId = pVnode->config.vgId; + + SRpcMsg rpcMsg = {0}; + rpcMsg.msgType = TDMT_VND_COMMIT; + rpcMsg.contLen = contLen; + rpcMsg.pCont = pHead; + rpcMsg.info.noResp = 1; + + bool isWeak = false; + if (vnodeProposeMsg(pVnode, &rpcMsg, isWeak) < 0) { + vTrace("vgId:%d, failed to propose vnode commit since %s", pVnode->config.vgId, terrstr()); + goto _out; + } + + vInfo("vgId:%d, proposed vnode commit", pVnode->config.vgId); + +_out: + vnodeUpdCommitSched(pVnode); + rpcFreeCont(rpcMsg.pCont); + rpcMsg.pCont = NULL; +} + #if BATCH_ENABLE static void inline vnodeProposeBatchMsg(SVnode *pVnode, SRpcMsg **pMsgArr, bool *pIsWeakArr, int32_t *arrSize) { @@ -178,6 +236,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) continue; } + vnodeProposeCommitOnNeed(pVnode); + code = vnodePreProcessWriteMsg(pVnode, pMsg); if (code != 0) { vGError("vgId:%d, msg:%p failed to pre-process since %s", vgId, pMsg, terrstr()); @@ -205,34 +265,6 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) #else -static int32_t inline vnodeProposeMsg(SVnode *pVnode, SRpcMsg *pMsg, bool isWeak) { - int64_t seq = 0; - - taosThreadMutexLock(&pVnode->lock); - int32_t code = syncPropose(pVnode->sync, pMsg, isWeak, &seq); - bool wait = (code == 0 && vnodeIsMsgBlock(pMsg->msgType)); - if (wait) { - ASSERT(!pVnode->blocked); - pVnode->blocked = true; - pVnode->blockSec = taosGetTimestampSec(); - pVnode->blockSeq = seq; -#if 0 - pVnode->blockInfo = pMsg->info; -#endif - } - taosThreadMutexUnlock(&pVnode->lock); - - if (code > 0) { - vnodeHandleWriteMsg(pVnode, pMsg); - } else if (code < 0) { - if (terrno != 0) code = terrno; - vnodeHandleProposeError(pVnode, pMsg, code); - } - - if (wait) vnodeWaitBlockMsg(pVnode, pMsg); - return code; -} - void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnode *pVnode = pInfo->ahandle; int32_t vgId = pVnode->config.vgId; @@ -256,6 +288,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) continue; } + vnodeProposeCommitOnNeed(pVnode); + code = vnodePreProcessWriteMsg(pVnode, pMsg); if (code != 0) { vGError("vgId:%d, msg:%p failed to pre-process since %s", vgId, pMsg, terrstr()); @@ -391,9 +425,9 @@ static int32_t vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsm const STraceId *trace = &pMsg->info.traceId; vGTrace("vgId:%d, commit-cb is excuted, fsm:%p, index:%" PRId64 ", term:%" PRIu64 ", msg-index:%" PRId64 - ", weak:%d, code:%d, state:%d %s, type:%s", + ", weak:%d, code:%d, state:%d %s, type:%s code:0x%x", pVnode->config.vgId, pFsm, pMeta->index, pMeta->term, pMsg->info.conn.applyIndex, pMeta->isWeak, pMeta->code, - pMeta->state, syncStr(pMeta->state), TMSG_INFO(pMsg->msgType)); + pMeta->state, syncStr(pMeta->state), TMSG_INFO(pMsg->msgType), pMsg->code); return tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, pMsg); } @@ -465,9 +499,9 @@ static int32_t vnodeSnapshotStopWrite(const SSyncFSM *pFsm, void *pWriter, bool static int32_t vnodeSnapshotDoWrite(const SSyncFSM *pFsm, void *pWriter, void *pBuf, int32_t len) { SVnode *pVnode = pFsm->data; - vDebug("vgId:%d, continue write vnode snapshot, len:%d", pVnode->config.vgId, len); + vDebug("vgId:%d, continue write vnode snapshot, blockLen:%d", pVnode->config.vgId, len); int32_t code = vnodeSnapWrite(pWriter, pBuf, len); - vDebug("vgId:%d, continue write vnode snapshot finished, len:%d", pVnode->config.vgId, len); + vDebug("vgId:%d, continue write vnode snapshot finished, blockLen:%d", pVnode->config.vgId, len); return code; } @@ -578,7 +612,8 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { vInfo("vgId:%d, start to open sync, replica:%d selfIndex:%d", pVnode->config.vgId, pCfg->replicaNum, pCfg->myIndex); for (int32_t i = 0; i < pCfg->replicaNum; ++i) { SNodeInfo *pNode = &pCfg->nodeInfo[i]; - vInfo("vgId:%d, index:%d ep:%s:%u", pVnode->config.vgId, i, pNode->nodeFqdn, pNode->nodePort); + vInfo("vgId:%d, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, pVnode->config.vgId, i, pNode->nodeFqdn, pNode->nodePort, + pNode->nodeId, pNode->clusterId); } pVnode->sync = syncOpen(&syncInfo); @@ -613,6 +648,11 @@ void vnodeSyncPreClose(SVnode *pVnode) { taosThreadMutexUnlock(&pVnode->lock); } +void vnodeSyncPostClose(SVnode *pVnode) { + vInfo("vgId:%d, post close sync", pVnode->config.vgId); + syncPostStop(pVnode->sync); +} + void vnodeSyncClose(SVnode *pVnode) { vInfo("vgId:%d, close sync", pVnode->config.vgId); syncStop(pVnode->sync); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index a6ce613882daeda0ea99898e104cf184b4c5add6..836ce87fbb7bb0620567be05030527c4ca7ac101 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -582,34 +582,34 @@ typedef struct SCtgOperation { #define CTG_LOCK(type, _lock) \ do { \ if (CTG_READ == (type)) { \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before read lock"); \ CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRLockLatch(_lock); \ CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) > 0); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value after read lock"); \ } else { \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before write lock"); \ CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWLockLatch(_lock); \ CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value after write lock"); \ } \ } while (0) #define CTG_UNLOCK(type, _lock) \ do { \ if (CTG_READ == (type)) { \ - assert(atomic_load_32((_lock)) > 0); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value before read unlock"); \ CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRUnLockLatch(_lock); \ CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after read unlock"); \ } else { \ - assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value before write unlock"); \ CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWUnLockLatch(_lock); \ CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after write unlock"); \ } \ } while (0) @@ -776,6 +776,7 @@ void ctgFreeHandleImpl(SCatalog* pCtg); int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup); int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx, char* dbFName, SArray* pNames, bool update); +int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFName, const char* pTbs[], int32_t tbNum, int32_t* vgId); void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache* dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index acd0fb9d8c45e022f2b74bac73671a7b4044e443..c7af0411bea055b5cf6afb13941735ce3c104eaa 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -551,6 +551,37 @@ _return: CTG_RET(code); } +int32_t ctgGetTbsHashVgId(SCatalog* pCtg, SRequestConnInfo* pConn, int32_t acctId, const char* pDb, const char* pTbs[], int32_t tbNum, int32_t* vgId) { + if (IS_SYS_DBNAME(pDb)) { + ctgError("no valid vgInfo for db, dbname:%s", pDb); + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + + SCtgDBCache* dbCache = NULL; + int32_t code = 0; + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + snprintf(dbFName, TSDB_DB_FNAME_LEN, "%d.%s", acctId, pDb); + + SDBVgInfo* vgInfo = NULL; + CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, dbFName, &dbCache, &vgInfo, NULL)); + + CTG_ERR_JRET(ctgGetVgIdsFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgCache.vgInfo, dbFName, pTbs, tbNum, vgId)); + +_return: + + if (dbCache) { + ctgRUnlockVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + } + + if (vgInfo) { + freeVgInfo(vgInfo); + } + + CTG_RET(code); +} + + int32_t ctgGetCachedTbVgMeta(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, STableMeta** pTableMeta) { int32_t code = 0; char db[TSDB_DB_FNAME_LEN] = {0}; @@ -1141,6 +1172,13 @@ int32_t catalogGetTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const CTG_API_LEAVE(ctgGetTbHashVgroup(pCtg, pConn, pTableName, pVgroup, NULL)); } +int32_t catalogGetTablesHashVgId(SCatalog* pCtg, SRequestConnInfo* pConn, int32_t acctId, const char* pDb, const char* pTableName[], + int32_t tableNum, int32_t *vgId) { + CTG_API_ENTER(); + + CTG_API_LEAVE(ctgGetTbsHashVgId(pCtg, pConn, acctId, pDb, pTableName, tableNum, vgId)); +} + int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index acd18fcca5108335a11e731c72e26df80020b993..438128203ef10867f433693940ab4d62e0c65ebd 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -471,17 +471,20 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, con } int32_t ctgInitTask(SCtgJob* pJob, CTG_TASK_TYPE type, void* param, int32_t* taskId) { + int32_t code = 0; int32_t tid = atomic_fetch_add_32(&pJob->taskIdx, 1); CTG_LOCK(CTG_WRITE, &pJob->taskLock); - CTG_ERR_RET((*gCtgAsyncFps[type].initFp)(pJob, tid, param)); - CTG_UNLOCK(CTG_WRITE, &pJob->taskLock); + CTG_ERR_JRET((*gCtgAsyncFps[type].initFp)(pJob, tid, param)); if (taskId) { *taskId = tid; } - return TSDB_CODE_SUCCESS; +_return: + CTG_UNLOCK(CTG_WRITE, &pJob->taskLock); + + return code; } int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const SCatalogReq* pReq, catalogCallback fp, @@ -902,6 +905,14 @@ int32_t ctgCallUserCb(void* param) { return TSDB_CODE_SUCCESS; } +void ctgUpdateJobErrCode(SCtgJob* pJob, int32_t errCode) { + if (!NEED_CLIENT_REFRESH_VG_ERROR(errCode) || errCode == TSDB_CODE_SUCCESS) return; + + atomic_store_32(&pJob->jobResCode, errCode); + qDebug("QID:0x%" PRIx64 " ctg job errCode updated to %s", pJob->queryId, tstrerror(errCode)); + return; +} + int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { SCtgJob* pJob = pTask->pJob; int32_t code = 0; @@ -921,6 +932,8 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { if (taskDone < taosArrayGetSize(pJob->pTasks)) { qDebug("QID:0x%" PRIx64 " task done: %d, total: %d", pJob->queryId, taskDone, (int32_t)taosArrayGetSize(pJob->pTasks)); + + ctgUpdateJobErrCode(pJob, rspCode); return TSDB_CODE_SUCCESS; } @@ -928,7 +941,8 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { _return: - pJob->jobResCode = code; + ctgUpdateJobErrCode(pJob, rspCode); + // pJob->jobResCode = code; // taosSsleep(2); // qDebug("QID:0x%" PRIx64 " ctg after sleep", pJob->queryId); @@ -1095,7 +1109,8 @@ _return: } if (code) { - ctgTaskError("Get table %d.%s.%s meta failed with error %s", pName->acctId, pName->dbname, pName->tname, tstrerror(code)); + ctgTaskError("Get table %d.%s.%s meta failed with error %s", pName->acctId, pName->dbname, pName->tname, + tstrerror(code)); } if (pTask->res || code) { ctgHandleTaskEnd(pTask, code); @@ -1283,7 +1298,8 @@ _return: TSWAP(pTask->res, ctx->pResList); taskDone = true; } - ctgTaskError("Get table %d.%s.%s meta failed with error %s", pName->acctId, pName->dbname, pName->tname, tstrerror(code)); + ctgTaskError("Get table %d.%s.%s meta failed with error %s", pName->acctId, pName->dbname, pName->tname, + tstrerror(code)); } if (pTask->res && taskDone) { diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index fe83854a911a26729beaa0e6626ed44c3be4f2b4..c266cc1df9fc0ab5bd61574f519920df327630c9 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -2500,6 +2500,7 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe CTG_LOCK(CTG_READ, &pCache->metaLock); if (NULL == pCache->pMeta) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); ctgDebug("tb %s meta not in cache, dbFName:%s", pName->tname, dbFName); ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index b6ff1c8b987278ee8ef223577e91771c12cefde8..a6c0d1c40157f31f8002b11e7d4ce0e95a6f3f21 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -22,7 +22,6 @@ extern SCatalogMgmt gCtgMgmt; SCtgDebug gCTGDebug = {0}; void ctgdUserCallback(SMetaData *pResult, void *param, int32_t code) { - ASSERT(*(int32_t *)param == 1); taosMemoryFree(param); qDebug("async call result: %s", tstrerror(code)); diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 7cc6c90d303c4c425358800eb646f986417cefdd..3b037e206290ff767a1f879e0b46aadf8b3e4f20 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -40,7 +40,9 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu msgNum = taosArrayGetSize(batchRsp.pRsps); } - ASSERT(taskNum == msgNum || 0 == msgNum); + if (ASSERTS(taskNum == msgNum || 0 == msgNum, "taskNum %d mis-match msgNum %d", taskNum, msgNum)) { + msgNum = 0; + } ctgDebug("QID:0x%" PRIx64 " ctg got batch %d rsp %s", pJob->queryId, cbParam->batchId, TMSG_INFO(cbParam->reqType + 1)); @@ -58,11 +60,19 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu if (msgNum > 0) { pRsp = taosArrayGet(batchRsp.pRsps, i); - taskMsg.msgType = pRsp->reqType; - taskMsg.pData = pRsp->msg; - taskMsg.len = pRsp->msgLen; - - ASSERT(pRsp->msgIdx == *msgIdx); + if (ASSERTS(pRsp->msgIdx == *msgIdx, "rsp msgIdx %d mis-match msgIdx %d", pRsp->msgIdx, *msgIdx)) { + pRsp = &rsp; + pRsp->msgIdx = *msgIdx; + pRsp->reqType = -1; + pRsp->rspCode = 0; + taskMsg.msgType = -1; + taskMsg.pData = NULL; + taskMsg.len = 0; + } else { + taskMsg.msgType = pRsp->reqType; + taskMsg.pData = pRsp->msg; + taskMsg.len = pRsp->msgLen; + } } else { pRsp = &rsp; pRsp->msgIdx = *msgIdx; diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 802ecde63eaae46645712a5d3f60ae76cdfeee6f..3dd40a413996c458dc204b851ae08e6ca4280529 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -986,6 +986,43 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* CTG_RET(code); } +int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFName, const char* pTbs[], int32_t tbNum, int32_t* vgId) { + int32_t code = 0; + CTG_ERR_RET(ctgMakeVgArray(dbInfo)); + + int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); + + if (vgNum <= 0) { + ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum); + CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); + } + + SVgroupInfo* vgInfo = NULL; + char tbFullName[TSDB_TABLE_FNAME_LEN]; + snprintf(tbFullName, sizeof(tbFullName), "%s.", dbFName); + int32_t offset = strlen(tbFullName); + + for (int32_t i = 0; i < tbNum; ++i) { + snprintf(tbFullName + offset, sizeof(tbFullName) - offset, "%s", pTbs[i]); + uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, + dbInfo->hashPrefix, dbInfo->hashSuffix); + + vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ); + if (NULL == vgInfo) { + ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, + (int32_t)taosArrayGetSize(dbInfo->vgArray)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + vgId[i] = vgInfo->vgId; + + ctgDebug("Got tb %s vgId:%d", tbFullName, vgInfo->vgId); + } + + CTG_RET(code); +} + + int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) { if (*(uint64_t*)key1 < ((SSTableVersion*)key2)->suid) { return -1; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index a179ec24f91cf7b5564862796633d4ad4f965f85..c87f6953eb5fbb2515b2db54ab1e85854111eeb2 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -21,6 +21,7 @@ #include "tdatablock.h" #include "tglobal.h" #include "tgrant.h" +#include "taosdef.h" extern SConfig* tsCfg; @@ -40,7 +41,6 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->numOfCols = htonl(numOfCols); int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols); - ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); return TSDB_CODE_SUCCESS; } @@ -154,6 +154,23 @@ static int32_t buildCreateDBResultDataBlock(SSDataBlock** pOutput) { return code; } +static int32_t buildAliveResultDataBlock(SSDataBlock** pOutput) { + SSDataBlock* pBlock = createDataBlock(); + if (NULL == pBlock) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1); + int32_t code = blockDataAppendColInfo(pBlock, &infoData); + + if (TSDB_CODE_SUCCESS == code) { + *pOutput = pBlock; + } else { + blockDataDestroy(pBlock); + } + return code; +} + int64_t getValOfDiffPrecision(int8_t unit, int64_t val) { int64_t v = 0; switch (unit) { @@ -281,6 +298,108 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, S colDataAppend(pCol2, 0, buf2, false); } +#define CHECK_LEADER(n) (row[n] && (fields[n].type == TSDB_DATA_TYPE_VARCHAR && strncasecmp(row[n], "leader", varDataLen((char *)row[n] - VARSTR_HEADER_SIZE)) == 0)) +// on this row, if have leader return true else return false +bool existLeaderRole(TAOS_ROW row, TAOS_FIELD* fields, int nFields) { + // vgroup_id | db_name | tables | v1_dnode | v1_status | v2_dnode | v2_status | v3_dnode | v3_status | v4_dnode | + // v4_status | cacheload | tsma | + if (nFields != 13) { + return false; + } + + // check have leader on cloumn v*_status on 4 6 8 10 + if (CHECK_LEADER(4) || CHECK_LEADER(6) || CHECK_LEADER(8) || CHECK_LEADER(10)) { + return true; + } + + return false; +} + +// get db alive status, return 1 is alive else return 0 +int32_t getAliveStatusFromApi(int64_t* pConnId, char* dbName, int32_t* pStatus) { + char sql[128 + TSDB_DB_NAME_LEN] = "select * from information_schema.ins_vgroups"; + int32_t code; + + // filter with db name + if (dbName && dbName[0] != 0) { + char str[64 + TSDB_DB_NAME_LEN] = ""; + // test db name exist + sprintf(str, "show create database %s ;", dbName); + TAOS_RES* dbRes = taos_query(pConnId, str); + code = taos_errno(dbRes); + if (code != TSDB_CODE_SUCCESS) { + taos_free_result(dbRes); + return code; + } + taos_free_result(dbRes); + + sprintf(str, " where db_name='%s' ;", dbName); + strcat(sql, str); + } + + TAOS_RES* res = taos_query(pConnId, sql); + code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + taos_free_result(res); + return code; + } + + TAOS_ROW row = NULL; + TAOS_FIELD* fields = taos_fetch_fields(res); + int32_t nFields = taos_num_fields(res); + int32_t nAvailble = 0; + int32_t nUnAvailble = 0; + + while ((row = taos_fetch_row(res)) != NULL) { + if (existLeaderRole(row, fields, nFields)) { + nAvailble++; + } else { + nUnAvailble++; + } + } + taos_free_result(res); + + int32_t status = 0; + if (nAvailble + nUnAvailble == 0 || nUnAvailble == 0) { + status = SHOW_STATUS_AVAILABLE; + } else if (nAvailble > 0 && nUnAvailble > 0) { + status = SHOW_STATUS_HALF_AVAILABLE; + } else { + status = SHOW_STATUS_NOT_AVAILABLE; + } + + if (pStatus) { + *pStatus = status; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t setAliveResultIntoDataBlock(int64_t* pConnId, SSDataBlock* pBlock, char* dbName) { + blockDataEnsureCapacity(pBlock, 1); + pBlock->info.rows = 1; + + SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0); + int32_t status = 0; + int32_t code = getAliveStatusFromApi(pConnId, dbName, &status); + if (code == TSDB_CODE_SUCCESS) { + colDataAppend(pCol1, 0, (const char*)&status, false); + } + return code; +} + +static int32_t execShowAliveStatus(int64_t* pConnId, SShowAliveStmt* pStmt, SRetrieveTableRsp** pRsp) { + SSDataBlock* pBlock = NULL; + int32_t code = buildAliveResultDataBlock(&pBlock); + if (TSDB_CODE_SUCCESS == code) { + code = setAliveResultIntoDataBlock(pConnId, pBlock, pStmt->dbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = buildRetrieveTableRsp(pBlock, SHOW_ALIVE_RESULT_COLS, pRsp); + } + blockDataDestroy(pBlock); + return code; +} + static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveTableRsp** pRsp) { SSDataBlock* pBlock = NULL; int32_t code = buildCreateDBResultDataBlock(&pBlock); @@ -736,7 +855,7 @@ static int32_t execSelectWithoutFrom(SSelectStmt* pSelect, SRetrieveTableRsp** p return code; } -int32_t qExecCommand(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp) { +int32_t qExecCommand(int64_t* pConnId, bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp) { switch (nodeType(pStmt)) { case QUERY_NODE_DESCRIBE_STMT: return execDescribe(sysInfoUser, pStmt, pRsp); @@ -754,6 +873,9 @@ int32_t qExecCommand(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp) { return execShowLocalVariables(pRsp); case QUERY_NODE_SELECT_STMT: return execSelectWithoutFrom((SSelectStmt*)pStmt, pRsp); + case QUERY_NODE_SHOW_DB_ALIVE_STMT: + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: + return execShowAliveStatus(pConnId, (SShowAliveStmt*)pStmt, pRsp); default: break; } diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 4624e471bf5c3066fc861c3f2f6558466b4afb88..fdbbcad9685eaa73b31a86a3401a36f3494a3789 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1574,7 +1574,6 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->numOfRows = htobe64((int64_t)rowNum); int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock)); - ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); rsp->compLen = htonl(len); diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 51150ede3c6b31c1a75dd4dd1197a4fe4f7d20a1..e0d2276e6fceed6aa31bdf22cd551e73a58dc4dc 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -18,7 +18,6 @@ #include "function.h" #include "nodes.h" #include "plannodes.h" -#include "tbuffer.h" #include "tcommon.h" #include "tpagedbuf.h" #include "tsimplehash.h" @@ -116,6 +115,10 @@ struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t i static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos, bool forUpdate) { SFilePage* bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId); + if (NULL == bufPage) { + return NULL; + } + if (forUpdate) { setBufPageDirty(bufPage, true); } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index dcb4a8a0ed26b55dd702c965de4005fa81dbc408..89c3b87b27cdb70016317b572a15c088e5146b52 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -126,12 +126,14 @@ enum { typedef struct { // TODO remove prepareStatus - STqOffsetVal prepareStatus; // for tmq - STqOffsetVal lastStatus; // for tmq - SMqMetaRsp metaRsp; // for tmq fetching meta - int8_t returned; - int64_t snapshotVer; - const SSubmitReq* pReq; + STqOffsetVal prepareStatus; // for tmq + STqOffsetVal lastStatus; // for tmq + SMqMetaRsp metaRsp; // for tmq fetching meta + int8_t returned; + int64_t snapshotVer; + // const SSubmitReq* pReq; + + SPackedData submit; SSchemaWrapper* schema; char tbName[TSDB_TABLE_NAME_LEN]; @@ -339,25 +341,26 @@ typedef struct STableScanInfo { } STableScanInfo; typedef struct STableMergeScanInfo { - int32_t tableStartIndex; - int32_t tableEndIndex; - bool hasGroupId; - uint64_t groupId; - SArray* queryConds; // array of queryTableDataCond - STableScanBase base; - int32_t bufPageSize; - uint32_t sortBufSize; // max buffer size for in-memory sort - SArray* pSortInfo; - SSortHandle* pSortHandle; - SSDataBlock* pSortInputBlock; - int64_t startTs; // sort start time - SArray* sortSourceParams; - SLimitInfo limitInfo; - int64_t numOfRows; - SScanInfo scanInfo; - SSDataBlock* pResBlock; - SSampleExecInfo sample; // sample execution info - SSortExecInfo sortExecInfo; + int32_t tableStartIndex; + int32_t tableEndIndex; + bool hasGroupId; + uint64_t groupId; + SArray* queryConds; // array of queryTableDataCond + STableScanBase base; + int32_t bufPageSize; + uint32_t sortBufSize; // max buffer size for in-memory sort + SArray* pSortInfo; + SSortHandle* pSortHandle; + SSDataBlock* pSortInputBlock; + int64_t startTs; // sort start time + SArray* sortSourceParams; + SLimitInfo limitInfo; + int64_t numOfRows; + SScanInfo scanInfo; + int32_t scanTimes; + SSDataBlock* pResBlock; + SSampleExecInfo sample; // sample execution info + SSortExecInfo sortExecInfo; } STableMergeScanInfo; typedef struct STagScanInfo { @@ -471,9 +474,11 @@ typedef struct SStreamScanInfo { SNode* pTagIndexCond; // recover - int32_t blockRecoverContiCnt; - int32_t blockRecoverTotCnt; + int32_t blockRecoverContiCnt; + int32_t blockRecoverTotCnt; + SSDataBlock* pRecoverRes; + SSDataBlock* pCreateTbRes; } SStreamScanInfo; typedef struct { @@ -564,6 +569,8 @@ typedef struct SStreamIntervalOperatorInfo { SStreamState* pState; SWinKey delKey; uint64_t numOfDatapack; + SArray* pUpdated; + SHashObj* pUpdatedMap; } SStreamIntervalOperatorInfo; typedef struct SDataGroupInfo { @@ -610,6 +617,8 @@ typedef struct SStreamSessionAggOperatorInfo { SPhysiNode* pPhyNode; // create new child bool isFinal; bool ignoreExpiredData; + SArray* pUpdated; + SSHashObj* pStUpdated; } SStreamSessionAggOperatorInfo; typedef struct SStreamStateAggOperatorInfo { @@ -625,6 +634,8 @@ typedef struct SStreamStateAggOperatorInfo { void* pDelIterator; SArray* pChildren; // cache for children's result; bool ignoreExpiredData; + SArray* pUpdated; + SSHashObj* pSeUpdated; } SStreamStateAggOperatorInfo; typedef struct SStreamPartitionOperatorInfo { @@ -635,9 +646,11 @@ typedef struct SStreamPartitionOperatorInfo { SExprSupp tagCalSup; SHashObj* pPartitions; void* parIte; + void* pTbNameIte; SSDataBlock* pInputDataBlock; int32_t tsColIndex; SSDataBlock* pDelRes; + SSDataBlock* pCreateTbRes; } SStreamPartitionOperatorInfo; typedef struct SStreamFillSupporter { @@ -693,7 +706,7 @@ void cleanupExprSupp(SExprSupp* pSup); void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs); int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, - const char* pkey); + const char* pkey, void* pState); void cleanupAggSup(SAggSupporter* pAggSup); void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows); @@ -760,7 +773,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream); +SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMergeIntervalPhysiNode* pIntervalPhyNode, SExecTaskInfo* pTaskInfo); @@ -838,7 +851,6 @@ bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pS void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp, void* pTbName); uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); -void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock); int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SExprSupp* pSup, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); @@ -855,6 +867,13 @@ int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResul int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize); void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order); int32_t qAppendTaskStopInfo(SExecTaskInfo* pTaskInfo, SExchangeOpStopInfo* pInfo); +int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, + int32_t order, int64_t* pData); +void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, int64_t groupId, + SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock); + +SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag); +SExprInfo* createExpr(SNodeList* pNodeList, int32_t* numOfExprs); void copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultRow* pRow, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const int32_t* rowEntryOffset, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index eff7a5ef938379a1d6e6a078c302e3f91ca9f544..a8051ea7c3eb294e7beb9ce6b5a23be601e635ef 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -62,8 +62,8 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp pEntry->numOfCols = taosArrayGetSize(pInput->pData->pDataBlock); pEntry->dataLen = sizeof(SDeleterRes); - ASSERT(1 == pEntry->numOfRows); - ASSERT(3 == pEntry->numOfCols); +// ASSERT(1 == pEntry->numOfRows); +// ASSERT(3 == pEntry->numOfCols); pBuf->useSize = sizeof(SDataCacheEntry); @@ -167,7 +167,6 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE SDataDeleterBuf* pBuf = NULL; taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf); - ASSERT(NULL != pBuf); memcpy(&pDeleter->nextOutput, pBuf, sizeof(SDataDeleterBuf)); taosFreeQitem(pBuf); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index c2fa438c808d42854794a56695d73ae192834247..a603bffba5f37b79ee9f4a42546b8bbb3a0c32ea 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -77,8 +77,8 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn pBuf->useSize = sizeof(SDataCacheEntry); pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols); - ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); - ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); +// ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); +// ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); pBuf->useSize += pEntry->dataLen; @@ -162,15 +162,14 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE SDataDispatchBuf* pBuf = NULL; taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); - ASSERT(NULL != pBuf); memcpy(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf)); taosFreeQitem(pBuf); SDataCacheEntry* pEntry = (SDataCacheEntry*)pDispatcher->nextOutput.pData; *pLen = pEntry->dataLen; - ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); - ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); +// ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); +// ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); *pQueryEnd = pDispatcher->queryEnd; qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, @@ -193,8 +192,8 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->numOfCols = pEntry->numOfCols; pOutput->compressed = pEntry->compressed; - ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); - ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); +// ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); +// ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen); atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 346fcc9729d30e896a24dc63bc3f996e14606da3..f78e3c22e13c8b99885c451cd16f76ed038b79a1 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -25,9 +25,9 @@ extern SDataSinkStat gDataSinkStat; typedef struct SSubmitRes { - int64_t affectedRows; - int32_t code; - SSubmitRsp* pRsp; + int64_t affectedRows; + int32_t code; + SSubmitRsp2* pRsp; } SSubmitRes; typedef struct SDataInserterHandle { @@ -41,6 +41,7 @@ typedef struct SDataInserterHandle { SHashObj* pCols; int32_t status; bool queryEnd; + bool fullOrderColList; uint64_t useconds; uint64_t cachedSize; TdThreadMutex mutex; @@ -58,22 +59,25 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) { pInserter->submitRes.code = code; if (code == TSDB_CODE_SUCCESS) { - pInserter->submitRes.pRsp = taosMemoryCalloc(1, sizeof(SSubmitRsp)); + pInserter->submitRes.pRsp = taosMemoryCalloc(1, sizeof(SSubmitRsp2)); SDecoder coder = {0}; tDecoderInit(&coder, pMsg->pData, pMsg->len); - code = tDecodeSSubmitRsp(&coder, pInserter->submitRes.pRsp); + code = tDecodeSSubmitRsp2(&coder, pInserter->submitRes.pRsp); if (code) { - tFreeSSubmitRsp(pInserter->submitRes.pRsp); + taosMemoryFree(pInserter->submitRes.pRsp); pInserter->submitRes.code = code; goto _return; } - if (pInserter->submitRes.pRsp->nBlocks > 0) { - for (int32_t i = 0; i < pInserter->submitRes.pRsp->nBlocks; ++i) { - SSubmitBlkRsp* blk = pInserter->submitRes.pRsp->pBlocks + i; - if (TSDB_CODE_SUCCESS != blk->code) { - code = blk->code; - tFreeSSubmitRsp(pInserter->submitRes.pRsp); + if (pInserter->submitRes.pRsp->affectedRows > 0) { + SArray* pCreateTbList = pInserter->submitRes.pRsp->aCreateTbRsp; + int32_t numOfTables = taosArrayGetSize(pCreateTbList); + + for (int32_t i = 0; i < numOfTables; ++i) { + SVCreateTbRsp* pRsp = taosArrayGet(pCreateTbList, i); + if (TSDB_CODE_SUCCESS != pRsp->code) { + code = pRsp->code; + taosMemoryFree(pInserter->submitRes.pRsp); pInserter->submitRes.code = code; goto _return; } @@ -81,22 +85,20 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) { } pInserter->submitRes.affectedRows += pInserter->submitRes.pRsp->affectedRows; - qDebug("submit rsp received, affectedRows:%d, total:%"PRId64, pInserter->submitRes.pRsp->affectedRows, + qDebug("submit rsp received, affectedRows:%d, total:%" PRId64, pInserter->submitRes.pRsp->affectedRows, pInserter->submitRes.affectedRows); - - tFreeSSubmitRsp(pInserter->submitRes.pRsp); + tDecoderClear(&coder); + taosMemoryFree(pInserter->submitRes.pRsp); } _return: - tsem_post(&pInserter->ready); - taosMemoryFree(pMsg->pData); - return TSDB_CODE_SUCCESS; } -static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMsg, void* pTransporter, SEpSet* pEpset) { +static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, void* pMsg, int32_t msgLen, void* pTransporter, + SEpSet* pEpset) { // send the fetch remote task result reques SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { @@ -111,7 +113,7 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMs pMsgSendInfo->param = pParam; pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->msgInfo.pData = pMsg; - pMsgSendInfo->msgInfo.len = ntohl(pMsg->length); + pMsgSendInfo->msgInfo.len = msgLen; pMsgSendInfo->msgType = TDMT_VND_SUBMIT; pMsgSendInfo->fp = inserterCallback; @@ -119,140 +121,230 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMs return asyncSendMsgToServer(pTransporter, pEpset, &transporterId, pMsgSendInfo); } -int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) { - const SArray* pBlocks = pInserter->pDataBlocks; - const STSchema* pTSchema = pInserter->pSchema; - int64_t uid = pInserter->pNode->tableId; - int64_t suid = pInserter->pNode->stableId; - int32_t vgId = pInserter->pNode->vgId; - bool fullCol = (pInserter->pNode->pCols->length == pTSchema->numOfCols); +static int32_t submitReqToMsg(int32_t vgId, SSubmitReq2* pReq, void** pData, int32_t* pLen) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t len = 0; + void* pBuf = NULL; + tEncodeSize(tEncodeSSubmitReq2, pReq, len, code); + if (TSDB_CODE_SUCCESS == code) { + SEncoder encoder; + len += sizeof(SMsgHead); + pBuf = taosMemoryMalloc(len); + if (NULL == pBuf) { + return TSDB_CODE_OUT_OF_MEMORY; + } + ((SMsgHead*)pBuf)->vgId = htonl(vgId); + ((SMsgHead*)pBuf)->contLen = htonl(len); + tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SMsgHead)), len - sizeof(SMsgHead)); + code = tEncodeSSubmitReq2(&encoder, pReq); + tEncoderClear(&encoder); + } - SSubmitReq* ret = NULL; - int32_t sz = taosArrayGetSize(pBlocks); + if (TSDB_CODE_SUCCESS == code) { + *pData = pBuf; + *pLen = len; + } else { + taosMemoryFree(pBuf); + } + return code; +} - // cal size - int32_t cap = sizeof(SSubmitReq); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i); - int32_t rows = pDataBlock->info.rows; - // TODO min - int32_t rowSize = pDataBlock->info.rowSize; - int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); +int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** ppReq, const SSDataBlock* pDataBlock, + const STSchema* pTSchema, int64_t uid, int32_t vgId, tb_uid_t suid) { + SSubmitReq2* pReq = *ppReq; + SArray* pVals = NULL; + int32_t numOfBlks = 0; - cap += sizeof(SSubmitBlk) + rows * maxLen; - } + terrno = TSDB_CODE_SUCCESS; - // assign data - // TODO - ret = taosMemoryCalloc(1, cap); - ret->header.vgId = htonl(vgId); - ret->version = htonl(pTSchema->version); - ret->length = sizeof(SSubmitReq); - ret->numOfBlocks = htonl(sz); + if (NULL == pReq) { + if (!(pReq = taosMemoryMalloc(sizeof(SSubmitReq2)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } - SSubmitBlk* blkHead = POINTER_SHIFT(ret, sizeof(SSubmitReq)); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i); + if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + } - blkHead->sversion = htonl(pTSchema->version); - // TODO - blkHead->suid = htobe64(suid); - blkHead->uid = htobe64(uid); - blkHead->schemaLen = htonl(0); - - int32_t rows = 0; - int32_t dataLen = 0; - STSRow* rowData = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); - int64_t lastTs = TSKEY_MIN; - bool ignoreRow = false; - for (int32_t j = 0; j < pDataBlock->info.rows; j++) { - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTSchema->version); - tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); - tdSRowResetBuf(&rb, rowData); + int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); + int32_t rows = pDataBlock->info.rows; - ignoreRow = false; - for (int32_t k = 0; k < pTSchema->numOfCols; k++) { - const STColumn* pColumn = &pTSchema->columns[k]; - SColumnInfoData* pColData = NULL; - int16_t colIdx = k; - if (!fullCol) { - int16_t* slotId = taosHashGet(pInserter->pCols, &pColumn->colId, sizeof(pColumn->colId)); - if (NULL == slotId) { - continue; - } + SSubmitTbData tbData = {0}; + if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow*)))) { + goto _end; + } + tbData.suid = suid; + tbData.uid = uid; + tbData.sver = pTSchema->version; - colIdx = *slotId; - } + if (!pVals && !(pVals = taosArrayInit(colNum, sizeof(SColVal)))) { + taosArrayDestroy(tbData.aRowP); + goto _end; + } - pColData = taosArrayGet(pDataBlock->pDataBlock, colIdx); - if (pColData->info.type != pColumn->type) { - qError("col type mis-match, schema type:%d, type in block:%d", pColumn->type, pColData->info.type); - terrno = TSDB_CODE_APP_ERROR; - return TSDB_CODE_APP_ERROR; + int64_t lastTs = TSKEY_MIN; + bool ignoreRow = false; + bool disorderTs = false; + + for (int32_t j = 0; j < rows; ++j) { // iterate by row + taosArrayClear(pVals); + + int32_t offset = 0; + for (int32_t k = 0; k < pTSchema->numOfCols; ++k) { // iterate by column + int16_t colIdx = k; + const STColumn* pCol = &pTSchema->columns[k]; + if (!pInserter->fullOrderColList) { + int16_t* slotId = taosHashGet(pInserter->pCols, &pCol->colId, sizeof(pCol->colId)); + if (NULL == slotId) { + continue; } - if (colDataIsNull_s(pColData, j)) { - if (0 == k && TSDB_DATA_TYPE_TIMESTAMP == pColumn->type) { - ignoreRow = true; - break; - } + colIdx = *slotId; + } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, pColumn->offset, k); - } else { - void* data = colDataGetData(pColData, j); - if (0 == k && TSDB_DATA_TYPE_TIMESTAMP == pColumn->type) { - if (*(int64_t*)data == lastTs) { - ignoreRow = true; - break; + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, colIdx); + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY + ASSERT(pColInfoData->info.type == pCol->type); + if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + void* data = colDataGetVarData(pColInfoData, j); + SValue sv = (SValue){.nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } + break; + } + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_MEDIUMBLOB: + uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); + ASSERT(0); + break; + default: + if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { + if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); // should use pCol->type + taosArrayPush(pVals, &cv); } else { - lastTs = *(int64_t*)data; + if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId) { + if (*(int64_t*)var == lastTs) { + ignoreRow = true; + } else if (*(int64_t*)var < lastTs) { + disorderTs = true; + } else { + lastTs = *(int64_t*)var; + } + } + + SValue sv; + memcpy(&sv.val, var, tDataTypes[pCol->type].bytes); + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); } + } else { + uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); + ASSERT(0); } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); - } - } - if (!fullCol) { - rb.hasNone = true; + break; } - tdSRowEnd(&rb); if (ignoreRow) { - continue; + break; } - - rows++; - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - dataLen += rowLen; } - blkHead->dataLen = htonl(dataLen); - blkHead->numOfRows = htonl(rows); + if (ignoreRow) { + ignoreRow = false; + continue; + } - ret->length += sizeof(SSubmitBlk) + dataLen; - blkHead = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk) + dataLen); + SRow* pRow = NULL; + if ((terrno = tRowBuild(pVals, pTSchema, &pRow)) < 0) { + tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + goto _end; + } + taosArrayPush(tbData.aRowP, &pRow); } - ret->length = htonl(ret->length); + if (disorderTs) { + tRowSort(tbData.aRowP); + if ((terrno = tRowMerge(tbData.aRowP, (STSchema*)pTSchema, 0)) != 0) { + goto _end; + } + } - *pReq = ret; + taosArrayPush(pReq->aSubmitTbData, &tbData); +_end: + taosArrayDestroy(pVals); + if (terrno != 0) { + *ppReq = NULL; + if (pReq) { + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); + } + return TSDB_CODE_FAILED; + } + *ppReq = pReq; return TSDB_CODE_SUCCESS; } +int32_t dataBlocksToSubmitReq(SDataInserterHandle* pInserter, void** pMsg, int32_t* msgLen) { + const SArray* pBlocks = pInserter->pDataBlocks; + const STSchema* pTSchema = pInserter->pSchema; + int64_t uid = pInserter->pNode->tableId; + int64_t suid = pInserter->pNode->stableId; + int32_t vgId = pInserter->pNode->vgId; + int32_t sz = taosArrayGetSize(pBlocks); + int32_t code = 0; + SSubmitReq2* pReq = NULL; + + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i); + + code = buildSubmitReqFromBlock(pInserter, &pReq, pDataBlock, pTSchema, uid, vgId, suid); + if (code) { + if (pReq) { + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); + } + + return code; + } + } + + code = submitReqToMsg(vgId, pReq, pMsg, msgLen); + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); + + return code; +} + static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle; taosArrayPush(pInserter->pDataBlocks, &pInput->pData); - SSubmitReq* pMsg = NULL; - int32_t code = dataBlockToSubmit(pInserter, &pMsg); + void* pMsg = NULL; + int32_t msgLen = 0; + int32_t code = dataBlocksToSubmitReq(pInserter, &pMsg, &msgLen); if (code) { return code; } taosArrayClear(pInserter->pDataBlocks); - - code = sendSubmitRequest(pInserter, pMsg, pInserter->pParam->readHandle->pMsgCb->clientRpc, &pInserter->pNode->epSet); + + code = sendSubmitRequest(pInserter, pMsg, msgLen, pInserter->pParam->readHandle->pMsgCb->clientRpc, + &pInserter->pNode->epSet); if (code) { return code; } @@ -346,12 +438,19 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat return TSDB_CODE_OUT_OF_MEMORY; } + inserter->fullOrderColList = pInserterNode->pCols->length == inserter->pSchema->numOfCols; + inserter->pCols = taosHashInit(pInserterNode->pCols->length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); - SNode* pNode = NULL; + SNode* pNode = NULL; + int32_t i = 0; FOREACH(pNode, pInserterNode->pCols) { SColumnNode* pCol = (SColumnNode*)pNode; taosHashPut(inserter->pCols, &pCol->colId, sizeof(pCol->colId), &pCol->slotId, sizeof(pCol->slotId)); + if (inserter->fullOrderColList && pCol->colId != inserter->pSchema->columns[i].colId) { + inserter->fullOrderColList = false; + } + ++i; } tsem_init(&inserter->ready, 0, 0); diff --git a/source/libs/executor/src/eventwindowoperator.c b/source/libs/executor/src/eventwindowoperator.c index d41875095fc9ab837bf23baa92e45ad52cb268d8..223b45dc8f88893e82dd8cb74040cf51a8ee1177 100644 --- a/source/libs/executor/src/eventwindowoperator.c +++ b/source/libs/executor/src/eventwindowoperator.c @@ -107,7 +107,8 @@ SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo SExprInfo* pExprInfo = createExprInfo(pEventWindowNode->window.pFuncs, NULL, &num); initResultSizeInfo(&pOperator->resultInfo, 4096); - code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str); + code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, + pTaskInfo->streamInfo.pState); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 4103ca82dcc7b122f925207717ea15698bf7b904..9873c520060d1a645255ef05b1542c39143c2623 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -373,7 +373,6 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pRsp->useconds = htobe64(pRsp->useconds); pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); - ASSERT(pRsp != NULL); qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%" PRId64 ", %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, pRsp->numOfRows, pExchangeInfo); } else { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index fc3cfbd0f65bc7668679fc23aa8466a11ee9ac81..cbb056b3b1021bc337c79d04e1276e5017428907 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1424,6 +1424,18 @@ void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode) { createExprFromOneNode(pExp, pTargetNode->pExpr, pTargetNode->slotId); } +SExprInfo* createExpr(SNodeList* pNodeList, int32_t* numOfExprs) { + *numOfExprs = LIST_LENGTH(pNodeList); + SExprInfo* pExprs = taosMemoryCalloc(*numOfExprs, sizeof(SExprInfo)); + + for (int32_t i = 0; i < (*numOfExprs); ++i) { + SExprInfo* pExp = &pExprs[i]; + createExprFromOneNode(pExp, nodesListGetNode(pNodeList, i), i + UD_TAG_COLUMN_INDEX); + } + + return pExprs; +} + SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs) { int32_t numOfFuncs = LIST_LENGTH(pNodeList); int32_t numOfGroupKeys = 0; @@ -1726,8 +1738,10 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI return w; } - w = getResultRowByPos(pBuf, &pResultRowInfo->cur, false)->win; - + SResultRow* pRow = getResultRowByPos(pBuf, &pResultRowInfo->cur, false); + if (pRow) { + w = pRow->win; + } // in case of typical time window, we can calculate time window directly. if (w.skey > ts || w.ekey < ts) { w = doCalculateTimeWindow(ts, pInterval); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index f83f938e61c18875fc4998304cfd4bd5d8d78c0b..6ec60ee85722d6342074cbc7e2e87fa41452b943 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -51,8 +51,8 @@ static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOf if (type == STREAM_INPUT__MERGED_SUBMIT) { for (int32_t i = 0; i < numOfBlocks; i++) { - SSubmitReq* pReq = *(void**)POINTER_SHIFT(input, i * sizeof(void*)); - taosArrayPush(pInfo->pBlockLists, &pReq); + SPackedData* pReq = POINTER_SHIFT(input, i * sizeof(SPackedData)); + taosArrayPush(pInfo->pBlockLists, pReq); } pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; } else if (type == STREAM_INPUT__DATA_SUBMIT) { @@ -61,7 +61,10 @@ static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOf } else if (type == STREAM_INPUT__DATA_BLOCK) { for (int32_t i = 0; i < numOfBlocks; ++i) { SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i]; - taosArrayPush(pInfo->pBlockLists, &pDataBlock); + SPackedData tmp = { + .pDataBlock = pDataBlock, + }; + taosArrayPush(pInfo->pBlockLists, &tmp); } pInfo->blockType = STREAM_INPUT__DATA_BLOCK; } @@ -115,18 +118,21 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu if (type == STREAM_INPUT__MERGED_SUBMIT) { // ASSERT(numOfBlocks > 1); for (int32_t i = 0; i < numOfBlocks; i++) { - SSubmitReq* pReq = *(void**)POINTER_SHIFT(input, i * sizeof(void*)); - taosArrayPush(pInfo->pBlockLists, &pReq); + SPackedData* pReq = POINTER_SHIFT(input, i * sizeof(SPackedData)); + taosArrayPush(pInfo->pBlockLists, pReq); } pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; } else if (type == STREAM_INPUT__DATA_SUBMIT) { ASSERT(numOfBlocks == 1); - taosArrayPush(pInfo->pBlockLists, &input); + taosArrayPush(pInfo->pBlockLists, input); pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; } else if (type == STREAM_INPUT__DATA_BLOCK) { for (int32_t i = 0; i < numOfBlocks; ++i) { SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i]; - taosArrayPush(pInfo->pBlockLists, &pDataBlock); + SPackedData tmp = { + .pDataBlock = pDataBlock, + }; + taosArrayPush(pInfo->pBlockLists, &tmp); } pInfo->blockType = STREAM_INPUT__DATA_BLOCK; } else { @@ -704,6 +710,15 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo, int32_t rspCode) { return TSDB_CODE_SUCCESS; } +bool qTaskIsExecuting(qTaskInfo_t qinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo; + if (NULL == pTaskInfo) { + return false; + } + + return 0 != atomic_load_64(&pTaskInfo->owner); +} + static void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo) { STaskCostInfo* pSummary = &pTaskInfo->cost; int64_t idleTime = pSummary->start - pSummary->created; @@ -936,7 +951,6 @@ int32_t qStreamRestoreParam(qTaskInfo_t tinfo) { } return 0; } - bool qStreamRecoverScanFinished(qTaskInfo_t tinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; return pTaskInfo->streamInfo.recoverScanFinished; @@ -1007,11 +1021,22 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s return TSDB_CODE_SUCCESS; } -int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq) { +#if 0 +int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq, int64_t scanVer) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); ASSERT(pTaskInfo->streamInfo.pReq == NULL); pTaskInfo->streamInfo.pReq = pReq; + pTaskInfo->streamInfo.scanVer = scanVer; + return 0; +} +#endif + +int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); + ASSERT(pTaskInfo->streamInfo.submit.msgStr == NULL); + pTaskInfo->streamInfo.submit = submit; return 0; } @@ -1191,3 +1216,4 @@ void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { rpcFreeCont(pMsg->pCont); destroySendMsgInfo(pSendInfo); } + diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 79ecd8189e829d9996f824b5d89c081c0849a119..21ef5dfab3891e8c00d164790df0762e047750af 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -83,6 +83,7 @@ typedef struct SAggOperatorInfo { uint64_t groupId; SGroupResInfo groupResInfo; SExprSupp scalarExprSup; + bool groupKeyOptimized; } SAggOperatorInfo; static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pBlock); @@ -104,8 +105,6 @@ static int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, void setOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; - ASSERT(pOperator->pTaskInfo != NULL); - pOperator->cost.totalCost = (taosGetTimestampUs() - pOperator->pTaskInfo->cost.start) / 1000.0; setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); } @@ -151,6 +150,11 @@ SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, i pData->num = sizeof(SFilePage); } else { pData = getBufPage(pResultBuf, *currentPageId); + if (pData == NULL) { + qError("failed to get buffer, code:%s", tstrerror(terrno)); + return NULL; + } + pageId = *currentPageId; if (pData->num + interBufSize > getBufPageSize(pResultBuf)) { @@ -201,6 +205,10 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR if (isIntervalQuery) { if (p1 != NULL) { // the *p1 may be NULL in case of sliding+offset exists. pResult = getResultRowByPos(pResultBuf, p1, true); + if (NULL == pResult) { + T_LONG_JMP(pTaskInfo->env, terrno); + } + ASSERT(pResult->pageId == p1->pageId && pResult->offset == p1->offset); } } else { @@ -209,6 +217,10 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR if (p1 != NULL) { // todo pResult = getResultRowByPos(pResultBuf, p1, true); + if (NULL == pResult) { + T_LONG_JMP(pTaskInfo->env, terrno); + } + ASSERT(pResult->pageId == p1->pageId && pResult->offset == p1->offset); } } @@ -217,6 +229,10 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR if (pResultRowInfo->cur.pageId != -1 && ((pResult == NULL) || (pResult->pageId != pResultRowInfo->cur.pageId))) { SResultRowPosition pos = pResultRowInfo->cur; SFilePage* pPage = getBufPage(pResultBuf, pos.pageId); + if (pPage == NULL) { + qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo)); + T_LONG_JMP(pTaskInfo->env, terrno); + } releaseBufPage(pResultBuf, pPage); } @@ -224,6 +240,9 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR if (pResult == NULL) { ASSERT(pSup->resultRowSize > 0); pResult = getNewResultRow(pResultBuf, &pSup->currentPageId, pSup->resultRowSize); + if (pResult == NULL) { + T_LONG_JMP(pTaskInfo->env, terrno); + } // add a new result set for a new group SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset}; @@ -261,6 +280,11 @@ static int32_t addNewWindowResultBuf(SResultRow* pWindowRes, SDiskbasedBuf* pRes } else { SPageInfo* pi = getLastPageInfo(list); pData = getBufPage(pResultBuf, getPageId(pi)); + if (pData == NULL) { + qError("failed to get buffer, code:%s", tstrerror(terrno)); + return terrno; + } + pageId = getPageId(pi); if (pData->num + size > getBufPageSize(pResultBuf)) { @@ -524,7 +548,7 @@ bool functionNeedToExecute(SqlFunctionCtx* pCtx) { return true; } -static int32_t doCreateConstantValColumnAggInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, int32_t type, +static int32_t doCreateConstantValColumnSMAInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, int32_t type, int32_t paramIndex, int32_t numOfRows) { if (pInput->pData[paramIndex] == NULL) { pInput->pData[paramIndex] = taosMemoryCalloc(1, sizeof(SColumnInfoData)); @@ -548,8 +572,6 @@ static int32_t doCreateConstantValColumnAggInfo(SInputColumnInfoData* pInput, SF da = pInput->pColumnDataAgg[paramIndex]; } - ASSERT(!IS_VAR_DATA_TYPE(type)); - if (type == TSDB_DATA_TYPE_BIGINT) { int64_t v = pFuncParam->param.i; *da = (SColumnDataAgg){.numOfNull = 0, .min = v, .max = v, .sum = v * numOfRows}; @@ -570,7 +592,7 @@ static int32_t doCreateConstantValColumnAggInfo(SInputColumnInfoData* pInput, SF } else if (type == TSDB_DATA_TYPE_TIMESTAMP) { // do nothing } else { - ASSERT(0); + qError("invalid constant type for sma info"); } return TSDB_CODE_SUCCESS; @@ -600,7 +622,7 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB // the data in the corresponding SColumnInfoData will not be used. pInput->pData[j] = taosArrayGet(pBlock->pDataBlock, slotId); } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { - doCreateConstantValColumnAggInfo(pInput, pFuncParam, pFuncParam->param.nType, j, pBlock->info.rows); + doCreateConstantValColumnSMAInfo(pInput, pFuncParam, pFuncParam->param.nType, j, pBlock->info.rows); } } } else { @@ -608,9 +630,7 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB } } -bool isTaskKilled(SExecTaskInfo* pTaskInfo) { - return (0 != pTaskInfo->code) ? true : false; -} +bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code) ? true : false; } void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { pTaskInfo->code = rspCode; } @@ -865,6 +885,7 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD return; } + int8_t* pIndicator = (int8_t*)p->pData; int32_t totalRows = pBlock->info.rows; if (status == FILTER_RESULT_ALL_QUALIFIED) { @@ -873,7 +894,8 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD pBlock->info.rows = 0; } else { int32_t bmLen = BitmapLen(totalRows); - char* pBitmap = NULL; + char* pBitmap = NULL; + int32_t maxRows = 0; size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { @@ -884,96 +906,121 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD } int32_t numOfRows = 0; + if (IS_VAR_DATA_TYPE(pDst->info.type)) { + int32_t j = 0; + pDst->varmeta.length = 0; + + while (j < totalRows) { + if (pIndicator[j] == 0) { + j += 1; + continue; + } - switch (pDst->info.type) { - case TSDB_DATA_TYPE_VARCHAR: - case TSDB_DATA_TYPE_NCHAR: - break; - default: - if (pBitmap == NULL) { - pBitmap = taosMemoryCalloc(1, bmLen); + if (colDataIsNull_var(pDst, j)) { + colDataSetNull_var(pDst, numOfRows); + } else { + char* p1 = colDataGetVarData(pDst, j); + colDataAppend(pDst, numOfRows, p1, false); } - memcpy(pBitmap, pDst->nullbitmap, bmLen); - memset(pDst->nullbitmap, 0, bmLen); - - int32_t j = 0; - - switch (pDst->info.type) { - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - while (j < totalRows) { - if (((int8_t*)p->pData)[j] == 0) { - j += 1; - continue; - } - - if (colDataIsNull_f(pBitmap, j)) { - colDataAppendNULL(pDst, numOfRows); - } else { - ((int64_t*)pDst->pData)[numOfRows] = ((int64_t*)pDst->pData)[j]; - } - numOfRows += 1; + numOfRows += 1; + j += 1; + } + + if (maxRows < numOfRows) { + maxRows = numOfRows; + } + } else { + if (pBitmap == NULL) { + pBitmap = taosMemoryCalloc(1, bmLen); + } + + memcpy(pBitmap, pDst->nullbitmap, bmLen); + memset(pDst->nullbitmap, 0, bmLen); + + int32_t j = 0; + + switch (pDst->info.type) { + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + while (j < totalRows) { + if (pIndicator[j] == 0) { + j += 1; + continue; + } + + if (colDataIsNull_f(pBitmap, j)) { + colDataSetNull_f(pDst->nullbitmap, numOfRows); + } else { + ((int64_t*)pDst->pData)[numOfRows] = ((int64_t*)pDst->pData)[j]; } - break; - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - while (j < totalRows) { - if (((int8_t*)p->pData)[j] == 0) { - j += 1; - continue; - } - if (colDataIsNull_f(pBitmap, j)) { - colDataAppendNULL(pDst, numOfRows); - } else { - ((int32_t*)pDst->pData)[numOfRows++] = ((int32_t*)pDst->pData)[j]; - } - numOfRows += 1; + numOfRows += 1; + j += 1; + } + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + while (j < totalRows) { + if (pIndicator[j] == 0) { + j += 1; + continue; } - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - while (j < totalRows) { - if (((int8_t*)p->pData)[j] == 0) { - j += 1; - continue; - } - if (colDataIsNull_f(pBitmap, j)) { - colDataAppendNULL(pDst, numOfRows); - } else { - ((int16_t*)pDst->pData)[numOfRows++] = ((int16_t*)pDst->pData)[j]; - } - numOfRows += 1; + if (colDataIsNull_f(pBitmap, j)) { + colDataSetNull_f(pDst->nullbitmap, numOfRows); + } else { + ((int32_t*)pDst->pData)[numOfRows] = ((int32_t*)pDst->pData)[j]; } - break; - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - while (j < totalRows) { - if (((int8_t*)p->pData)[j] == 0) { - j += 1; - continue; - } - if (colDataIsNull_f(pBitmap, j)) { - colDataAppendNULL(pDst, numOfRows); - } else { - ((int8_t*)pDst->pData)[numOfRows] = ((int8_t*)pDst->pData)[j]; - } - numOfRows += 1; + numOfRows += 1; + j += 1; + } + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + while (j < totalRows) { + if (pIndicator[j] == 0) { + j += 1; + continue; } - break; - } - }; + if (colDataIsNull_f(pBitmap, j)) { + colDataSetNull_f(pDst->nullbitmap, numOfRows); + } else { + ((int16_t*)pDst->pData)[numOfRows] = ((int16_t*)pDst->pData)[j]; + } + numOfRows += 1; + j += 1; + } + break; + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + while (j < totalRows) { + if (pIndicator[j] == 0) { + j += 1; + continue; + } + if (colDataIsNull_f(pBitmap, j)) { + colDataSetNull_f(pDst->nullbitmap, numOfRows); + } else { + ((int8_t*)pDst->pData)[numOfRows] = ((int8_t*)pDst->pData)[j]; + } + numOfRows += 1; + j += 1; + } + break; + } + } - // todo this value can be assigned directly - if (pBlock->info.rows == totalRows) { - pBlock->info.rows = numOfRows; - } else { - ASSERT(pBlock->info.rows == numOfRows); + if (maxRows < numOfRows) { + maxRows = numOfRows; } } + + pBlock->info.rows = maxRows; + if (pBitmap != NULL) { + taosMemoryFree(pBitmap); + } } } @@ -997,7 +1044,7 @@ void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uin if (pResultRow->pageId == -1) { int32_t ret = addNewWindowResultBuf(pResultRow, pAggInfo->aggSup.pResultBuf, pAggInfo->binfo.pRes->info.rowSize); if (ret != TSDB_CODE_SUCCESS) { - return; + T_LONG_JMP(pTaskInfo->env, terrno); } } @@ -1076,7 +1123,12 @@ void copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultR // todo refactor. SResultRow has direct pointer in miainfo int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SExprSupp* pSup, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { - SFilePage* page = getBufPage(pBuf, resultRowPosition->pageId); + SFilePage* page = getBufPage(pBuf, resultRowPosition->pageId); + if (page == NULL) { + qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo)); + T_LONG_JMP(pTaskInfo->env, terrno); + } + SResultRow* pRow = (SResultRow*)((char*)page + resultRowPosition->offset); SqlFunctionCtx* pCtx = pSup->pCtx; @@ -1120,6 +1172,10 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) { SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i); SFilePage* page = getBufPage(pBuf, pPos->pos.pageId); + if (page == NULL) { + qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo)); + T_LONG_JMP(pTaskInfo->env, terrno); + } SResultRow* pRow = (SResultRow*)((char*)page + pPos->pos.offset); @@ -1432,20 +1488,25 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan } } -static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock **ppBlock) { +static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock** ppBlock) { if (!tsCountAlwaysReturnValue) { return TSDB_CODE_SUCCESS; } + SAggOperatorInfo* pAggInfo = pOperator->info; + if (pAggInfo->groupKeyOptimized) { + return TSDB_CODE_SUCCESS; + } + SOperatorInfo* downstream = pOperator->pDownstream[0]; if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_PARTITION || (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN && - ((STableScanInfo *)downstream->info)->hasGroupByTag == true)) { + ((STableScanInfo*)downstream->info)->hasGroupByTag == true)) { return TSDB_CODE_SUCCESS; } SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; - bool hasCountFunc = false; + bool hasCountFunc = false; for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { const char* pName = pCtx[i].pExpr->pExpr->_function.functionName; @@ -1494,7 +1555,7 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc return TSDB_CODE_SUCCESS; } -static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock **ppBlock) { +static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock** ppBlock) { if (!blockAllocated) { return; } @@ -1520,8 +1581,8 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { int32_t order = TSDB_ORDER_ASC; int32_t scanFlag = MAIN_SCAN; - bool hasValidBlock = false; - bool blockAllocated = false; + bool hasValidBlock = false; + bool blockAllocated = false; while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); @@ -1564,7 +1625,6 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { } destroyDataBlockForEmptyInput(blockAllocated, &pBlock); - } // the downstream operator may return with error code, so let's check the code before generating results. @@ -1656,8 +1716,7 @@ void destroyOperatorInfo(SOperatorInfo* pOperator) { // each operator should be set their own function to return total cost buffer int32_t optrDefaultBufFn(SOperatorInfo* pOperator) { if (pOperator->blocking) { - ASSERT(0); - return 0; + return -1; } else { return 0; } @@ -1720,7 +1779,7 @@ void cleanupAggSup(SAggSupporter* pAggSup) { } int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, - const char* pkey) { + const char* pkey, void* pState) { int32_t code = initExprSupp(pSup, pExprInfo, numOfCols); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1732,7 +1791,13 @@ int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo } for (int32_t i = 0; i < numOfCols; ++i) { - pSup->pCtx[i].saveHandle.pBuf = pAggSup->pResultBuf; + if (pState) { + pSup->pCtx[i].saveHandle.pBuf = NULL; + pSup->pCtx[i].saveHandle.pState = pState; + pSup->pCtx[i].exprIdx = i; + } else { + pSup->pCtx[i].saveHandle.pBuf = pAggSup->pResultBuf; + } } return TSDB_CODE_SUCCESS; @@ -1817,7 +1882,8 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); - int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str); + int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, + pTaskInfo->streamInfo.pState); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1839,11 +1905,13 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN } pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock; + pInfo->groupKeyOptimized = pAggNode->groupKeyOptimized; pInfo->groupId = UINT64_MAX; setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, optrDefaultBufFn, NULL); + pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, + optrDefaultBufFn, NULL); if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { STableScanInfo* pTableScanInfo = downstream->info; @@ -2202,9 +2270,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type) { SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; - - bool isStream = (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type); - pOptr = createIntervalOperatorInfo(ops[0], pIntervalPhyNode, pTaskInfo, isStream); + pOptr = createIntervalOperatorInfo(ops[0], pIntervalPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { pOptr = createStreamIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) { @@ -2282,7 +2348,6 @@ static int32_t extractTbscanInStreamOpTree(SOperatorInfo* pOperator, STableScanI return extractTbscanInStreamOpTree(pOperator->pDownstream[0], ppInfo); } else { SStreamScanInfo* pInfo = pOperator->info; - ASSERT(pInfo->pTableScanOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); *ppInfo = pInfo->pTableScanOp->info; return 0; } @@ -2294,13 +2359,11 @@ int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) { *ppNode = (STableScanPhysiNode*)pNode; return 0; } else { - ASSERT(0); terrno = TSDB_CODE_APP_ERROR; return -1; } } else { if (LIST_LENGTH(pNode->pChildren) != 1) { - ASSERT(0); terrno = TSDB_CODE_APP_ERROR; return -1; } @@ -2310,32 +2373,6 @@ int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) { return -1; } -#if 0 -int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHandle, int64_t uid, int64_t ts) { - STableScanInfo* pTableScanInfo = NULL; - if (extractTbscanInStreamOpTree(pOperator, &pTableScanInfo) < 0) { - return -1; - } - - STableScanPhysiNode* pNode = NULL; - if (extractTableScanNode(plan->pNode, &pNode) < 0) { - ASSERT(0); - } - - tsdbReaderClose(pTableScanInfo->dataReader); - - STableListInfo info = {0}; - pTableScanInfo->dataReader = doCreateDataReader(pNode, pHandle, &info, NULL); - if (pTableScanInfo->dataReader == NULL) { - ASSERT(0); - qError("failed to create data reader"); - return TSDB_CODE_APP_ERROR; - } - // TODO: set uid and ts to data reader - return 0; -} -#endif - int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle) { SExecTaskInfo* pTask = *(SExecTaskInfo**)pTaskInfo; @@ -2719,3 +2756,24 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta blockDataUpdateTsWindow(pBlock, 0); return TSDB_CODE_SUCCESS; } + +void qStreamCloseTsdbReader(void* task) { + if (task == NULL) return; + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)task; + SOperatorInfo* pOp = pTaskInfo->pRoot; + qDebug("stream close tsdb reader, reset status uid %" PRId64 " ts %" PRId64, pTaskInfo->streamInfo.lastStatus.uid, + pTaskInfo->streamInfo.lastStatus.ts); + pTaskInfo->streamInfo.lastStatus = (STqOffsetVal){0}; + while (pOp->numOfDownstream == 1 && pOp->pDownstream[0]) { + SOperatorInfo* pDownstreamOp = pOp->pDownstream[0]; + if (pDownstreamOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + SStreamScanInfo* pInfo = pDownstreamOp->info; + if (pInfo->pTableScanOp) { + STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; + tsdbReaderClose(pTSInfo->base.dataReader); + pTSInfo->base.dataReader = NULL; + return; + } + } + } +} diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 59266dad7eef4040f79cb3383974453eb4727324..3bc00b79b1aea8bd8dc6d586ed8598b3b5672e4a 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -458,7 +458,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); - code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, pInfo->groupKeyLen, pTaskInfo->id.str); + code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, pInfo->groupKeyLen, pTaskInfo->id.str, + pTaskInfo->streamInfo.pState); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -491,13 +492,17 @@ _error: static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { SPartitionOperatorInfo* pInfo = pOperator->info; - + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + for (int32_t j = 0; j < pBlock->info.rows; ++j) { recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j); int32_t len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); SDataGroupInfo* pGroupInfo = NULL; void* pPage = getCurrentDataGroupInfo(pInfo, &pGroupInfo, len); + if (pPage == NULL) { + T_LONG_JMP(pTaskInfo->env, terrno); + } pGroupInfo->numOfRows += 1; @@ -594,6 +599,10 @@ void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInf } else { int32_t* curId = taosArrayGetLast(p->pPageList); pPage = getBufPage(pInfo->pBuf, *curId); + if (pPage == NULL) { + qError("failed to get buffer, code:%s", tstrerror(terrno)); + return pPage; + } int32_t* rows = (int32_t*)pPage; if (*rows >= pInfo->rowCapacity) { @@ -673,7 +682,8 @@ static int compareDataGroupInfo(const void* group1, const void* group2) { static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { SPartitionOperatorInfo* pInfo = pOperator->info; - + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SDataGroupInfo* pGroupInfo = (pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL; if (pInfo->groupIndex == -1 || pInfo->pageIndex >= taosArrayGetSize(pGroupInfo->pPageList)) { @@ -691,7 +701,11 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { int32_t* pageId = taosArrayGet(pGroupInfo->pPageList, pInfo->pageIndex); void* page = getBufPage(pInfo->pBuf, *pageId); - + if (page == NULL) { + qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo)); + T_LONG_JMP(pTaskInfo->env, terrno); + } + blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->rowCapacity); blockDataFromBuf1(pInfo->binfo.pRes, page, pInfo->rowCapacity); @@ -903,6 +917,7 @@ uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, S } static bool hasRemainPartion(SStreamPartitionOperatorInfo* pInfo) { return pInfo->parIte != NULL; } +static bool hasRemainTbName(SStreamPartitionOperatorInfo* pInfo) { return pInfo->pTbNameIte != NULL; } static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { SStreamPartitionOperatorInfo* pInfo = pOperator->info; @@ -923,40 +938,13 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { colDataAppend(pDestCol, pDest->info.rows, pSrcData, isNull); } pDest->info.rows++; - if (pInfo->tbnameCalSup.numOfExprs > 0 && i == 0) { - void* tbname = NULL; - if (streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname) == 0) { - memcpy(pDest->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); - tdbFree(tbname); - } else { - SSDataBlock* pTmpBlock = blockCopyOneRow(pSrc, rowIndex); - SSDataBlock* pResBlock = createDataBlock(); - pResBlock->info.rowSize = TSDB_TABLE_NAME_LEN; - SColumnInfoData data = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_TABLE_NAME_LEN, 0); - taosArrayPush(pResBlock->pDataBlock, &data); - blockDataEnsureCapacity(pResBlock, 1); - projectApplyFunctions(pInfo->tbnameCalSup.pExprInfo, pResBlock, pTmpBlock, pInfo->tbnameCalSup.pCtx, 1, NULL); - ASSERT(pResBlock->info.rows == 1); - ASSERT(taosArrayGetSize(pResBlock->pDataBlock) == 1); - SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, 0); - ASSERT(pCol->info.type == TSDB_DATA_TYPE_VARCHAR); - void* pData = colDataGetVarData(pCol, 0); - // TODO check tbname validity - if (pData != (void*)-1) { - memset(pDest->info.parTbName, 0, TSDB_TABLE_NAME_LEN); - int32_t len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); - memcpy(pDest->info.parTbName, varDataVal(pData), len); - /*pDest->info.parTbName[len + 1] = 0;*/ - } else { - pDest->info.parTbName[0] = 0; - } - if (pParInfo->groupId && pDest->info.parTbName[0]) { - streamStatePutParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, pDest->info.parTbName); - } - /*printf("\n\n set name %s\n\n", pDest->info.parTbName);*/ - blockDataDestroy(pTmpBlock); - blockDataDestroy(pResBlock); - } + } + pDest->info.parTbName[0] = 0; + if (pInfo->tbnameCalSup.numOfExprs > 0) { + void* tbname = NULL; + if (streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname) == 0) { + memcpy(pDest->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); + tdbFree(tbname); } } taosArrayDestroy(pParInfo->rowIds); @@ -972,6 +960,60 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { return pDest; } +void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, int64_t groupId, + SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock) { + void* pValue = NULL; + if (groupId != 0 && streamStateGetParName(pState, groupId, &pValue) != 0) { + SSDataBlock* pTmpBlock = blockCopyOneRow(pSrcBlock, rowId); + if (pTableSup->numOfExprs > 0) { + projectApplyFunctions(pTableSup->pExprInfo, pDestBlock, pTmpBlock, pTableSup->pCtx, pTableSup->numOfExprs, NULL); + SColumnInfoData* pTbCol = taosArrayGet(pDestBlock->pDataBlock, UD_TABLE_NAME_COLUMN_INDEX); + void* pData = colDataGetVarData(pTbCol, pDestBlock->info.rows - 1); + char* tbName = pSrcBlock->info.parTbName; + memset(tbName, 0, TSDB_TABLE_NAME_LEN); + int32_t len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); + memcpy(tbName, varDataVal(pData), len); + streamStatePutParName(pState, groupId, tbName); + pDestBlock->info.rows--; + } else { + void* pTbNameCol = taosArrayGet(pDestBlock->pDataBlock, UD_TABLE_NAME_COLUMN_INDEX); + colDataAppendNULL(pTbNameCol, pDestBlock->info.rows); + pSrcBlock->info.parTbName[0] = 0; + } + + if (pTagSup->numOfExprs > 0) { + projectApplyFunctions(pTagSup->pExprInfo, pDestBlock, pTmpBlock, pTagSup->pCtx, pTagSup->numOfExprs, NULL); + pDestBlock->info.rows--; + } + + void* pGpIdCol = taosArrayGet(pDestBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX); + colDataAppend(pGpIdCol, pDestBlock->info.rows, (const char*)&groupId, false); + + pDestBlock->info.rows++; + blockDataDestroy(pTmpBlock); + } + streamStateReleaseBuf(pState, NULL, pValue); +} + +static SSDataBlock* buildStreamCreateTableResult(SOperatorInfo* pOperator) { + SStreamPartitionOperatorInfo* pInfo = pOperator->info; + if ( (pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) || taosHashGetSize(pInfo->pPartitions) == 0) { + return NULL; + } + blockDataCleanup(pInfo->pCreateTbRes); + blockDataEnsureCapacity(pInfo->pCreateTbRes, taosHashGetSize(pInfo->pPartitions)); + SSDataBlock* pSrc = pInfo->pInputDataBlock; + + while (pInfo->pTbNameIte != NULL) { + SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->pTbNameIte; + int32_t rowId = *(int32_t*) taosArrayGet(pParInfo->rowIds, 0); + appendCreateTableRow(pOperator->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, + pParInfo->groupId, pSrc, rowId, pInfo->pCreateTbRes); + pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, pInfo->pTbNameIte); + } + return pInfo->pCreateTbRes->info.rows > 0 ? pInfo->pCreateTbRes : NULL; +} + static void doStreamHashPartitionImpl(SStreamPartitionOperatorInfo* pInfo, SSDataBlock* pBlock) { pInfo->pInputDataBlock = pBlock; for (int32_t i = 0; i < pBlock->info.rows; ++i) { @@ -998,6 +1040,15 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamPartitionOperatorInfo* pInfo = pOperator->info; + SSDataBlock* pCtRes = NULL; + + if (hasRemainTbName(pInfo)) { + pCtRes = buildStreamCreateTableResult(pOperator); + if (pCtRes != NULL) { + return pCtRes; + } + } + if (hasRemainPartion(pInfo)) { return buildStreamPartitionResult(pOperator); } @@ -1025,6 +1076,7 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { return pInfo->pDelRes; } break; default: + ASSERTS(pBlock->info.type == STREAM_CREATE_CHILD_TABLE, "invalid SSDataBlock type"); return pBlock; } @@ -1042,6 +1094,11 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; pInfo->parIte = taosHashIterate(pInfo->pPartitions, NULL); + pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, NULL); + pCtRes = buildStreamCreateTableResult(pOperator); + if (pCtRes != NULL) { + return pCtRes; + } return buildStreamPartitionResult(pOperator); } @@ -1062,6 +1119,7 @@ static void destroyStreamPartitionOperatorInfo(void* param) { cleanupExprSupp(&pInfo->tagCalSup); blockDataDestroy(pInfo->pDelRes); taosHashCleanup(pInfo->pPartitions); + blockDataDestroy(pInfo->pCreateTbRes); taosMemoryFreeClear(param); } @@ -1077,6 +1135,46 @@ void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup } } +SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag) { + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + pBlock->info.hasVarCol = false; + pBlock->info.id.groupId = 0; + pBlock->info.rows = 0; + pBlock->info.type = STREAM_CREATE_CHILD_TABLE; + pBlock->info.watermark = INT64_MIN; + + pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + SColumnInfoData infoData = {0}; + infoData.info.type = TSDB_DATA_TYPE_VARCHAR; + if (tbName->numOfExprs > 0) { + infoData.info.bytes = tbName->pExprInfo->base.resSchema.bytes; + } else { + infoData.info.bytes = 1; + } + pBlock->info.rowSize += infoData.info.bytes; + // sub table name + taosArrayPush(pBlock->pDataBlock, &infoData); + + SColumnInfoData gpIdData = {0}; + gpIdData.info.type = TSDB_DATA_TYPE_UBIGINT; + gpIdData.info.bytes = 8; + pBlock->info.rowSize += gpIdData.info.bytes; + // group id + taosArrayPush(pBlock->pDataBlock, &gpIdData); + + for (int32_t i = 0; i < tag->numOfExprs; i++) { + SColumnInfoData tagCol = {0}; + tagCol.info.type = tag->pExprInfo[i].base.resSchema.type; + tagCol.info.bytes = tag->pExprInfo[i].base.resSchema.bytes; + tagCol.info.precision = tag->pExprInfo[i].base.resSchema.precision; + // tag info + taosArrayPush(pBlock->pDataBlock, &tagCol); + pBlock->info.rowSize += tagCol.info.bytes; + } + + return pBlock; +} + SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) { SStreamPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamPartitionOperatorInfo)); @@ -1096,6 +1194,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr } } + pInfo->tbnameCalSup.numOfExprs = 0; if (pPartNode->pSubtable != NULL) { SExprInfo* pSubTableExpr = taosMemoryCalloc(1, sizeof(SExprInfo)); if (pSubTableExpr == NULL) { @@ -1110,9 +1209,10 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr } } + pInfo->tagCalSup.numOfExprs = 0; if (pPartNode->pTags != NULL) { int32_t numOfTags; - SExprInfo* pTagExpr = createExprInfo(pPartNode->pTags, NULL, &numOfTags); + SExprInfo* pTagExpr = createExpr(pPartNode->pTags, &numOfTags); if (pTagExpr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _error; @@ -1123,6 +1223,12 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr } } + if (pInfo->tbnameCalSup.numOfExprs != 0 || pInfo->tagCalSup.numOfExprs != 0) { + pInfo->pCreateTbRes = buildCreateTableBlock(&pInfo->tbnameCalSup, &pInfo->tagCalSup); + } else { + pInfo->pCreateTbRes = NULL; + } + int32_t keyLen = 0; code = initGroupOptrInfo(&pInfo->partitionSup.pGroupColVals, &keyLen, &pInfo->partitionSup.keyBuf, pInfo->partitionSup.pGroupCols); @@ -1139,6 +1245,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr blockDataEnsureCapacity(pInfo->binfo.pRes, 4096); pInfo->parIte = NULL; + pInfo->pTbNameIte = NULL; pInfo->pInputDataBlock = NULL; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 8a097a23ceeee705907cb8460a5b34f43985effa..88ed9eccb323561718a3a7cc1bc8dfd737bef53c 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -42,38 +42,40 @@ typedef struct SJoinOperatorInfo { static void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode); static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator); static void destroyMergeJoinOperator(void* param); -static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstream, int32_t numOfDownstream, - SSortMergeJoinPhysiNode* pJoinNode); +static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstream, int32_t num, + SSortMergeJoinPhysiNode* pJoinNode, const char* idStr); -static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstream, int32_t numOfDownstream, - SSortMergeJoinPhysiNode* pJoinNode) { +static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstream, int32_t num, + SSortMergeJoinPhysiNode* pJoinNode, const char* idStr) { SNode* pMergeCondition = pJoinNode->pMergeCondition; - if (nodeType(pMergeCondition) == QUERY_NODE_OPERATOR) { - SOperatorNode* pNode = (SOperatorNode*)pMergeCondition; - SColumnNode* col1 = (SColumnNode*)pNode->pLeft; - SColumnNode* col2 = (SColumnNode*)pNode->pRight; - SColumnNode* leftTsCol = NULL; - SColumnNode* rightTsCol = NULL; - if (col1->dataBlockId == col2->dataBlockId ) { + if (nodeType(pMergeCondition) != QUERY_NODE_OPERATOR) { + qError("not support this in join operator, %s", idStr); + return; // do not handle this + } + + SOperatorNode* pNode = (SOperatorNode*)pMergeCondition; + SColumnNode* col1 = (SColumnNode*)pNode->pLeft; + SColumnNode* col2 = (SColumnNode*)pNode->pRight; + SColumnNode* leftTsCol = NULL; + SColumnNode* rightTsCol = NULL; + if (col1->dataBlockId == col2->dataBlockId) { + leftTsCol = col1; + rightTsCol = col2; + } else { + if (col1->dataBlockId == pDownstream[0]->resultDataBlockId) { + ASSERT(col2->dataBlockId == pDownstream[1]->resultDataBlockId); leftTsCol = col1; rightTsCol = col2; } else { - if (col1->dataBlockId == pDownstream[0]->resultDataBlockId) { - ASSERT(col2->dataBlockId == pDownstream[1]->resultDataBlockId); - leftTsCol = col1; - rightTsCol = col2; - } else { - ASSERT(col1->dataBlockId == pDownstream[1]->resultDataBlockId); - ASSERT(col2->dataBlockId == pDownstream[0]->resultDataBlockId); - leftTsCol = col2; - rightTsCol = col1; - } + ASSERT(col1->dataBlockId == pDownstream[1]->resultDataBlockId); + ASSERT(col2->dataBlockId == pDownstream[0]->resultDataBlockId); + leftTsCol = col2; + rightTsCol = col1; } - setJoinColumnInfo(&pInfo->leftCol, leftTsCol); - setJoinColumnInfo(&pInfo->rightCol, rightTsCol); - } else { - ASSERT(false); - }} + } + setJoinColumnInfo(&pInfo->leftCol, leftTsCol); + setJoinColumnInfo(&pInfo->rightCol, rightTsCol); +} SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) { @@ -97,7 +99,7 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->exprSupp.numOfExprs = numOfCols; - extractTimeCondition(pInfo, pDownstream, numOfDownstream, pJoinNode); + extractTimeCondition(pInfo, pDownstream, numOfDownstream, pJoinNode, GET_TASKID(pTaskInfo)); if (pJoinNode->pOnConditions != NULL && pJoinNode->node.pConditions != NULL) { pInfo->pCondAfterMerge = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); @@ -364,8 +366,6 @@ static bool mergeJoinGetNextTimestamp(SOperatorInfo* pOperator, int64_t* pLeftTs char* pRightVal = colDataGetData(pRightCol, pJoinInfo->rightPos); *pRightTs = *(int64_t*)pRightVal; - ASSERT(pLeftCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(pRightCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); return true; } diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index de353b4bac837ecc544c179f40acede17741cc38..73c3ac43111569cdf2181dda2531fc19380d5b18 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -102,7 +102,8 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys } initResultSizeInfo(&pOperator->resultInfo, numOfRows); - code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); + code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, + pTaskInfo->streamInfo.pState); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -221,7 +222,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { blockDataCleanup(pFinalRes); SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - if (pTaskInfo->streamInfo.pReq) { + if (pTaskInfo->streamInfo.submit.msgStr) { pOperator->status = OP_OPENED; } @@ -277,7 +278,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { // for stream interval if (pBlock->info.type == STREAM_RETRIEVE || pBlock->info.type == STREAM_DELETE_RESULT || - pBlock->info.type == STREAM_DELETE_DATA) { + pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { // printDataBlock1(pBlock, "project1"); return pBlock; } @@ -317,7 +318,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { if (pProjectInfo->mergeDataBlocks) { if (pRes->info.rows > 0) { - pFinalRes->info.id.groupId = 0; //clear groupId + pFinalRes->info.id.groupId = 0; // clear groupId pFinalRes->info.version = pRes->info.version; // continue merge data, ignore the group id @@ -402,7 +403,8 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy initResultSizeInfo(&pOperator->resultInfo, numOfRows); blockDataEnsureCapacity(pResBlock, numOfRows); - int32_t code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfExpr, keyBufSize, pTaskInfo->id.str); + int32_t code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfExpr, keyBufSize, pTaskInfo->id.str, + pTaskInfo->streamInfo.pState); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 175e52565f919cf3e5c2dc104e04b6392184a2ba..fdfd81f52f7562c6a3d53985b9e30d8772504e02 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -170,6 +170,10 @@ static SResultRow* getTableGroupOutputBuf(SOperatorInfo* pOperator, uint64_t gro } *pPage = getBufPage(pTableScanInfo->base.pdInfo.pAggSup->pResultBuf, p1->pageId); + if (NULL == *pPage) { + return NULL; + } + return (SResultRow*)((char*)(*pPage) + p1->offset); } @@ -232,30 +236,6 @@ static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, if (!allColumnsHaveAgg) { return false; } - -#if 0 - // if (allColumnsHaveAgg == true) { - int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - - // todo create this buffer during creating operator - if (pBlock->pBlockAgg == NULL) { - pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, POINTER_BYTES); - if (pBlock->pBlockAgg == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); - } - } - - size_t num = taosArrayGetSize(pTableScanInfo->matchInfo.pList); - for (int32_t i = 0; i < num; ++i) { - SColMatchItem* pColMatchInfo = taosArrayGet(pTableScanInfo->matchInfo.pList, i); - if (!pColMatchInfo->needOutput) { - continue; - } - - pBlock->pBlockAgg[pColMatchInfo->dstSlotId] = pColAgg[i]; - } -#endif - return true; } @@ -326,12 +306,14 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->filterOutBlocks += 1; pCost->totalRows += pBlock->info.rows; + tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); pCost->skipBlocks += 1; + tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_SMA_LOAD) { pCost->loadBlockStatis += 1; @@ -341,6 +323,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); + tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else { qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo)); @@ -362,6 +345,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->filterOutBlocks += 1; (*status) = FUNC_DATA_REQUIRED_FILTEROUT; + tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } } @@ -376,7 +360,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; - + tsdbReleaseDataBlock(pTableScanInfo->dataReader); *status = FUNC_DATA_REQUIRED_FILTEROUT; return TSDB_CODE_SUCCESS; } @@ -1377,54 +1361,16 @@ void calBlockTag(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { } #endif -void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { +static void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { SExprSupp* pTbNameCalSup = &pInfo->tbnameCalSup; SStreamState* pState = pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState; - if (pTbNameCalSup == NULL || pTbNameCalSup->numOfExprs == 0) return; - if (pBlock == NULL || pBlock->info.rows == 0) return; - - void* tbname = NULL; - if (streamStateGetParName(pState, pBlock->info.id.groupId, &tbname) == 0) { - memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); - tdbFree(tbname); - return; - } else { + blockDataCleanup(pInfo->pCreateTbRes); + if (pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) { pBlock->info.parTbName[0] = 0; - } - tdbFree(tbname); - - SSDataBlock* pSrcBlock = blockCopyOneRow(pBlock, 0); - ASSERT(pSrcBlock->info.rows == 1); - - SSDataBlock* pResBlock = createDataBlock(); - pResBlock->info.rowSize = VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN; - SColumnInfoData data = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_TABLE_NAME_LEN, 0); - taosArrayPush(pResBlock->pDataBlock, &data); - blockDataEnsureCapacity(pResBlock, 1); - - projectApplyFunctions(pTbNameCalSup->pExprInfo, pResBlock, pSrcBlock, pTbNameCalSup->pCtx, 1, NULL); - ASSERT(pResBlock->info.rows == 1); - ASSERT(taosArrayGetSize(pResBlock->pDataBlock) == 1); - SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, 0); - ASSERT(pCol->info.type == TSDB_DATA_TYPE_VARCHAR); - - void* pData = colDataGetData(pCol, 0); - // TODO check tbname validation - if (pData != (void*)-1 && pData != NULL) { - memset(pBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN); - int32_t len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); - memcpy(pBlock->info.parTbName, varDataVal(pData), len); - /*pBlock->info.parTbName[len + 1] = 0;*/ } else { - pBlock->info.parTbName[0] = 0; + appendCreateTableRow(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, + pBlock->info.id.groupId, pBlock, 0, pInfo->pCreateTbRes); } - - if (pBlock->info.id.groupId && pBlock->info.parTbName[0]) { - streamStatePutParName(pState, pBlock->info.id.groupId, pBlock->info.parTbName); - } - - blockDataDestroy(pSrcBlock); - blockDataDestroy(pResBlock); } void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, @@ -1560,14 +1506,18 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { qDebug("queue scan called"); - if (pTaskInfo->streamInfo.pReq != NULL) { - if (pInfo->tqReader->pMsg == NULL) { - pInfo->tqReader->pMsg = pTaskInfo->streamInfo.pReq; - const SSubmitReq* pSubmit = pInfo->tqReader->pMsg; - if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) { - qError("submit msg messed up when initing stream submit block %p", pSubmit); - pInfo->tqReader->pMsg = NULL; - pTaskInfo->streamInfo.pReq = NULL; + if (pTaskInfo->streamInfo.submit.msgStr != NULL) { + if (pInfo->tqReader->msg2.msgStr == NULL) { + /*pInfo->tqReader->pMsg = pTaskInfo->streamInfo.pReq;*/ + + /*const SSubmitReq* pSubmit = pInfo->tqReader->pMsg;*/ + /*if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) {*/ + /*void* msgStr = pTaskInfo->streamInfo.*/ + SPackedData submit = pTaskInfo->streamInfo.submit; + if (tqReaderSetSubmitReq2(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) { + qError("submit msg messed up when initing stream submit block %p", submit.msgStr); + pInfo->tqReader->msg2 = (SPackedData){0}; + pInfo->tqReader->setMsg = 0; ASSERT(0); } } @@ -1575,10 +1525,10 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { blockDataCleanup(pInfo->pRes); SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; - while (tqNextDataBlock(pInfo->tqReader)) { + while (tqNextDataBlock2(pInfo->tqReader)) { SSDataBlock block = {0}; - int32_t code = tqRetrieveDataBlock(&block, pInfo->tqReader); + int32_t code = tqRetrieveDataBlock2(&block, pInfo->tqReader); if (code != TSDB_CODE_SUCCESS || block.info.rows == 0) { continue; @@ -1591,15 +1541,17 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { } } - pInfo->tqReader->pMsg = NULL; - pTaskInfo->streamInfo.pReq = NULL; + pInfo->tqReader->msg2 = (SPackedData){0}; + pInfo->tqReader->setMsg = 0; + pTaskInfo->streamInfo.submit = (SPackedData){0}; return NULL; } if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) { SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); if (pResult && pResult->info.rows > 0) { - qDebug("queue scan tsdb return %d rows", pResult->info.rows); + qDebug("queue scan tsdb return %d rows min:%" PRId64 " max:%" PRId64, pResult->info.rows, + pResult->info.window.skey, pResult->info.window.ekey); pTaskInfo->streamInfo.returned = 1; return pResult; } else { @@ -1720,47 +1672,30 @@ static void setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { } } +static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey) { + if (pInfo->pUpdateInfo) { + checkUpdateData(pInfo, true, pInfo->pRes, true); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, endKey); + if (pInfo->pUpdateDataRes->info.rows > 0) { + pInfo->updateResIndex = 0; + if (pInfo->pUpdateDataRes->info.type == STREAM_CLEAR) { + pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES; + } else if (pInfo->pUpdateDataRes->info.type == STREAM_INVERT) { + pInfo->scanMode = STREAM_SCAN_FROM_RES; + // return pInfo->pUpdateDataRes; + } else if (pInfo->pUpdateDataRes->info.type == STREAM_DELETE_DATA) { + pInfo->scanMode = STREAM_SCAN_FROM_DELETE_DATA; + } + } + } +} + static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamScanInfo* pInfo = pOperator->info; qDebug("stream scan called"); -#if 0 - SStreamState* pState = pTaskInfo->streamInfo.pState; - if (pState) { - printf(">>>>>>>> stream write backend\n"); - SWinKey key = { - .ts = 1, - .groupId = 2, - }; - char tmp[100] = "abcdefg1"; - if (streamStatePut(pState, &key, &tmp, strlen(tmp) + 1) < 0) { - ASSERT(0); - } - - key.ts = 2; - char tmp2[100] = "abcdefg2"; - if (streamStatePut(pState, &key, &tmp2, strlen(tmp2) + 1) < 0) { - ASSERT(0); - } - - key.groupId = 5; - key.ts = 1; - char tmp3[100] = "abcdefg3"; - if (streamStatePut(pState, &key, &tmp3, strlen(tmp3) + 1) < 0) { - ASSERT(0); - } - - char* val2 = NULL; - int32_t sz; - if (streamStateGet(pState, &key, (void**)&val2, &sz) < 0) { - ASSERT(0); - } - printf("stream read %s %d\n", val2, sz); - streamFreeVal(val2); - } -#endif if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE1 || pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE2) { @@ -1781,6 +1716,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { /*resetTableScanInfo(pTSInfo, pWin);*/ tsdbReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; + pInfo->pTableScanOp->status = OP_OPENED; pTSInfo->scanTimes = 0; pTSInfo->currentGroupId = -1; @@ -1794,17 +1730,32 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pInfo->blockRecoverContiCnt = 0; return NULL; } - SSDataBlock* pBlock = doTableScan(pInfo->pTableScanOp); - if (pBlock != NULL) { + + switch (pInfo->scanMode) { + case STREAM_SCAN_FROM_RES: { + pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; + printDataBlock(pInfo->pRecoverRes, "scan recover"); + return pInfo->pRecoverRes; + } break; + default: + break; + } + + pInfo->pRecoverRes = doTableScan(pInfo->pTableScanOp); + if (pInfo->pRecoverRes != NULL) { pInfo->blockRecoverContiCnt++; - calBlockTbName(pInfo, pBlock); + calBlockTbName(pInfo, pInfo->pRecoverRes); if (pInfo->pUpdateInfo) { - TSKEY maxTs = updateInfoFillBlockData(pInfo->pUpdateInfo, pBlock, pInfo->primaryTsIndex); + TSKEY maxTs = updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); } - qDebug("stream recover scan get block, rows %d", pBlock->info.rows); - printDataBlock(pBlock, "scan recover"); - return pBlock; + if (pInfo->pCreateTbRes->info.rows > 0) { + pInfo->scanMode = STREAM_SCAN_FROM_RES; + return pInfo->pCreateTbRes; + } + qDebug("stream recover scan get block, rows %d", pInfo->pRecoverRes->info.rows); + printDataBlock(pInfo->pRecoverRes, "scan recover"); + return pInfo->pRecoverRes; } pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; @@ -1829,7 +1780,8 @@ FETCH_NEXT_BLOCK: } int32_t current = pInfo->validBlockIndex++; - SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current); + SPackedData* pPacked = taosArrayGet(pInfo->pBlockLists, current); + SSDataBlock* pBlock = pPacked->pDataBlock; if (pBlock->info.id.groupId && pBlock->info.parTbName[0]) { streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, pBlock->info.parTbName); } @@ -1905,8 +1857,11 @@ FETCH_NEXT_BLOCK: qDebug("scan mode %d", pInfo->scanMode); switch (pInfo->scanMode) { case STREAM_SCAN_FROM_RES: { - blockDataDestroy(pInfo->pUpdateRes); pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; + doCheckUpdate(pInfo, pInfo->pRes->info.window.ekey); + doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); + pInfo->pRes->info.dataLoad = 1; + blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); return pInfo->pRes; } break; case STREAM_SCAN_FROM_DELETE_DATA: { @@ -1959,7 +1914,7 @@ FETCH_NEXT_BLOCK: NEXT_SUBMIT_BLK: while (1) { - if (pInfo->tqReader->pMsg == NULL) { + if (pInfo->tqReader->msg2.msgStr == NULL) { if (pInfo->validBlockIndex >= totBlockNum) { updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); doClearBufferedBlocks(pInfo); @@ -1967,22 +1922,22 @@ FETCH_NEXT_BLOCK: return NULL; } - int32_t current = pInfo->validBlockIndex++; - SSubmitReq* pSubmit = taosArrayGetP(pInfo->pBlockLists, current); - if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) { + int32_t current = pInfo->validBlockIndex++; + SPackedData* pSubmit = taosArrayGet(pInfo->pBlockLists, current); + /*if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) {*/ + if (tqReaderSetSubmitReq2(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) { qError("submit msg messed up when initing stream submit block %p, current %d, total %d", pSubmit, current, totBlockNum); - pInfo->tqReader->pMsg = NULL; continue; } } blockDataCleanup(pInfo->pRes); - while (tqNextDataBlock(pInfo->tqReader)) { + while (tqNextDataBlock2(pInfo->tqReader)) { SSDataBlock block = {0}; - int32_t code = tqRetrieveDataBlock(&block, pInfo->tqReader); + int32_t code = tqRetrieveDataBlock2(&block, pInfo->tqReader); if (code != TSDB_CODE_SUCCESS || block.info.rows == 0) { continue; @@ -1997,22 +1952,12 @@ FETCH_NEXT_BLOCK: continue; } - if (pInfo->pUpdateInfo) { - checkUpdateData(pInfo, true, pInfo->pRes, true); - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlockInfo->window.ekey); - if (pInfo->pUpdateDataRes->info.rows > 0) { - pInfo->updateResIndex = 0; - if (pInfo->pUpdateDataRes->info.type == STREAM_CLEAR) { - pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES; - } else if (pInfo->pUpdateDataRes->info.type == STREAM_INVERT) { - pInfo->scanMode = STREAM_SCAN_FROM_RES; - return pInfo->pUpdateDataRes; - } else if (pInfo->pUpdateDataRes->info.type == STREAM_DELETE_DATA) { - pInfo->scanMode = STREAM_SCAN_FROM_DELETE_DATA; - } - } + if (pInfo->pCreateTbRes->info.rows > 0) { + pInfo->scanMode = STREAM_SCAN_FROM_RES; + return pInfo->pCreateTbRes; } + doCheckUpdate(pInfo, pBlockInfo->window.ekey); doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); pInfo->pRes->info.dataLoad = 1; blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); @@ -2024,10 +1969,8 @@ FETCH_NEXT_BLOCK: if (pBlockInfo->rows > 0 || pInfo->pUpdateDataRes->info.rows > 0) { break; } else { - pInfo->tqReader->pMsg = NULL; continue; } - /*blockDataCleanup(pInfo->pRes);*/ } // record the scan action. @@ -2233,6 +2176,7 @@ static void destroyStreamScanOperatorInfo(void* param) { } cleanupExprSupp(&pStreamScan->tbnameCalSup); + cleanupExprSupp(&pStreamScan->tagCalSup); updateInfoDestroy(pStreamScan->pUpdateInfo); blockDataDestroy(pStreamScan->pRes); @@ -2240,12 +2184,14 @@ static void destroyStreamScanOperatorInfo(void* param) { blockDataDestroy(pStreamScan->pPullDataRes); blockDataDestroy(pStreamScan->pDeleteDataRes); blockDataDestroy(pStreamScan->pUpdateDataRes); + blockDataDestroy(pStreamScan->pCreateTbRes); taosArrayDestroy(pStreamScan->pBlockLists); taosMemoryFree(pStreamScan); } SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, SExecTaskInfo* pTaskInfo) { + SArray* pColIds = NULL; SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -2268,7 +2214,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys } int32_t numOfOutput = taosArrayGetSize(pInfo->matchInfo.pList); - SArray* pColIds = taosArrayInit(numOfOutput, sizeof(int16_t)); + pColIds = taosArrayInit(numOfOutput, sizeof(int16_t)); for (int32_t i = 0; i < numOfOutput; ++i) { SColMatchItem* id = taosArrayGet(pInfo->matchInfo.pList, i); @@ -2294,7 +2240,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys if (pTableScanNode->pTags != NULL) { int32_t numOfTags; - SExprInfo* pTagExpr = createExprInfo(pTableScanNode->pTags, NULL, &numOfTags); + SExprInfo* pTagExpr = createExpr(pTableScanNode->pTags, &numOfTags); if (pTagExpr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _error; @@ -2305,7 +2251,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys } } - pInfo->pBlockLists = taosArrayInit(4, POINTER_BYTES); + pInfo->pBlockLists = taosArrayInit(4, sizeof(SPackedData)); if (pInfo->pBlockLists == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _error; @@ -2352,6 +2298,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->readHandle = *pHandle; pInfo->tableUid = pScanPhyNode->uid; pTaskInfo->streamInfo.snapshotVer = pHandle->version; + pInfo->pCreateTbRes = buildCreateTableBlock(&pInfo->tbnameCalSup, &pInfo->tagCalSup); + blockDataEnsureCapacity(pInfo->pCreateTbRes, 8); // set the extract column id to streamHandle tqReaderSetColIdList(pInfo->tqReader, pColIds); @@ -2365,6 +2313,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys memcpy(&pTaskInfo->streamInfo.tableCond, &pTSInfo->base.cond, sizeof(SQueryTableDataCond)); } else { taosArrayDestroy(pColIds); + pColIds = NULL; } // create the pseduo columns info @@ -2567,6 +2516,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } STsdbReader* reader = pInfo->base.dataReader; + qTrace("tsdb/read-table-data: %p, enter next reader", reader); while (tsdbNextDataBlock(reader)) { if (isTaskKilled(pTaskInfo)) { T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); @@ -2601,6 +2551,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { pOperator->resultInfo.totalRows += pBlock->info.rows; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; + qTrace("tsdb/read-table-data: %p, close reader", reader); tsdbReaderClose(pInfo->base.dataReader); pInfo->base.dataReader = NULL; return pBlock; @@ -3271,7 +3222,9 @@ static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, S uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); pRes->info.id.groupId = groupId; int64_t ntbNum = metaGetNtbNum(pInfo->readHandle.meta); - fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes); + if (ntbNum != 0) { + fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes); + } } static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 7ac007b7cb914f5b4fe3faa8110d3acd0b72a852..f5dc6cc623ca800cb459923cf69b495e70447360 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -46,13 +46,15 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pOperator->pTaskInfo = pTaskInfo; SDataBlockDescNode* pDescNode = pSortNode->node.pOutputDataBlockDesc; - int32_t numOfCols = 0; - SSDataBlock* pResBlock = createDataBlockFromDescNode(pDescNode); - SExprInfo* pExprInfo = createExprInfo(pSortNode->pExprs, NULL, &numOfCols); + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pSortNode->pExprs, NULL, &numOfCols); int32_t numOfOutputCols = 0; int32_t code = extractColMatchInfo(pSortNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset); initResultSizeInfo(&pOperator->resultInfo, 1024); @@ -61,7 +63,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* goto _error; } - pInfo->binfo.pRes = pResBlock; + pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode); pInfo->pSortInfo = createSortInfo(pSortNode->pSortKeys); initLimitInfo(pSortNode->node.pLimit, pSortNode->node.pSlimit, &pInfo->limitInfo); @@ -86,7 +88,10 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* _error: pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(pInfo); + if (pInfo != NULL) { + destroySortOperatorInfo(pInfo); + } + taosMemoryFree(pOperator); return NULL; } @@ -139,7 +144,6 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i int32_t numOfCols = taosArrayGetSize(pColMatchInfo); for (int32_t i = 0; i < numOfCols; ++i) { SColMatchItem* pmInfo = taosArrayGet(pColMatchInfo, i); - // ASSERT(pmInfo->matchType == COL_MATCH_FROM_SLOT_ID); SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->dstSlotId); @@ -272,7 +276,6 @@ void destroySortOperatorInfo(void* param) { } int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { - ASSERT(pOptr != NULL); SSortExecInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo)); SSortOperatorInfo* pOperatorInfo = (SSortOperatorInfo*)pOptr->info; @@ -329,7 +332,6 @@ SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo int32_t numOfCols = taosArrayGetSize(pColMatchInfo); for (int32_t i = 0; i < numOfCols; ++i) { SColMatchItem* pmInfo = taosArrayGet(pColMatchInfo, i); - // ASSERT(pmInfo->matchType == COL_MATCH_FROM_SLOT_ID); SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->dstSlotId); @@ -746,7 +748,6 @@ void destroyMultiwayMergeOperatorInfo(void* param) { } int32_t getMultiwayMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { - ASSERT(pOptr != NULL); SSortExecInfo* pSortExecInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo)); SMultiwayMergeOperatorInfo* pInfo = (SMultiwayMergeOperatorInfo*)pOptr->info; diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index a88f673e0b33135f41c1fa5e868d61c66942ee9f..ddcaaf2c72778d1a8488ee353722bef2e337fb42 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -65,6 +65,8 @@ typedef struct SSysTableScanInfo { SSDataBlock* pRes; int64_t numOfBlocks; // extract basic running information. SLoadRemoteDataInfo loadInfo; + + int32_t tbnameSlotId; } SSysTableScanInfo; typedef struct { @@ -138,6 +140,10 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrChildTable, const char* dbname, const char* tableName, int32_t* pNumOfRows, const SSDataBlock* dataBlock); +static int32_t sysTableUserColsFillOneTableCols(const SSysTableScanInfo* pInfo, const char* dbname, + int32_t* pNumOfRows, const SSDataBlock* dataBlock, + char* tName, SSchemaWrapper* schemaRow, char* tableType); + static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock, SFilterInfo* pFilterInfo); @@ -346,6 +352,11 @@ static int32_t optSysTabFilteImpl(void* arg, SNode* cond, SArray* result); static int32_t optSysCheckOper(SNode* pOpear); static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt); +static SSDataBlock* sysTableScanFromMNode(SOperatorInfo* pOperator, SSysTableScanInfo* pInfo, const char* name, + SExecTaskInfo* pTaskInfo); +void extractTbnameSlotId(SSysTableScanInfo* pInfo, const SScanPhysiNode* pScanNode); +static SSDataBlock* sysTableScanFillTbName(SOperatorInfo* pOperator, const SSysTableScanInfo* pInfo, + const char* name, SSDataBlock* pBlock); __optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse) { if (ctype == OP_TYPE_LOWER_EQUAL || ctype == OP_TYPE_LOWER_THAN) { *reverse = true; @@ -406,6 +417,176 @@ static bool sysTableIsCondOnOneTable(SNode* pCond, char* condTable) { return false; } +static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { + qDebug("sysTableScanUserCols get cols start"); + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + blockDataCleanup(pInfo->pRes); + int32_t numOfRows = 0; + + SSDataBlock* dataBlock = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_COLS); + blockDataEnsureCapacity(dataBlock, pOperator->resultInfo.capacity); + + const char* db = NULL; + int32_t vgId = 0; + vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + + tNameGetDbName(&sn, varDataVal(dbname)); + varDataSetLen(dbname, strlen(varDataVal(dbname))); + + // optimize when sql like where table_name='tablename' and xxx. + if (pInfo->req.filterTb[0]) { + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(tableName, pInfo->req.filterTb); + + SMetaReader smrTable = {0}; + metaReaderInit(&smrTable, pInfo->readHandle.meta, 0); + int32_t code = metaGetTableEntryByName(&smrTable, pInfo->req.filterTb); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByName, therefore, return directly + metaReaderClear(&smrTable); + blockDataDestroy(dataBlock); + pInfo->loadInfo.totalRows = 0; + return NULL; + } + + if (smrTable.me.type == TSDB_SUPER_TABLE) { + metaReaderClear(&smrTable); + blockDataDestroy(dataBlock); + pInfo->loadInfo.totalRows = 0; + return NULL; + } + + if (smrTable.me.type == TSDB_CHILD_TABLE) { + int64_t suid = smrTable.me.ctbEntry.suid; + metaReaderClear(&smrTable); + metaReaderInit(&smrTable, pInfo->readHandle.meta, 0); + code = metaGetTableEntryByUid(&smrTable, suid); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByName, therefore, return directly + metaReaderClear(&smrTable); + blockDataDestroy(dataBlock); + pInfo->loadInfo.totalRows = 0; + return NULL; + } + } + + char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + SSchemaWrapper *schemaRow = NULL; + if(smrTable.me.type == TSDB_SUPER_TABLE){ + schemaRow = &smrTable.me.stbEntry.schemaRow; + STR_TO_VARSTR(typeName, "CHILD_TABLE"); + }else if(smrTable.me.type == TSDB_NORMAL_TABLE){ + schemaRow = &smrTable.me.ntbEntry.schemaRow; + STR_TO_VARSTR(typeName, "NORMAL_TABLE"); + } + + sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName); + metaReaderClear(&smrTable); + + if (numOfRows > 0) { + relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); + numOfRows = 0; + } + blockDataDestroy(dataBlock); + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + setOperatorCompleted(pOperator); + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; + } + + int32_t ret = 0; + if (pInfo->pCur == NULL) { + pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + } + + SHashObj *stableSchema = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + taosHashSetFreeFp(stableSchema, tDeleteSSchemaWrapperForHash); + while ((ret = metaTbCursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0) { + char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + + SSchemaWrapper *schemaRow = NULL; + + if(pInfo->pCur->mr.me.type == TSDB_SUPER_TABLE){ + qDebug("sysTableScanUserCols cursor get super table"); + void *schema = taosHashGet(stableSchema, &pInfo->pCur->mr.me.uid, sizeof(int64_t)); + if(schema == NULL){ + SSchemaWrapper *schemaWrapper = tCloneSSchemaWrapper(&pInfo->pCur->mr.me.stbEntry.schemaRow); + taosHashPut(stableSchema, &pInfo->pCur->mr.me.uid, sizeof(int64_t), &schemaWrapper, POINTER_BYTES); + } + continue; + }else if (pInfo->pCur->mr.me.type == TSDB_CHILD_TABLE) { + qDebug("sysTableScanUserCols cursor get child table"); + STR_TO_VARSTR(typeName, "CHILD_TABLE"); + STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); + + int64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; + void *schema = taosHashGet(stableSchema, &pInfo->pCur->mr.me.ctbEntry.suid, sizeof(int64_t)); + if(schema != NULL){ + schemaRow = *(SSchemaWrapper **)schema; + }else{ + tDecoderClear(&pInfo->pCur->mr.coder); + int code = metaGetTableEntryByUid(&pInfo->pCur->mr, suid); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByName, therefore, return directly + qError("sysTableScanUserCols get meta by suid:%"PRId64 " error, code:%d", suid, code); + blockDataDestroy(dataBlock); + pInfo->loadInfo.totalRows = 0; + taosHashCleanup(stableSchema); + return NULL; + } + schemaRow = &pInfo->pCur->mr.me.stbEntry.schemaRow; + } + }else if(pInfo->pCur->mr.me.type == TSDB_NORMAL_TABLE){ + qDebug("sysTableScanUserCols cursor get normal table"); + schemaRow = &pInfo->pCur->mr.me.ntbEntry.schemaRow; + STR_TO_VARSTR(typeName, "NORMAL_TABLE"); + STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); + }else{ + qDebug("sysTableScanUserCols cursor get invalid table"); + continue; + } + + sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName); + + if (numOfRows >= pOperator->resultInfo.capacity) { + relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); + numOfRows = 0; + + if (pInfo->pRes->info.rows > 0) { + break; + } + } + } + + taosHashCleanup(stableSchema); + + if (numOfRows > 0) { + relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); + numOfRows = 0; + } + + blockDataDestroy(dataBlock); + if (ret != 0) { + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + setOperatorCompleted(pOperator); + } + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + qDebug("sysTableScanUserCols get cols success, rows:%" PRIu64, pInfo->loadInfo.totalRows); + + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; +} + static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSysTableScanInfo* pInfo = pOperator->info; @@ -484,7 +665,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); } - while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { + while ((ret = metaTbCursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) { if (pInfo->pCur->mr.me.type != TSDB_CHILD_TABLE) { continue; } @@ -721,6 +902,67 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, return TSDB_CODE_SUCCESS; } +static int32_t sysTableUserColsFillOneTableCols(const SSysTableScanInfo* pInfo, const char* dbname, + int32_t* pNumOfRows, const SSDataBlock* dataBlock, char* tName, + SSchemaWrapper* schemaRow, char* tableType) { + if(schemaRow == NULL){ + qError("sysTableUserColsFillOneTableCols schemaRow is NULL"); + return TSDB_CODE_SUCCESS; + } + int32_t numOfRows = *pNumOfRows; + + int32_t numOfCols = schemaRow->nCols; + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfoData = NULL; + + // table name + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 0); + colDataAppend(pColInfoData, numOfRows, tName, false); + + // database name + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, dbname, false); + + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, tableType, false); + + // col name + char colName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(colName, schemaRow->pSchema[i].name); + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, colName, false); + + // col type + int8_t colType = schemaRow->pSchema[i].type; + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 4); + char colTypeStr[VARSTR_HEADER_SIZE + 32]; + int colTypeLen = sprintf(varDataVal(colTypeStr), "%s", tDataTypes[colType].name); + if (colType == TSDB_DATA_TYPE_VARCHAR) { + colTypeLen += sprintf(varDataVal(colTypeStr) + colTypeLen, "(%d)", + (int32_t)(schemaRow->pSchema[i].bytes - VARSTR_HEADER_SIZE)); + } else if (colType == TSDB_DATA_TYPE_NCHAR) { + colTypeLen += sprintf( + varDataVal(colTypeStr) + colTypeLen, "(%d)", + (int32_t)((schemaRow->pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); + } + varDataSetLen(colTypeStr, colTypeLen); + colDataAppend(pColInfoData, numOfRows, (char*)colTypeStr, false); + + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (const char*)&schemaRow->pSchema[i].bytes, false); + + for (int32_t j = 6; j <= 8; ++j) { + pColInfoData = taosArrayGet(dataBlock->pDataBlock, j); + colDataAppendNULL(pColInfoData, numOfRows); + } + ++numOfRows; + } + + *pNumOfRows = numOfRows; + + return TSDB_CODE_SUCCESS; +} + static SSDataBlock* buildInfoSchemaTableMetaBlock(char* tableName) { size_t size = 0; const SSysTableMeta* pMeta = NULL; @@ -1022,7 +1264,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t ret = 0; - while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { + while ((ret = metaTbCursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) { STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); // table name @@ -1308,90 +1550,126 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { if (pInfo->showRewrite) { getDBNameFromCondition(pInfo->pCondition, dbName); sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); + }else if(strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0){ + getDBNameFromCondition(pInfo->pCondition, dbName); + if(dbName[0]) sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); + sysTableIsCondOnOneTable(pInfo->pCondition, pInfo->req.filterTb); } + SSDataBlock* pBlock = NULL; if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { - return sysTableScanUserTables(pOperator); + pBlock = sysTableScanUserTables(pOperator); } else if (strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { - return sysTableScanUserTags(pOperator); + pBlock = sysTableScanUserTags(pOperator); + } else if (strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->readHandle.mnd == NULL) { + pBlock = sysTableScanUserCols(pOperator); } else if (strncasecmp(name, TSDB_INS_TABLE_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->showRewrite && IS_SYS_DBNAME(dbName)) { - return sysTableScanUserSTables(pOperator); + pBlock = sysTableScanUserSTables(pOperator); } else { // load the meta from mnode of the given epset - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - while (1) { - int64_t startTs = taosGetTimestampUs(); - tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); - tstrncpy(pInfo->req.user, pInfo->pUser, tListLen(pInfo->req.user)); + pBlock = sysTableScanFromMNode(pOperator, pInfo, name, pTaskInfo); + } - int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); - char* buf1 = taosMemoryCalloc(1, contLen); - tSerializeSRetrieveTableReq(buf1, contLen, &pInfo->req); + return sysTableScanFillTbName(pOperator, pInfo, name, pBlock); +} - // send the fetch remote task result reques - SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (NULL == pMsgSendInfo) { - qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - return NULL; +static SSDataBlock* sysTableScanFillTbName(SOperatorInfo* pOperator, const SSysTableScanInfo* pInfo, + const char* name, SSDataBlock* pBlock) { + if (pBlock != NULL) { + if (pInfo->tbnameSlotId != -1) { + SColumnInfoData* pColumnInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, pInfo->tbnameSlotId); + char varTbName[TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE] = {0}; + memcpy(varDataVal(varTbName), name, strlen(name)); + varDataSetLen(varTbName, strlen(name)); + for (int i = 0; i < pBlock->info.rows; ++i) { + colDataAppend(pColumnInfoData, i, varTbName, NULL); } + doFilterResult(pBlock, pOperator->exprSupp.pFilterInfo); + } + } + if (pBlock && pBlock->info.rows != 0) { + return pBlock; + } else { + return NULL; + } +} - int32_t msgType = (strcasecmp(name, TSDB_INS_TABLE_DNODE_VARIABLES) == 0) ? TDMT_DND_SYSTABLE_RETRIEVE - : TDMT_MND_SYSTABLE_RETRIEVE; +static SSDataBlock* sysTableScanFromMNode(SOperatorInfo* pOperator, SSysTableScanInfo* pInfo, const char* name, + SExecTaskInfo* pTaskInfo) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } - pMsgSendInfo->param = pOperator; - pMsgSendInfo->msgInfo.pData = buf1; - pMsgSendInfo->msgInfo.len = contLen; - pMsgSendInfo->msgType = msgType; - pMsgSendInfo->fp = loadSysTableCallback; - pMsgSendInfo->requestId = pTaskInfo->id.queryId; + while (1) { + int64_t startTs = taosGetTimestampUs(); + tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); + tstrncpy(pInfo->req.user, pInfo->pUser, tListLen(pInfo->req.user)); - int64_t transporterId = 0; - int32_t code = - asyncSendMsgToServer(pInfo->readHandle.pMsgCb->clientRpc, &pInfo->epSet, &transporterId, pMsgSendInfo); - tsem_wait(&pInfo->ready); + int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); + char* buf1 = taosMemoryCalloc(1, contLen); + tSerializeSRetrieveTableReq(buf1, contLen, &pInfo->req); - if (pTaskInfo->code) { - qDebug("%s load meta data from mnode failed, totalRows:%" PRIu64 ", code:%s", GET_TASKID(pTaskInfo), - pInfo->loadInfo.totalRows, tstrerror(pTaskInfo->code)); - return NULL; - } + // send the fetch remote task result reques + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } - SRetrieveMetaTableRsp* pRsp = pInfo->pRsp; - pInfo->req.showId = pRsp->handle; + int32_t msgType = (strcasecmp(name, TSDB_INS_TABLE_DNODE_VARIABLES) == 0) ? TDMT_DND_SYSTABLE_RETRIEVE + : TDMT_MND_SYSTABLE_RETRIEVE; - if (pRsp->numOfRows == 0 || pRsp->completed) { - pOperator->status = OP_EXEC_DONE; - qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64, GET_TASKID(pTaskInfo), - pRsp->numOfRows, pInfo->loadInfo.totalRows); + pMsgSendInfo->param = pOperator; + pMsgSendInfo->msgInfo.pData = buf1; + pMsgSendInfo->msgInfo.len = contLen; + pMsgSendInfo->msgType = msgType; + pMsgSendInfo->fp = loadSysTableCallback; + pMsgSendInfo->requestId = pTaskInfo->id.queryId; - if (pRsp->numOfRows == 0) { - taosMemoryFree(pRsp); - return NULL; - } - } + int64_t transporterId = 0; + int32_t code = + asyncSendMsgToServer(pInfo->readHandle.pMsgCb->clientRpc, &pInfo->epSet, &transporterId, pMsgSendInfo); + tsem_wait(&pInfo->ready); - char* pStart = pRsp->data; - extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pInfo->matchInfo.pList, &pStart); - updateLoadRemoteInfo(&pInfo->loadInfo, pRsp->numOfRows, pRsp->compLen, startTs, pOperator); + if (pTaskInfo->code) { + qError("%s load meta data from mnode failed, totalRows:%" PRIu64 ", code:%s", GET_TASKID(pTaskInfo), + pInfo->loadInfo.totalRows, tstrerror(pTaskInfo->code)); + return NULL; + } - // todo log the filter info - doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); - taosMemoryFree(pRsp); - if (pInfo->pRes->info.rows > 0) { - return pInfo->pRes; - } else if (pOperator->status == OP_EXEC_DONE) { + SRetrieveMetaTableRsp* pRsp = pInfo->pRsp; + pInfo->req.showId = pRsp->handle; + + if (pRsp->numOfRows == 0 || pRsp->completed) { + pOperator->status = OP_EXEC_DONE; + qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64, GET_TASKID(pTaskInfo), + pRsp->numOfRows, pInfo->loadInfo.totalRows); + + if (pRsp->numOfRows == 0) { + taosMemoryFree(pRsp); return NULL; } } + + char* pStart = pRsp->data; + extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pInfo->matchInfo.pList, &pStart); + updateLoadRemoteInfo(&pInfo->loadInfo, pRsp->numOfRows, pRsp->compLen, startTs, pOperator); + + // todo log the filter info + doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); + taosMemoryFree(pRsp); + if (pInfo->pRes->info.rows > 0) { + return pInfo->pRes; + } else if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } } } SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo) { + int32_t code = TDB_CODE_SUCCESS; SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -1402,11 +1680,13 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; int32_t num = 0; - int32_t code = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo); + code = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo); if (code != TSDB_CODE_SUCCESS) { goto _error; } + extractTbnameSlotId(pInfo, pScanNode); + pInfo->accountId = pScanPhyNode->accountId; pInfo->pUser = taosMemoryStrDup((void*)pUser); pInfo->sysInfo = pScanPhyNode->sysInfo; @@ -1449,6 +1729,26 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan return NULL; } +void extractTbnameSlotId(SSysTableScanInfo* pInfo, const SScanPhysiNode* pScanNode) { + pInfo->tbnameSlotId = -1; + if (pScanNode->pScanPseudoCols != NULL) { + SNode* pNode = NULL; + FOREACH(pNode, pScanNode->pScanPseudoCols) { + STargetNode* pTargetNode = NULL; + if (nodeType(pNode) == QUERY_NODE_TARGET) { + pTargetNode = (STargetNode*)pNode; + SNode* expr = pTargetNode->pExpr; + if (nodeType(expr) == QUERY_NODE_FUNCTION) { + SFunctionNode* pFuncNode = (SFunctionNode*)expr; + if (pFuncNode->funcType == FUNCTION_TYPE_TBNAME) { + pInfo->tbnameSlotId = pTargetNode->slotId; + } + } + } + } + } +} + void destroySysScanOperator(void* param) { SSysTableScanInfo* pInfo = (SSysTableScanInfo*)param; tsem_destroy(&pInfo->ready); @@ -1456,7 +1756,8 @@ void destroySysScanOperator(void* param) { const char* name = tNameGetTableName(&pInfo->name); if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || - strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { + strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || + strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0|| pInfo->pCur != NULL) { metaCloseTbCursor(pInfo->pCur); pInfo->pCur = NULL; } @@ -1861,6 +2162,13 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { colDataAppend(pColInfo, 0, p, false); taosMemoryFree(p); + // make the valgrind happy that all memory buffer has been initialized already. + if (slotId != 0) { + SColumnInfoData* p1 = taosArrayGet(pBlock->pDataBlock, 0); + int64_t v = 0; + colDataAppendInt64(p1, 0, &v); + } + pBlock->info.rows = 1; pOperator->status = OP_EXEC_DONE; return pBlock; diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index ab1feabf60b235f3df62c6b8e487dff3e7071350..2d921d43d34726b64b0be3dde97a52a95cf19f61 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -39,14 +39,10 @@ static void doSetVal(SColumnInfoData* pDstColInfoData, int32_t rowIndex, const S static void setNotFillColumn(SFillInfo* pFillInfo, SColumnInfoData* pDstColInfo, int32_t rowIndex, int32_t colIdx) { SRowVal* p = NULL; - if (FILL_IS_ASC_FILL(pFillInfo)) { - if (pFillInfo->prev.key != 0) { - p = &pFillInfo->prev; // prev has been set value - } else { // otherwise, use the value in the next row - p = &pFillInfo->next; - } + if (pFillInfo->type == TSDB_FILL_NEXT) { + p = FILL_IS_ASC_FILL(pFillInfo) ? &pFillInfo->next : &pFillInfo->prev; } else { - p = &pFillInfo->next; + p = FILL_IS_ASC_FILL(pFillInfo) ? &pFillInfo->prev : &pFillInfo->next; } SGroupKeys* pKey = taosArrayGet(p->pRowVal, colIdx); @@ -257,9 +253,16 @@ static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SRowVa for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { int32_t type = pFillInfo->pFillCol[i].pExpr->pExpr->nodeType; - if (type == QUERY_NODE_COLUMN || type == QUERY_NODE_OPERATOR || type == QUERY_NODE_FUNCTION) { + if ( type == QUERY_NODE_COLUMN || type == QUERY_NODE_OPERATOR || type == QUERY_NODE_FUNCTION) { + if (!pFillInfo->pFillCol[i].notFillCol && pFillInfo->type != TSDB_FILL_NEXT) { + continue; + } int32_t srcSlotId = GET_DEST_SLOT_ID(&pFillInfo->pFillCol[i]); + if (srcSlotId == pFillInfo->srcTsSlotId && pFillInfo->type == TSDB_FILL_LINEAR) { + continue; + } + SColumnInfoData* pSrcCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, srcSlotId); bool isNull = colDataIsNull_s(pSrcCol, rowIndex); @@ -288,8 +291,12 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t int64_t ts = ((int64_t*)pTsCol->pData)[pFillInfo->index]; // set the next value for interpolation - if ((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) { - copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, &pFillInfo->next); + if (pFillInfo->currentKey < ts && ascFill) { + SRowVal* pRVal = pFillInfo->type == TSDB_FILL_NEXT ? &pFillInfo->next : &pFillInfo->prev; + copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pRVal); + } else if (pFillInfo->currentKey > ts && !ascFill) { + SRowVal* pRVal = pFillInfo->type == TSDB_FILL_NEXT ? &pFillInfo->prev : &pFillInfo->next; + copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pRVal); } if (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) && diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 32e0c517b43f6a30a9ae937d7af71c9270fe26bf..eccdcb85bf048969d6e04d98c9dc4a4bc24dacb8 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -22,7 +22,7 @@ #include "tfill.h" #include "ttime.h" -#define IS_FINAL_OP(op) ((op)->isFinal) +#define IS_FINAL_OP(op) ((op)->isFinal) #define DEAULT_DELETE_MARK (1000LL * 60LL * 60LL * 24LL * 365LL * 10LL); typedef struct SSessionAggOperatorInfo { @@ -119,8 +119,8 @@ static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsL pRowSup->groupId = groupId; } -static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, - int16_t pos, int16_t order, int64_t* pData) { +FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, + int32_t pos, int32_t order, int64_t* pData) { int32_t forwardRows = 0; if (order == TSDB_ORDER_ASC) { @@ -636,6 +636,10 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num } SResultRow* pr = getResultRowByPos(pInfo->aggSup.pResultBuf, p1, false); + if (NULL == pr) { + T_LONG_JMP(pTaskInfo->env, terrno); + } + ASSERT(pr->offset == p1->offset && pr->pageId == p1->pageId); if (pr->closed) { @@ -666,8 +670,8 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num setNotInterpoWindowKey(pSup->pCtx, numOfExprs, RESULT_ROW_START_INTERP); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &w, true); - applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, 0, pBlock->info.rows, - numOfExprs); + applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, 0, + pBlock->info.rows, numOfExprs); if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) { closeResultRow(pr); @@ -817,13 +821,13 @@ static int32_t savePullWindow(SPullWindowInfo* pPullInfo, SArray* pPullWins) { } else { int32_t code = comparePullWinKey(pPullInfo, pPullWins, index); if (code == 0) { - SPullWindowInfo* pos = taosArrayGet(pPullWins ,index); + SPullWindowInfo* pos = taosArrayGet(pPullWins, index); pos->window.skey = TMIN(pos->window.skey, pPullInfo->window.skey); pos->window.ekey = TMAX(pos->window.ekey, pPullInfo->window.ekey); pos->calWin.skey = TMIN(pos->calWin.skey, pPullInfo->calWin.skey); pos->calWin.ekey = TMAX(pos->calWin.ekey, pPullInfo->calWin.ekey); return TSDB_CODE_SUCCESS; - } else if (code > 0 ){ + } else if (code > 0) { index++; } } @@ -879,10 +883,10 @@ int32_t compareWinRes(void* pKey, void* data, int32_t index) { } else if (pRKey->groupId < pDataPos->groupId) { return -1; } - + if (*(int64_t*)pRKey->key > pDataPos->ts) { return 1; - } else if (*(int64_t*)pRKey->key < pDataPos->ts){ + } else if (*(int64_t*)pRKey->key < pDataPos->ts) { return -1; } return 0; @@ -907,7 +911,7 @@ static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) { } bool isOverdue(TSKEY ekey, STimeWindowAggSupp* pTwSup) { - ASSERT(pTwSup->maxTs == INT64_MIN || pTwSup->maxTs > 0); + ASSERTS(pTwSup->maxTs == INT64_MIN || pTwSup->maxTs > 0, "maxts should greater than 0"); return pTwSup->maxTs != INT64_MIN && ekey < pTwSup->maxTs - pTwSup->waterMark; } @@ -961,8 +965,8 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); - applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows, - numOfOutput); + applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, + pBlock->info.rows, numOfOutput); doCloseWindow(pResultRowInfo, pInfo, pResult); @@ -996,8 +1000,8 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul } #endif updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows, - numOfOutput); + applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, + pBlock->info.rows, numOfOutput); doCloseWindow(pResultRowInfo, pInfo, pResult); } @@ -1164,7 +1168,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, - pRowSup->numOfRows, pBlock->info.rows, numOfOutput); + pRowSup->numOfRows, pBlock->info.rows, numOfOutput); // here we start a new session window doKeepNewWindowStartInfo(pRowSup, tsList, j, gid); @@ -1188,8 +1192,8 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); - applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, - pBlock->info.rows, numOfOutput); + applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + pRowSup->numOfRows, pBlock->info.rows, numOfOutput); } static int32_t openStateWindowAggOptr(SOperatorInfo* pOperator) { @@ -1315,6 +1319,10 @@ static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp* pSup, int32_t numOfOutput) { SResultRow* pResult = getResultRowByPos(pResultBuf, p1, false); + if (NULL == pResult) { + return; + } + SqlFunctionCtx* pCtx = pSup->pCtx; for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].resultInfo = getResultEntryInfo(pResult, i, pSup->rowEntryInfoOffset); @@ -1328,6 +1336,9 @@ static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, } } SFilePage* bufPage = getBufPage(pResultBuf, p1->pageId); + if (NULL == bufPage) { + return; + } setBufPageDirty(bufPage, true); releaseBufPage(pResultBuf, bufPage); } @@ -1394,9 +1405,8 @@ static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) { size_t keyLen = 0; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { - void* key = tSimpleHashGetKey(pIte, &keyLen); - uint64_t groupId = *(uint64_t*)key; - ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))); + void* key = tSimpleHashGetKey(pIte, &keyLen); + uint64_t groupId = *(uint64_t*)key; TSKEY ts = *(int64_t*)((char*)key + sizeof(uint64_t)); SResultRowPosition* pPos = (SResultRowPosition*)pIte; int32_t code = saveWinResult(ts, pPos->pageId, pPos->offset, groupId, resWins); @@ -1547,7 +1557,7 @@ static void closeChildIntervalWindow(SOperatorInfo* pOperator, SArray* pChildren for (int32_t i = 0; i < size; i++) { SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i); SStreamIntervalOperatorInfo* pChInfo = pChildOp->info; - ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE); + ASSERTS(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE, "children trigger type should be at once"); pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs); closeStreamIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL, NULL, pOperator); @@ -1657,7 +1667,7 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt // the primary timestamp column bool needed = false; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SExprInfo* pExpr = pCtx[i].pExpr; if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) { needed = true; @@ -1725,12 +1735,12 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSuppor void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) { for (int32_t i = 0; i < numOfExpr; i++) { -// pCtx[i].isStream = true; + // pCtx[i].isStream = true; } } SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode, - SExecTaskInfo* pTaskInfo, bool isStream) { + SExecTaskInfo* pTaskInfo) { SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -1749,7 +1759,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num); - int32_t code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str); + int32_t code = + initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, pTaskInfo->streamInfo.pState); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1767,8 +1778,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh .maxTs = INT64_MIN, }; - ASSERT(as.calTrigger != STREAM_TRIGGER_MAX_DELAY); - pInfo->win = pTaskInfo->window; pInfo->inputOrder = (pPhyNode->window.inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; pInfo->resultTsOrder = (pPhyNode->window.outputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; @@ -1803,8 +1812,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh setOperatorInfo(pOperator, "TimeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, true, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = - createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo, optrDefaultBufFn, NULL); + pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo, + optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -1872,7 +1881,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator // pInfo->numOfRows data belong to the current session window updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, - pRowSup->numOfRows, pBlock->info.rows, numOfOutput); + pRowSup->numOfRows, pBlock->info.rows, numOfOutput); // here we start a new session window doKeepNewWindowStartInfo(pRowSup, tsList, j, gid); @@ -1889,8 +1898,8 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); - applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, - pBlock->info.rows, numOfOutput); + applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + pRowSup->numOfRows, pBlock->info.rows, numOfOutput); } static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { @@ -2004,7 +2013,8 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num); initResultSizeInfo(&pOperator->resultInfo, 4096); - code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str); + code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, + pTaskInfo->streamInfo.pState); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -2022,8 +2032,8 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi setOperatorInfo(pOperator, "StateWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = - createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo, optrDefaultBufFn, NULL); + pOperator->fpSet = createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo, + optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -2072,7 +2082,8 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW SSDataBlock* pResBlock = createDataBlockFromDescNode(pSessionNode->window.node.pOutputDataBlockDesc); initBasicInfo(&pInfo->binfo, pResBlock); - int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); + int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, + pTaskInfo->streamInfo.pState); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -2095,8 +2106,8 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW setOperatorInfo(pOperator, "SessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = - createOperatorFpSet(optrDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo, optrDefaultBufFn, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo, + optrDefaultBufFn, NULL); pOperator->pTaskInfo = pTaskInfo; code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -2253,7 +2264,6 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB return; } blockDataEnsureCapacity(pBlock, size - (*pIndex)); - ASSERT(3 <= taosArrayGetSize(pBlock->pDataBlock)); SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); @@ -2314,7 +2324,8 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo) { if (needDeleteWindowBuf(&nextWin, &pInfo->twAggSup) && !pInfo->ignoreExpiredData) { void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey)); if (!chIds) { - SPullWindowInfo pull = {.window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; + SPullWindowInfo pull = { + .window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; // add pull data request if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) { int32_t size1 = taosArrayGetSize(pInfo->pChildren); @@ -2347,6 +2358,17 @@ void doBuildResult(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* buildDataBlockFromGroupRes(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo); } +static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, + TSKEY* primaryKeys, int32_t prevPosition) { + int32_t startPos = prevPosition + 1; + if (startPos == pDataBlockInfo->rows) { + startPos = -1; + } else { + *pNext = getFinalTimeWindow(primaryKeys[startPos], pInterval); + } + return startPos; +} + static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t groupId, SHashObj* pUpdatedMap) { SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info; @@ -2360,7 +2382,6 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p SResultRow* pResult = NULL; int32_t forwardRows = 0; - ASSERT(pSDataBlock->pDataBlock != NULL); SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; @@ -2390,7 +2411,8 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p }; void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey)); if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup) && !chIds) { - SPullWindowInfo pull = {.window = nextWin, .groupId = groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; + SPullWindowInfo pull = { + .window = nextWin, .groupId = groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; // add pull data request if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) { int32_t size = taosArrayGetSize(pInfo->pChildren); @@ -2445,7 +2467,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, - pSDataBlock->info.rows, numOfOutput); + pSDataBlock->info.rows, numOfOutput); SWinKey key = { .ts = nextWin.skey, .groupId = groupId, @@ -2457,8 +2479,12 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p } int32_t prevEndPos = (forwardRows - 1) * step + startPos; ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0); - startPos = - getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC); + if (IS_FINAL_OP(pInfo)) { + startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos); + } else { + startPos = + getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC); + } if (startPos < 0) { break; } @@ -2468,12 +2494,8 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - SOperatorInfo* downstream = pOperator->pDownstream[0]; - TSKEY maxTs = INT64_MIN; - TSKEY minTs = INT64_MAX; - - SExprSupp* pSup = &pOperator->exprSupp; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + SExprSupp* pSup = &pOperator->exprSupp; qDebug("interval status %d %s", pOperator->status, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); @@ -2483,7 +2505,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes); if (pInfo->pPullDataRes->info.rows != 0) { // process the rest of the data - ASSERT(IS_FINAL_OP(pInfo)); printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); return pInfo->pPullDataRes; } @@ -2530,55 +2551,64 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } } - SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); + if (!pInfo->pUpdated) { + pInfo->pUpdated = taosArrayInit(4, POINTER_BYTES); + } + if (!pInfo->pUpdatedMap) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); + } + while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { pOperator->status = OP_RES_TO_RETURN; - qDebug("===stream===return data:%s. recv datablock num:%" PRIu64 , IS_FINAL_OP(pInfo) ? "interval final" : "interval semi", pInfo->numOfDatapack); + qDebug("===stream===return data:%s. recv datablock num:%" PRIu64, + IS_FINAL_OP(pInfo) ? "interval final" : "interval semi", pInfo->numOfDatapack); pInfo->numOfDatapack = 0; break; } pInfo->numOfDatapack++; printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval final recv" : "interval semi recv"); - ASSERT(pBlock->info.type != STREAM_INVERT); if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) { pInfo->binfo.pRes->info.type = pBlock->info.type; } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); - doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pUpdatedMap); + doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap); if (IS_FINAL_OP(pInfo)) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SStreamIntervalOperatorInfo* pChildInfo = pChildOp->info; SExprSupp* pChildSup = &pChildOp->exprSupp; doDeleteWindows(pChildOp, &pChildInfo->interval, pBlock, NULL, NULL); - rebuildIntervalWindow(pOperator, delWins, pUpdatedMap); + rebuildIntervalWindow(pOperator, delWins, pInfo->pUpdatedMap); addRetriveWindow(delWins, pInfo); taosArrayAddAll(pInfo->pDelWins, delWins); taosArrayDestroy(delWins); continue; } - removeResults(delWins, pUpdatedMap); + removeResults(delWins, pInfo->pUpdatedMap); taosArrayAddAll(pInfo->pDelWins, delWins); taosArrayDestroy(delWins); break; } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) { - getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap); + getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) { - doDeleteWindows(pOperator, &pInfo->interval, pBlock, NULL, pUpdatedMap); - if (taosArrayGetSize(pUpdated) > 0) { + doDeleteWindows(pOperator, &pInfo->interval, pBlock, NULL, pInfo->pUpdatedMap); + if (taosArrayGetSize(pInfo->pUpdated) > 0) { break; } continue; } else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_OP(pInfo)) { processPullOver(pBlock, pInfo->pPullDataMap, &pInfo->interval); continue; + } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + return pBlock; + } else { + ASSERTS(pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); } if (pInfo->scalarSupp.pExprInfo != NULL) { @@ -2586,7 +2616,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); } setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); - doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pUpdatedMap); + doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pInfo->pUpdatedMap); if (IS_FINAL_OP(pInfo)) { int32_t chIndex = getChildIndex(pBlock); int32_t size = taosArrayGetSize(pInfo->pChildren); @@ -2606,35 +2636,34 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { setInputDataBlock(&pChildOp->exprSupp, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); doStreamIntervalAggImpl(pChildOp, pBlock, pBlock->info.id.groupId, NULL); } - maxTs = TMAX(maxTs, pBlock->info.window.ekey); - maxTs = TMAX(maxTs, pBlock->info.watermark); - minTs = TMIN(minTs, pBlock->info.window.skey); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark); + pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey); } - removeDeleteResults(pUpdatedMap, pInfo->pDelWins); - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); - pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs); + removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins); if (IS_FINAL_OP(pInfo)) { closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, - pInfo->pPullDataMap, pUpdatedMap, pInfo->pDelWins, pOperator); + pInfo->pPullDataMap, pInfo->pUpdatedMap, pInfo->pDelWins, pOperator); closeChildIntervalWindow(pOperator, pInfo->pChildren, pInfo->twAggSup.maxTs); } pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs; void* pIte = NULL; - while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { - taosArrayPush(pUpdated, pIte); + while ((pIte = taosHashIterate(pInfo->pUpdatedMap, pIte)) != NULL) { + taosArrayPush(pInfo->pUpdated, pIte); } - taosHashCleanup(pUpdatedMap); - taosArraySort(pUpdated, resultrowComparAsc); + taosHashCleanup(pInfo->pUpdatedMap); + pInfo->pUpdatedMap = NULL; + taosArraySort(pInfo->pUpdated, resultrowComparAsc); - initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); + pInfo->pUpdated = NULL; blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes); if (pInfo->pPullDataRes->info.rows != 0) { // process the rest of the data - ASSERT(IS_FINAL_OP(pInfo)); printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); return pInfo->pPullDataRes; } @@ -2659,7 +2688,7 @@ int64_t getDeleteMark(SIntervalPhysiNode* pIntervalPhyNode) { if (pIntervalPhyNode->window.deleteMark <= 0) { return DEAULT_DELETE_MARK; } - int64_t deleteMark = TMAX(pIntervalPhyNode->window.deleteMark,pIntervalPhyNode->window.watermark); + int64_t deleteMark = TMAX(pIntervalPhyNode->window.deleteMark, pIntervalPhyNode->window.watermark); deleteMark = TMAX(deleteMark, pIntervalPhyNode->interval); return deleteMark; } @@ -2689,7 +2718,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, .deleteMarkSaved = 0, .calTriggerSaved = 0, }; - ASSERT(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY); + ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay"); pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(&pOperator->resultInfo, 4096); @@ -2707,14 +2736,14 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); initBasicInfo(&pInfo->binfo, pResBlock); - int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); + int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, + pTaskInfo->streamInfo.pState); if (code != TSDB_CODE_SUCCESS) { goto _error; } initStreamFunciton(pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs); - ASSERT(numOfCols > 0); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); @@ -2725,6 +2754,9 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->pChildren = NULL; if (numOfChild > 0) { pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*)); + if (!pInfo->pChildren) { + goto _error; + } for (int32_t i = 0; i < numOfChild; i++) { SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pPhyNode, pTaskInfo, 0); if (pChildOp) { @@ -2747,7 +2779,6 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, // semi interval operator does not catch result pInfo->isFinal = false; pOperator->name = "StreamSemiIntervalOperator"; - ASSERT(pInfo->aggSup.currentPageId == -1); } if (!IS_FINAL_OP(pInfo) || numOfChild == 0) { @@ -2765,14 +2796,16 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->delKey.ts = INT64_MAX; pInfo->delKey.groupId = 0; pInfo->numOfDatapack = 0; + pInfo->pUpdated = NULL; + pInfo->pUpdatedMap = NULL; pOperator->operatorType = pPhyNode->type; pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->fpSet = - createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL); + pOperator->fpSet = createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, + optrDefaultBufFn, NULL); if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) { initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup); } @@ -3163,15 +3196,6 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData } } -void deleteWindow(SArray* pWinInfos, int32_t index, FDelete fp) { - ASSERT(index >= 0 && index < taosArrayGetSize(pWinInfos)); - if (fp) { - void* ptr = taosArrayGet(pWinInfos, index); - fp(ptr); - } - taosArrayRemove(pWinInfos, index); -} - static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) { SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* startDatas = (TSKEY*)pStartTsCol->pData; @@ -3219,7 +3243,6 @@ static int32_t copyUpdateResult(SSHashObj* pStUpdated, SArray* pUpdated) { int32_t iter = 0; while ((pIte = tSimpleHashIterate(pStUpdated, pIte, &iter)) != NULL) { void* key = tSimpleHashGetKey(pIte, &keyLen); - ASSERT(keyLen == sizeof(SSessionKey)); taosArrayPush(pUpdated, key); } taosArraySort(pUpdated, sessionKeyCompareAsc); @@ -3280,7 +3303,6 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; int32_t numOfOutput = pSup->numOfExprs; int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren); - ASSERT(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { SSessionKey* pWinKey = taosArrayGet(pWinArray, i); @@ -3381,7 +3403,6 @@ static void copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted) { void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList) { pGroupResInfo->pRows = pArrayList; pGroupResInfo->index = 0; - ASSERT(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo)); } void doBuildSessionResult(SOperatorInfo* pOperator, SStreamState* pState, SGroupResInfo* pGroupResInfo, @@ -3406,7 +3427,6 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SExprSupp* pSup = &pOperator->exprSupp; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; - TSKEY maxTs = INT64_MIN; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -3426,10 +3446,14 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { return NULL; } - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SSHashObj* pStUpdated = tSimpleHashInit(64, hashFn); SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(16, sizeof(SSessionKey)); // SResKeyPos + if (!pInfo->pUpdated) { + pInfo->pUpdated = taosArrayInit(16, sizeof(SSessionKey)); + } + if (!pInfo->pStUpdated) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pStUpdated = tSimpleHashInit(64, hashFn); + } while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -3442,21 +3466,25 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); // gap must be 0 doDeleteTimeWindows(pAggSup, pBlock, pWins); - removeSessionResults(pStUpdated, pWins); + removeSessionResults(pInfo->pStUpdated, pWins); if (IS_FINAL_OP(pInfo)) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; // gap must be 0 doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, NULL); - rebuildSessionWindow(pOperator, pWins, pStUpdated); + rebuildSessionWindow(pOperator, pWins, pInfo->pStUpdated); } copyDeleteWindowInfo(pWins, pInfo->pStDeleted); taosArrayDestroy(pWins); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllSessionWindow(pAggSup->pResultRows, pStUpdated); + getAllSessionWindow(pAggSup->pResultRows, pInfo->pStUpdated); continue; + } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + return pBlock; + } else { + ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); } if (pInfo->scalarSupp.pExprInfo != NULL) { @@ -3465,7 +3493,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); - doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, IS_FINAL_OP(pInfo)); + doStreamSessionAggImpl(pOperator, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, IS_FINAL_OP(pInfo)); if (IS_FINAL_OP(pInfo)) { int32_t chIndex = getChildIndex(pBlock); int32_t size = taosArrayGetSize(pInfo->pChildren); @@ -3482,20 +3510,20 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { setInputDataBlock(&pChildOp->exprSupp, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); doStreamSessionAggImpl(pChildOp, pBlock, NULL, NULL, true); } - maxTs = TMAX(maxTs, pBlock->info.window.ekey); - maxTs = TMAX(maxTs, pBlock->info.watermark); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark); } - - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pStUpdated); + closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated); closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); - copyUpdateResult(pStUpdated, pUpdated); - removeSessionResults(pInfo->pStDeleted, pUpdated); - tSimpleHashCleanup(pStUpdated); - initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + copyUpdateResult(pInfo->pStUpdated, pInfo->pUpdated); + removeSessionResults(pInfo->pStDeleted, pInfo->pUpdated); + tSimpleHashCleanup(pInfo->pStUpdated); + pInfo->pStUpdated = NULL; + initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); + pInfo->pUpdated = NULL; blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); #if 0 @@ -3580,11 +3608,13 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->isFinal = false; pInfo->pPhyNode = pPhyNode; pInfo->ignoreExpiredData = pSessionNode->window.igExpired; + pInfo->pUpdated = NULL; + pInfo->pStUpdated = NULL; setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = - createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, + optrDefaultBufFn, NULL); if (downstream) { initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup); @@ -3640,10 +3670,14 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { } } - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SSHashObj* pStUpdated = tSimpleHashInit(64, hashFn); SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(16, sizeof(SSessionKey)); + if (!pInfo->pUpdated) { + pInfo->pUpdated = taosArrayInit(16, sizeof(SSessionKey)); + } + if (!pInfo->pStUpdated) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pStUpdated = tSimpleHashInit(64, hashFn); + } while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -3658,13 +3692,17 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { // gap must be 0 SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, pWins); - removeSessionResults(pStUpdated, pWins); + removeSessionResults(pInfo->pStUpdated, pWins); copyDeleteWindowInfo(pWins, pInfo->pStDeleted); taosArrayDestroy(pWins); break; } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllSessionWindow(pInfo->streamAggSup.pResultRows, pStUpdated); + getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pStUpdated); continue; + } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + return pBlock; + } else { + ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); } if (pInfo->scalarSupp.pExprInfo != NULL) { @@ -3673,18 +3711,20 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); - doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, NULL, false); + doStreamSessionAggImpl(pOperator, pBlock, pInfo->pStUpdated, NULL, false); maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs; - copyUpdateResult(pStUpdated, pUpdated); - removeSessionResults(pInfo->pStDeleted, pUpdated); - tSimpleHashCleanup(pStUpdated); + copyUpdateResult(pInfo->pStUpdated, pInfo->pUpdated); + removeSessionResults(pInfo->pStDeleted, pInfo->pUpdated); + tSimpleHashCleanup(pInfo->pStUpdated); + pInfo->pStUpdated = NULL; - initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); + pInfo->pUpdated = NULL; blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); #if 0 @@ -3944,7 +3984,6 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { SExprSupp* pSup = &pOperator->exprSupp; SStreamStateAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; - int64_t maxTs = INT64_MIN; if (pOperator->status == OP_RES_TO_RETURN) { doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { @@ -3962,10 +4001,14 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { return NULL; } - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SSHashObj* pSeUpdated = tSimpleHashInit(64, hashFn); SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(16, sizeof(SSessionKey)); + if (!pInfo->pUpdated) { + pInfo->pUpdated = taosArrayInit(16, sizeof(SSessionKey)); + } + if (!pInfo->pSeUpdated) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pSeUpdated = tSimpleHashInit(64, hashFn); + } while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -3977,13 +4020,17 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { pBlock->info.type == STREAM_CLEAR) { SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, pWins); - removeSessionResults(pSeUpdated, pWins); + removeSessionResults(pInfo->pSeUpdated, pWins); copyDeleteWindowInfo(pWins, pInfo->pSeDeleted); taosArrayDestroy(pWins); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllSessionWindow(pInfo->streamAggSup.pResultRows, pSeUpdated); + getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pSeUpdated); continue; + } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + return pBlock; + } else { + ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); } if (pInfo->scalarSupp.pExprInfo != NULL) { @@ -3992,19 +4039,20 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); - doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted); - maxTs = TMAX(maxTs, pBlock->info.window.ekey); + doStreamStateAggImpl(pOperator, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); } - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pSeUpdated); - copyUpdateResult(pSeUpdated, pUpdated); - removeSessionResults(pInfo->pSeDeleted, pUpdated); - tSimpleHashCleanup(pSeUpdated); + closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pInfo->pSeUpdated); + copyUpdateResult(pInfo->pSeUpdated, pInfo->pUpdated); + removeSessionResults(pInfo->pSeDeleted, pInfo->pUpdated); + tSimpleHashCleanup(pInfo->pSeUpdated); + pInfo->pSeUpdated = NULL; - initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); + pInfo->pUpdated = NULL; blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); #if 0 @@ -4085,11 +4133,13 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); pInfo->pChildren = NULL; pInfo->ignoreExpiredData = pStateNode->window.igExpired; + pInfo->pUpdated = NULL; + pInfo->pSeUpdated = NULL; setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = - createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, optrDefaultBufFn, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, + optrDefaultBufFn, NULL); initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -4112,6 +4162,9 @@ void destroyMAIOperatorInfo(void* param) { static SResultRow* doSetSingleOutputTupleBuf(SResultRowInfo* pResultRowInfo, SAggSupporter* pSup) { SResultRow* pResult = getNewResultRow(pSup->pResultBuf, &pSup->currentPageId, pSup->resultRowSize); + if (NULL == pResult) { + return pResult; + } pResultRowInfo->cur = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; return pResult; } @@ -4195,7 +4248,7 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos, - pBlock->info.rows, pSup->numOfExprs); + pBlock->info.rows, pSup->numOfExprs); } static void cleanupAfterGroupResultGen(SMergeAlignedIntervalAggOperatorInfo* pMiaInfo, SSDataBlock* pRes) { @@ -4346,7 +4399,8 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num); - code = initAggSup(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str); + code = initAggSup(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, + pTaskInfo->streamInfo.pState); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4365,8 +4419,8 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, setOperatorInfo(pOperator, "TimeMergeAlignedIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL, false, OP_NOT_OPENED, miaInfo, pTaskInfo); - pOperator->fpSet = - createOperatorFpSet(optrDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo, optrDefaultBufFn, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo, + optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -4501,7 +4555,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &win, true); applyAggFunctionOnPartialTuples(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows, - pBlock->info.rows, numOfOutput); + pBlock->info.rows, numOfOutput); doCloseWindow(pResultRowInfo, iaInfo, pResult); // output previous interval results after this interval (&win) is closed @@ -4533,7 +4587,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &nextWin, true); applyAggFunctionOnPartialTuples(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows, - pBlock->info.rows, numOfOutput); + pBlock->info.rows, numOfOutput); doCloseWindow(pResultRowInfo, iaInfo, pResult); // output previous interval results after this interval (&nextWin) is closed @@ -4650,7 +4704,8 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(&pOperator->resultInfo, 4096); - int32_t code = initAggSup(pExprSupp, &pIntervalInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str); + int32_t code = initAggSup(pExprSupp, &pIntervalInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, + pTaskInfo->streamInfo.pState); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4670,8 +4725,8 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge initResultRowInfo(&pIntervalInfo->binfo.resultRowInfo); setOperatorInfo(pOperator, "TimeMergeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, false, OP_NOT_OPENED, pMergeIntervalInfo, pTaskInfo); - pOperator->fpSet = - createOperatorFpSet(optrDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo, optrDefaultBufFn, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo, + optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -4693,8 +4748,6 @@ _error: static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - int64_t maxTs = INT64_MIN; - int64_t minTs = INT64_MAX; SExprSupp* pSup = &pOperator->exprSupp; if (pOperator->status == OP_EXEC_DONE) { @@ -4722,9 +4775,14 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); // SResKeyPos - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); + if (!pInfo->pUpdated) { + pInfo->pUpdated = taosArrayInit(4, POINTER_BYTES); + } + if (!pInfo->pUpdatedMap) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); + } + while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); @@ -4738,11 +4796,15 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { - doDeleteWindows(pOperator, &pInfo->interval, pBlock, pInfo->pDelWins, pUpdatedMap); + doDeleteWindows(pOperator, &pInfo->interval, pBlock, pInfo->pDelWins, pInfo->pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap); + getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); continue; + } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + return pBlock; + } else { + ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); } if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) { @@ -4763,27 +4825,27 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { setInverFunction(pSup->pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.type); } - maxTs = TMAX(maxTs, pBlock->info.window.ekey); - minTs = TMIN(minTs, pBlock->info.window.skey); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); + pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey); - doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pUpdatedMap); + doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pInfo->pUpdatedMap); } - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); - pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs); pOperator->status = OP_RES_TO_RETURN; - removeDeleteResults(pUpdatedMap, pInfo->pDelWins); - closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap, + removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins); + closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pInfo->pUpdatedMap, pInfo->pDelWins, pOperator); void* pIte = NULL; - while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { - taosArrayPush(pUpdated, pIte); + while ((pIte = taosHashIterate(pInfo->pUpdatedMap, pIte)) != NULL) { + taosArrayPush(pInfo->pUpdated, pIte); } - taosArraySort(pUpdated, resultrowComparAsc); + taosArraySort(pInfo->pUpdated, resultrowComparAsc); - initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); + pInfo->pUpdated = NULL; blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - taosHashCleanup(pUpdatedMap); + taosHashCleanup(pInfo->pUpdatedMap); + pInfo->pUpdatedMap = NULL; doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); if (pInfo->pDelRes->info.rows > 0) { @@ -4812,7 +4874,6 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys int32_t code = TSDB_CODE_SUCCESS; int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols); - ASSERT(numOfCols > 0); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); SInterval interval = { @@ -4832,7 +4893,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys .deleteMark = getDeleteMark(pIntervalPhyNode), }; - ASSERT(twAggSupp.calTrigger != STREAM_TRIGGER_MAX_DELAY); + ASSERTS(twAggSupp.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay"); pOperator->pTaskInfo = pTaskInfo; pInfo->interval = interval; @@ -4849,7 +4910,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys initResultSizeInfo(&pOperator->resultInfo, 4096); size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); + code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, + pTaskInfo->streamInfo.pState); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4884,6 +4946,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->delKey.ts = INT64_MAX; pInfo->delKey.groupId = 0; pInfo->numOfDatapack = 0; + pInfo->pUpdated = NULL; + pInfo->pUpdatedMap = NULL; setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -4904,4 +4968,3 @@ _error: pTaskInfo->code = code; return NULL; } - diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index 484d9170698a2ca84b21079cfc9964d261713805..fd6215e3a1a5b8b7b99ab3f2a94416fda9c8adff 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -49,7 +49,9 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { } SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn) { - ASSERT(fn != NULL); + if (fn == NULL) { + return NULL; + } if (capacity == 0) { capacity = 4; @@ -66,7 +68,6 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn) { pHashObj->equalFp = memcmp; pHashObj->hashFp = fn; - ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); pHashObj->hashList = (SHNode **)taosMemoryCalloc(pHashObj->capacity, sizeof(void *)); if (!pHashObj->hashList) { diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 30911887bbb42dd9e116f8c3cea2e587bc456144..661e9f97b74e33fd4f372a7eba51ccb42df385d2 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -108,7 +108,7 @@ static int32_t sortComparCleanup(SMsortComparParam* cmpParam) { return TSDB_CODE_SUCCESS; } -void tsortClearOrderdSource(SArray *pOrderedSource) { +void tsortClearOrderdSource(SArray* pOrderedSource) { for (size_t i = 0; i < taosArrayGetSize(pOrderedSource); i++) { SSortSource** pSource = taosArrayGet(pOrderedSource, i); if (NULL == *pSource) { @@ -121,6 +121,12 @@ void tsortClearOrderdSource(SArray *pOrderedSource) { if ((*pSource)->param && !(*pSource)->onlyRef) { taosMemoryFree((*pSource)->param); } + + if (!(*pSource)->onlyRef && (*pSource)->src.pBlock) { + blockDataDestroy((*pSource)->src.pBlock); + (*pSource)->src.pBlock = NULL; + } + taosMemoryFreeClear(*pSource); } @@ -245,7 +251,8 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32 if (pHandle->pBuf == NULL) { if (!osTempSpaceAvailable()) { code = TSDB_CODE_NO_AVAIL_DISK; - qError("Sort compare init failed since %s, %s", terrstr(code), pHandle->idStr); + terrno = code; + qError("Sort compare init failed since %s, %s", tstrerror(code), pHandle->idStr); return code; } @@ -253,6 +260,7 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32 "sortComparInit", tsTempDir); dBufSetPrintInfo(pHandle->pBuf); if (code != TSDB_CODE_SUCCESS) { + terrno = code; return code; } } @@ -270,8 +278,13 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32 int32_t* pPgId = taosArrayGet(pSource->pageIdList, pSource->pageIndex); void* pPage = getBufPage(pHandle->pBuf, *pPgId); + if (NULL == pPage) { + return terrno; + } + code = blockDataFromBuf(pSource->src.pBlock, pPage); if (code != TSDB_CODE_SUCCESS) { + terrno = code; return code; } @@ -337,6 +350,11 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT int32_t* pPgId = taosArrayGet(pSource->pageIdList, pSource->pageIndex); void* pPage = getBufPage(pHandle->pBuf, *pPgId); + if (pPage == NULL) { + qError("failed to get buffer, code:%s", tstrerror(terrno)); + return terrno; + } + int32_t code = blockDataFromBuf(pSource->src.pBlock, pPage); if (code != TSDB_CODE_SUCCESS) { return code; @@ -620,9 +638,9 @@ static int32_t createInitialSources(SSortHandle* pHandle) { if (pHandle->type == SORT_SINGLESOURCE_SORT) { SSortSource** pSource = taosArrayGet(pHandle->pOrderedSource, 0); - SSortSource* source = *pSource; + SSortSource* source = *pSource; *pSource = NULL; - + tsortClearOrderdSource(pHandle->pOrderedSource); while (1) { @@ -650,6 +668,10 @@ static int32_t createInitialSources(SSortHandle* pHandle) { if (source->param && !source->onlyRef) { taosMemoryFree(source->param); } + if (!source->onlyRef && source->src.pBlock) { + blockDataDestroy(source->src.pBlock); + source->src.pBlock = NULL; + } taosMemoryFree(source); return code; } @@ -663,6 +685,10 @@ static int32_t createInitialSources(SSortHandle* pHandle) { if (source->param && !source->onlyRef) { taosMemoryFree(source->param); } + if (!source->onlyRef && source->src.pBlock) { + blockDataDestroy(source->src.pBlock); + source->src.pBlock = NULL; + } taosMemoryFree(source); return code; } @@ -800,6 +826,7 @@ STupleHandle* tsortNextTuple(SSortHandle* pHandle) { } } + // all sources are completed. if (pHandle->cmpParam.numOfSources == pHandle->numOfCompletedSources) { return NULL; } @@ -839,8 +866,8 @@ SSortExecInfo tsortGetSortExecInfo(SSortHandle* pHandle) { SSortExecInfo info = {0}; if (pHandle == NULL) { - info.sortMethod = SORT_QSORT_T; // by default - info.sortBuffer = 2 * 1048576; // 2mb by default + info.sortMethod = SORT_QSORT_T; // by default + info.sortBuffer = 2 * 1048576; // 2mb by default } else { info.sortBuffer = pHandle->pageSize * pHandle->numOfPages; info.sortMethod = pHandle->inMemSort ? SORT_QSORT_T : SORT_SPILLED_MERGE_SORT_T; diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 2efd06f4401b94968e493aa79775544e01bf11b7..913aae6cc3ae489aa725ea255ce9ecce4482a13d 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -43,6 +43,19 @@ enum { data_desc = 0x3, }; +TEST(testCase, windowFunctionTest) { + int64_t tsCol[100000]; + int32_t rows = 100000; + for (int32_t i = 0; i < rows; i++) { + tsCol[i] = 1648791213000 + i; + } + int32_t ekeyNum = 50000; + int32_t pos = 40000; + int64_t ekey = tsCol[ekeyNum]; + int32_t num = getForwardStepsInBlock(rows, binarySearchForKey, ekey, pos, TSDB_ORDER_ASC, tsCol); + ASSERT_EQ(num, ekeyNum - pos + 1); +} + typedef struct SDummyInputInfo { int32_t totalPages; // numOfPages int32_t current; diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index a7ac53fab61593240a0859d16b44f4dd27a49ddd..dc884a058193e0abbefa04cff086788733f59a87 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -32,7 +32,7 @@ typedef struct SSumRes { int16_t type; int64_t prevTs; bool isPrevTsSet; - bool overflow; // if overflow is true, dsum to be used for any type; + bool overflow; // if overflow is true, dsum to be used for any type; } SSumRes; typedef struct SMinmaxResInfo { @@ -44,9 +44,10 @@ typedef struct SMinmaxResInfo { bool nullTupleSaved; int16_t type; } SMinmaxResInfo; -int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc); -STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, const STupleKey* pKey); +int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems); + +int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos); diff --git a/source/libs/function/inc/tpercentile.h b/source/libs/function/inc/tpercentile.h index 873dc46a0890bd6659e86feb294539dddc3a031a..80159460f5c58327930cdbeb08fcdd14263a1ebd 100644 --- a/source/libs/function/inc/tpercentile.h +++ b/source/libs/function/inc/tpercentile.h @@ -73,10 +73,10 @@ void tMemBucketDestroy(tMemBucket *pBucket); int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size); -double getPercentile(tMemBucket *pMemBucket, double percent); +int32_t getPercentile(tMemBucket *pMemBucket, double percent, double *result); #endif // TDENGINE_TPERCENTILE_H #ifdef __cplusplus } -#endif \ No newline at end of file +#endif diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 06406d158f95d4d0ebbefd7cb6345a59ca9131ad..1a0437c26a6657c17557e4772fd91403a647bf43 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1016,7 +1016,10 @@ static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* err intervals[0] = -INFINITY; intervals[numOfBins - 1] = INFINITY; // in case of desc bin orders, -inf/inf should be swapped - ASSERT(numOfBins >= 4); + if (numOfBins < 4) { + return false; + } + if (intervals[1] > intervals[numOfBins - 2]) { TSWAP(intervals[0], intervals[numOfBins - 1]); } @@ -1361,7 +1364,7 @@ static int32_t translateCsum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { } else if (IS_FLOAT_TYPE(colType)) { resType = TSDB_DATA_TYPE_DOUBLE; } else { - ASSERT(0); + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index d03a7fce75e95f3866c34a44f0da7ff63d5fd9fd..224537c1f110f9d3d629320d031baf1dc6e7d108 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -41,15 +41,15 @@ #define HLL_BUCKET_MASK (HLL_BUCKETS - 1) #define HLL_ALPHA_INF 0.721347520444481703680 // constant for 0.5/ln(2) -//typedef struct SMinmaxResInfo { -// bool assign; // assign the first value or not -// int64_t v; -// STuplePos tuplePos; +// typedef struct SMinmaxResInfo { +// bool assign; // assign the first value or not +// int64_t v; +// STuplePos tuplePos; // -// STuplePos nullTuplePos; -// bool nullTupleSaved; -// int16_t type; -//} SMinmaxResInfo; +// STuplePos nullTuplePos; +// bool nullTupleSaved; +// int16_t type; +// } SMinmaxResInfo; typedef struct STopBotResItem { SVariant v; @@ -506,7 +506,6 @@ static int32_t getNumOfElems(SqlFunctionCtx* pCtx) { SColumnInfoData* pInputCol = pInput->pData[0]; if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows && !IS_VAR_DATA_TYPE(pInputCol->info.type)) { numOfElem = pInput->numOfRows - pInput->pColumnDataAgg[0]->numOfNull; - ASSERT(numOfElem >= 0); } else { if (pInputCol->hasNull) { for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) { @@ -540,7 +539,7 @@ int32_t countFunction(SqlFunctionCtx* pCtx) { if (IS_NULL_TYPE(type)) { // select count(NULL) returns 0 numOfElem = 1; - *((int64_t*)buf) = 0; + *((int64_t*)buf) += 0; } else { numOfElem = getNumOfElems(pCtx); *((int64_t*)buf) += numOfElem; @@ -596,7 +595,6 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) { if (pInput->colDataSMAIsSet) { numOfElem = pInput->numOfRows - pAgg->numOfNull; - ASSERT(numOfElem >= 0); if (IS_SIGNED_NUMERIC_TYPE(type)) { pSumRes->isum += pAgg->sum; @@ -661,7 +659,6 @@ int32_t sumInvertFunction(SqlFunctionCtx* pCtx) { if (pInput->colDataSMAIsSet) { numOfElem = pInput->numOfRows - pAgg->numOfNull; - ASSERT(numOfElem >= 0); if (IS_SIGNED_NUMERIC_TYPE(type)) { pSumRes->isum -= pAgg->sum; @@ -757,24 +754,33 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { } int32_t minFunction(SqlFunctionCtx* pCtx) { - int32_t numOfElems = doMinMaxHelper(pCtx, 1); + int32_t numOfElems = 0; + int32_t code = doMinMaxHelper(pCtx, 1, &numOfElems); + if (code != TSDB_CODE_SUCCESS) { + return code; + } SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); return TSDB_CODE_SUCCESS; } int32_t maxFunction(SqlFunctionCtx* pCtx) { - int32_t numOfElems = doMinMaxHelper(pCtx, 0); + int32_t numOfElems = 0; + int32_t code = doMinMaxHelper(pCtx, 0, &numOfElems); + if (code != TSDB_CODE_SUCCESS) { + return code; + } SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); return TSDB_CODE_SUCCESS; } -static void setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t rowIndex); -static void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, +static int32_t setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t rowIndex); +static int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, int32_t rowIndex); int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { - SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); + int32_t code = TSDB_CODE_SUCCESS; + SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); SMinmaxResInfo* pRes = GET_ROWCELL_INTERBUF(pEntryInfo); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; @@ -784,24 +790,24 @@ int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { pEntryInfo->isNullRes = (pEntryInfo->numOfRes == 0) ? 1 : 0; if (pCol->info.type == TSDB_DATA_TYPE_FLOAT) { - float v = *(float*)&pRes->v; + float v = GET_FLOAT_VAL(&pRes->v); colDataAppend(pCol, currentRow, (const char*)&v, pEntryInfo->isNullRes); } else { colDataAppend(pCol, currentRow, (const char*)&pRes->v, pEntryInfo->isNullRes); } if (pEntryInfo->numOfRes > 0) { - setSelectivityValue(pCtx, pBlock, &pRes->tuplePos, currentRow); + code = setSelectivityValue(pCtx, pBlock, &pRes->tuplePos, currentRow); } else { - setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, currentRow); + code = setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, currentRow); } - return pEntryInfo->numOfRes; + return code; } -void setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t rowIndex) { +int32_t setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t rowIndex) { if (pCtx->subsidiaries.num <= 0) { - return; + return TSDB_CODE_SUCCESS; } for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) { @@ -811,44 +817,47 @@ void setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); colDataAppendNULL(pDstCol, rowIndex); } + + return TSDB_CODE_SUCCESS; } -void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, int32_t rowIndex) { +int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, int32_t rowIndex) { if (pCtx->subsidiaries.num <= 0) { - return; + return TSDB_CODE_SUCCESS; } - if (pCtx->saveHandle.pBuf != NULL) { - if (pTuplePos->pageId != -1) { - int32_t numOfCols = pCtx->subsidiaries.num; - const char* p = loadTupleData(pCtx, pTuplePos); + if ((pCtx->saveHandle.pBuf != NULL && pTuplePos->pageId != -1) || + (pCtx->saveHandle.pState && pTuplePos->streamTupleKey.ts > 0)) { + int32_t numOfCols = pCtx->subsidiaries.num; + const char* p = loadTupleData(pCtx, pTuplePos); + if (p == NULL) { + terrno = TSDB_CODE_NO_AVAIL_DISK; + return terrno; + } - bool* nullList = (bool*)p; - char* pStart = (char*)(nullList + numOfCols * sizeof(bool)); + bool* nullList = (bool*)p; + char* pStart = (char*)(nullList + numOfCols * sizeof(bool)); - // todo set the offset value to optimize the performance. - for (int32_t j = 0; j < numOfCols; ++j) { - SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j]; - int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; + // todo set the offset value to optimize the performance. + for (int32_t j = 0; j < numOfCols; ++j) { + SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j]; + int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; - SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); - ASSERT(pc->pExpr->base.resSchema.bytes == pDstCol->info.bytes); - if (nullList[j]) { - colDataAppendNULL(pDstCol, rowIndex); - } else { - colDataAppend(pDstCol, rowIndex, pStart, false); - } - pStart += pDstCol->info.bytes; + SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); + if (nullList[j]) { + colDataAppendNULL(pDstCol, rowIndex); + } else { + colDataAppend(pDstCol, rowIndex, pStart, false); } + pStart += pDstCol->info.bytes; } - } -} -void releaseSource(STuplePos* pPos) { - if (pPos->pageId == -1) { - return; + if (pCtx->saveHandle.pState) { + tdbFree((void*)p); + } } - // Todo(liuyao) relase row + + return TSDB_CODE_SUCCESS; } // This function append the selectivity to subsidiaries function context directly, without fetching data @@ -873,7 +882,6 @@ void appendSelectivityValue(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t pos) int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; SColumnInfoData* pDstCol = taosArrayGet(pCtx->pDstBlock->pDataBlock, dstSlotId); - ASSERT(pc->pExpr->base.resSchema.bytes == pDstCol->info.bytes); if (colDataIsNull_s(pSrcCol, rowIndex) == true) { colDataAppendNULL(pDstCol, pos); @@ -884,7 +892,6 @@ void appendSelectivityValue(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t pos) } void replaceTupleData(STuplePos* pDestPos, STuplePos* pSourcePos) { - releaseSource(pDestPos); *pDestPos = *pSourcePos; } @@ -1142,7 +1149,9 @@ static void stddevTransferInfo(SStddevRes* pInput, SStddevRes* pOutput) { int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; - ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + if (pCol->info.type != TSDB_DATA_TYPE_BINARY) { + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; + } SStddevRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); @@ -1567,7 +1576,7 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { // all data are null, set it completed if (pInfo->numOfElems == 0) { pResInfo->complete = true; - return 0; + return TSDB_CODE_SUCCESS; } else { pInfo->pMemBucket = tMemBucketCreate(pCol->info.bytes, type, pInfo->minval, pInfo->maxval); } @@ -1630,7 +1639,11 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pCol, i); numOfElems += 1; - tMemBucketPut(pInfo->pMemBucket, data, 1); + int32_t code = tMemBucketPut(pInfo->pMemBucket, data, 1); + if (code != TSDB_CODE_SUCCESS) { + tMemBucketDestroy(pInfo->pMemBucket); + return code; + } } SET_VAL(pResInfo, numOfElems, 1); @@ -1641,7 +1654,9 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SVariant* pVal = &pCtx->param[1].param; + int32_t code = 0; double v = 0; + GET_TYPED_DATA(v, double, pVal->nType, &pVal->i); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); @@ -1649,10 +1664,14 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { tMemBucket* pMemBucket = ppInfo->pMemBucket; if (pMemBucket != NULL && pMemBucket->total > 0) { // check for null - SET_DOUBLE_VAL(&ppInfo->result, getPercentile(pMemBucket, v)); + code = getPercentile(pMemBucket, v, &ppInfo->result); } tMemBucketDestroy(pMemBucket); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + return functionFinalize(pCtx, pBlock); } @@ -1768,7 +1787,10 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { double v = 0; GET_TYPED_DATA(v, double, type, data); - tHistogramAdd(&pInfo->pHisto, v); + int32_t code = tHistogramAdd(&pInfo->pHisto, v); + if (code != 0) { + return TSDB_CODE_FAILED; + } } qDebug("%s after add %d elements into histogram, total:%" PRId64 ", numOfEntry:%d, pHisto:%p, elems: %p", @@ -1838,7 +1860,9 @@ int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; - ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + if (pCol->info.type != TSDB_DATA_TYPE_BINARY) { + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; + } SAPercentileInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -2006,25 +2030,26 @@ static void prepareBuf(SqlFunctionCtx* pCtx) { pCtx->subsidiaries.rowLen = rowLen + pCtx->subsidiaries.num * sizeof(bool); pCtx->subsidiaries.buf = taosMemoryMalloc(pCtx->subsidiaries.rowLen); } - - ASSERT(pCtx->subsidiaries.buf != NULL); - ASSERT(pCtx->subsidiaries.rowLen > 0); } -static void firstlastSaveTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SqlFunctionCtx* pCtx, +static int32_t firstlastSaveTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SqlFunctionCtx* pCtx, SFirstLastRes* pInfo) { + int32_t code = TSDB_CODE_SUCCESS; + if (pCtx->subsidiaries.num <= 0) { - return; + return TSDB_CODE_SUCCESS; } if (!pInfo->hasResult) { - pInfo->pos = saveTupleData(pCtx, rowIndex, pSrcBlock, NULL); + code = saveTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos); } else { - updateTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos); + code = updateTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos); } + + return code; } -static void doSaveCurrentVal(SqlFunctionCtx* pCtx, int32_t rowIndex, int64_t currentTs, int32_t type, char* pData) { +static int32_t doSaveCurrentVal(SqlFunctionCtx* pCtx, int32_t rowIndex, int64_t currentTs, int32_t type, char* pData) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -2034,9 +2059,13 @@ static void doSaveCurrentVal(SqlFunctionCtx* pCtx, int32_t rowIndex, int64_t cur memcpy(pInfo->buf, pData, pInfo->bytes); pInfo->ts = currentTs; - firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pInfo->hasResult = true; + return TSDB_CODE_SUCCESS; } // This ordinary first function does not care if current scan is ascending order or descending order scan @@ -2057,10 +2086,12 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { } // All null data column, return directly. - if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { - ASSERT(pInputCol->hasNull == true); + if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows) && pInputCol->hasNull == true) { // save selectivity value for column consisted of all null values - firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } return TSDB_CODE_SUCCESS; } @@ -2071,8 +2102,8 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; - // please ref. to the comment in lastRowFunction for the reason why disabling the opt version of last/first function. - // we will use this opt implementation in an new version that is only available in scan subplan + // please ref. to the comment in lastRowFunction for the reason why disabling the opt version of last/first + // function. we will use this opt implementation in an new version that is only available in scan subplan #if 0 if (blockDataOrder == TSDB_ORDER_ASC) { // filter according to current result firstly @@ -2133,7 +2164,10 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = pts[i]; if (pResInfo->numOfRes == 0 || pInfo->ts > cts) { - doSaveCurrentVal(pCtx, i, cts, pInputCol->info.type, data); + int32_t code = doSaveCurrentVal(pCtx, i, cts, pInputCol->info.type, data); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pResInfo->numOfRes = 1; } } @@ -2141,7 +2175,10 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { if (numOfElems == 0) { // save selectivity value for column consisted of all null values - firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; @@ -2165,10 +2202,12 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } // All null data column, return directly. - if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { - ASSERT(pInputCol->hasNull == true); + if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows) && pInputCol->hasNull == true) { // save selectivity value for column consisted of all null values - firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } return TSDB_CODE_SUCCESS; } @@ -2179,7 +2218,8 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; - // please ref. to the comment in lastRowFunction for the reason why disabling the opt version of last/first function. + // please ref. to the comment in lastRowFunction for the reason why disabling the opt version of last/first + // function. #if 0 if (blockDataOrder == TSDB_ORDER_ASC) { for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) { @@ -2236,9 +2276,9 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { int32_t round = pInput->numOfRows >> 2; int32_t reminder = pInput->numOfRows & 0x03; - for (int32_t i = pInput->startRowIndex, tick = 0; tick < round; i += 4, tick += 1) { - int64_t cts = pts[i]; - int32_t chosen = i; + for (int32_t i = pInput->startRowIndex, tick = 0; tick < round; i += 4, tick += 1) { + int64_t cts = pts[i]; + int32_t chosen = i; if (cts < pts[i + 1]) { cts = pts[i + 1]; @@ -2257,29 +2297,38 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { char* data = colDataGetData(pInputCol, chosen); - doSaveCurrentVal(pCtx, i, cts, type, data); + int32_t code = doSaveCurrentVal(pCtx, i, cts, type, data); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pResInfo->numOfRes = 1; } } - for (int32_t i = pInput->startRowIndex + round * 4; i < pInput->startRowIndex + pInput->numOfRows; ++i) { - if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { - char* data = colDataGetData(pInputCol, i); - doSaveCurrentVal(pCtx, i, pts[i], type, data); - pResInfo->numOfRes = 1; + for (int32_t i = pInput->startRowIndex + round * 4; i < pInput->startRowIndex + pInput->numOfRows; ++i) { + if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { + char* data = colDataGetData(pInputCol, i); + int32_t code = doSaveCurrentVal(pCtx, i, pts[i], type, data); + if (code != TSDB_CODE_SUCCESS) { + return code; } + pResInfo->numOfRes = 1; + } + } + } else { + for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) { + if (colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { + continue; } - } else { - for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) { - if (colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { - continue; - } numOfElems++; if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { char* data = colDataGetData(pInputCol, i); - doSaveCurrentVal(pCtx, i, pts[i], type, data); + int32_t code = doSaveCurrentVal(pCtx, i, pts[i], type, data); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pResInfo->numOfRes = 1; } } @@ -2290,7 +2339,10 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { // save selectivity value for column consisted of all null values if (numOfElems == 0) { - firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } // SET_VAL(pResInfo, numOfElems, 1); @@ -2318,18 +2370,25 @@ static int32_t firstLastTransferInfoImpl(SFirstLastRes* pInput, SFirstLastRes* p return TSDB_CODE_SUCCESS; } -static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst, +static int32_t firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst, int32_t rowIndex) { if (TSDB_CODE_SUCCESS == firstLastTransferInfoImpl(pInput, pOutput, isFirst)) { - firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pOutput); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pOutput); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pOutput->hasResult = true; } + + return TSDB_CODE_SUCCESS; } static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuery) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; - ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + if (pCol->info.type != TSDB_DATA_TYPE_BINARY) { + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; + } SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); @@ -2339,7 +2398,10 @@ static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuer for (int32_t i = start; i < start + pInput->numOfRows; ++i) { char* data = colDataGetData(pCol, i); SFirstLastRes* pInputInfo = (SFirstLastRes*)varDataVal(data); - firstLastTransferInfo(pCtx, pInputInfo, pInfo, isFirstQuery, i); + int32_t code = firstLastTransferInfo(pCtx, pInputInfo, pInfo, isFirstQuery, i); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (!numOfElems) { numOfElems = pInputInfo->hasResult ? 1 : 0; } @@ -2354,6 +2416,7 @@ int32_t firstFunctionMerge(SqlFunctionCtx* pCtx) { return firstLastFunctionMerge int32_t lastFunctionMerge(SqlFunctionCtx* pCtx) { return firstLastFunctionMergeImpl(pCtx, false); } int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); @@ -2364,12 +2427,14 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { colDataAppend(pCol, pBlock->info.rows, pRes->buf, pRes->isNull || pResInfo->isNullRes); // handle selectivity - setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); + code = setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); - return pResInfo->numOfRes; + return code; } int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; + SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pEntryInfo); @@ -2385,10 +2450,10 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); colDataAppend(pCol, pBlock->info.rows, res, false); - setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); + code = setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); taosMemoryFree(res); - return 1; + return code; } int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { @@ -2408,7 +2473,7 @@ int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { return TSDB_CODE_SUCCESS; } -static void doSaveLastrow(SqlFunctionCtx* pCtx, char* pData, int32_t rowIndex, int64_t cts, SFirstLastRes* pInfo) { +static int32_t doSaveLastrow(SqlFunctionCtx* pCtx, char* pData, int32_t rowIndex, int64_t cts, SFirstLastRes* pInfo) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; @@ -2425,9 +2490,14 @@ static void doSaveLastrow(SqlFunctionCtx* pCtx, char* pData, int32_t rowIndex, i } pInfo->ts = cts; - firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pInfo->hasResult = true; + + return TSDB_CODE_SUCCESS; } int32_t lastRowFunction(SqlFunctionCtx* pCtx) { @@ -2439,7 +2509,7 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; - int32_t type = pInputCol->info.type; + int32_t type = pInputCol->info.type; int32_t bytes = pInputCol->info.bytes; pInfo->bytes = bytes; @@ -2489,7 +2559,10 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { numOfElems++; if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - doSaveLastrow(pCtx, data, i, cts, pInfo); + int32_t code = doSaveLastrow(pCtx, data, i, cts, pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pResInfo->numOfRes = 1; } } @@ -2524,7 +2597,7 @@ bool diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { return true; } -static void doSetPrevVal(SDiffInfo* pDiffInfo, int32_t type, const char* pv, int64_t ts) { +static int32_t doSetPrevVal(SDiffInfo* pDiffInfo, int32_t type, const char* pv, int64_t ts) { switch (type) { case TSDB_DATA_TYPE_BOOL: pDiffInfo->prev.i64 = *(bool*)pv ? 1 : 0; @@ -2549,12 +2622,14 @@ static void doSetPrevVal(SDiffInfo* pDiffInfo, int32_t type, const char* pv, int pDiffInfo->prev.d64 = *(double*)pv; break; default: - ASSERT(0); + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; } pDiffInfo->prevTs = ts; + + return TSDB_CODE_SUCCESS; } -static void doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, SColumnInfoData* pOutput, int32_t pos, +static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, SColumnInfoData* pOutput, int32_t pos, int32_t order, int64_t ts) { int32_t factor = (order == TSDB_ORDER_ASC) ? 1 : -1; pDiffInfo->prevTs = ts; @@ -2563,7 +2638,7 @@ static void doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, SCo int32_t v = *(int32_t*)pv; int64_t delta = factor * (v - pDiffInfo->prev.i64); // direct previous may be null if (delta < 0 && pDiffInfo->ignoreNegative) { - colDataSetNull_f(pOutput->nullbitmap, pos); + colDataSetNull_f_s(pOutput, pos); } else { colDataAppendInt64(pOutput, pos, &delta); } @@ -2576,7 +2651,7 @@ static void doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, SCo int8_t v = *(int8_t*)pv; int64_t delta = factor * (v - pDiffInfo->prev.i64); // direct previous may be null if (delta < 0 && pDiffInfo->ignoreNegative) { - colDataSetNull_f(pOutput->nullbitmap, pos); + colDataSetNull_f_s(pOutput, pos); } else { colDataAppendInt64(pOutput, pos, &delta); } @@ -2587,7 +2662,7 @@ static void doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, SCo int16_t v = *(int16_t*)pv; int64_t delta = factor * (v - pDiffInfo->prev.i64); // direct previous may be null if (delta < 0 && pDiffInfo->ignoreNegative) { - colDataSetNull_f(pOutput->nullbitmap, pos); + colDataSetNull_f_s(pOutput, pos); } else { colDataAppendInt64(pOutput, pos, &delta); } @@ -2599,7 +2674,7 @@ static void doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, SCo int64_t v = *(int64_t*)pv; int64_t delta = factor * (v - pDiffInfo->prev.i64); // direct previous may be null if (delta < 0 && pDiffInfo->ignoreNegative) { - colDataSetNull_f(pOutput->nullbitmap, pos); + colDataSetNull_f_s(pOutput, pos); } else { colDataAppendInt64(pOutput, pos, &delta); } @@ -2610,7 +2685,7 @@ static void doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, SCo float v = *(float*)pv; double delta = factor * (v - pDiffInfo->prev.d64); // direct previous may be null if ((delta < 0 && pDiffInfo->ignoreNegative) || isinf(delta) || isnan(delta)) { // check for overflow - colDataSetNull_f(pOutput->nullbitmap, pos); + colDataSetNull_f_s(pOutput, pos); } else { colDataAppendDouble(pOutput, pos, &delta); } @@ -2621,7 +2696,7 @@ static void doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, SCo double v = *(double*)pv; double delta = factor * (v - pDiffInfo->prev.d64); // direct previous may be null if ((delta < 0 && pDiffInfo->ignoreNegative) || isinf(delta) || isnan(delta)) { // check for overflow - colDataSetNull_f(pOutput->nullbitmap, pos); + colDataSetNull_f_s(pOutput, pos); } else { colDataAppendDouble(pOutput, pos, &delta); } @@ -2629,8 +2704,10 @@ static void doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, SCo break; } default: - ASSERT(0); + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; } + + return TSDB_CODE_SUCCESS; } int32_t diffFunction(SqlFunctionCtx* pCtx) { @@ -2654,7 +2731,7 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { if (pDiffInfo->includeNull) { - colDataSetNull_f(pOutput->nullbitmap, pos); + colDataSetNull_f_s(pOutput, pos); numOfElems += 1; } @@ -2667,7 +2744,10 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { if (tsList[i] == pDiffInfo->prevTs) { return TSDB_CODE_FUNC_DUP_TIMESTAMP; } - doHandleDiff(pDiffInfo, pInputCol->info.type, pv, pOutput, pos, pCtx->order, tsList[i]); + int32_t code = doHandleDiff(pDiffInfo, pInputCol->info.type, pv, pOutput, pos, pCtx->order, tsList[i]); + if (code != TSDB_CODE_SUCCESS) { + return code; + } // handle selectivity if (pCtx->subsidiaries.num > 0) { appendSelectivityValue(pCtx, i, pos); @@ -2675,7 +2755,10 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { numOfElems++; } else { - doSetPrevVal(pDiffInfo, pInputCol->info.type, pv, tsList[i]); + int32_t code = doSetPrevVal(pDiffInfo, pInputCol->info.type, pv, tsList[i]); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } pDiffInfo->hasPrev = true; @@ -2686,8 +2769,7 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { if (pDiffInfo->includeNull) { - colDataSetNull_f(pOutput->nullbitmap, pos); - + colDataSetNull_f_s(pOutput, pos); numOfElems += 1; } continue; @@ -2700,7 +2782,10 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { if (tsList[i] == pDiffInfo->prevTs) { return TSDB_CODE_FUNC_DUP_TIMESTAMP; } - doHandleDiff(pDiffInfo, pInputCol->info.type, pv, pOutput, pos, pCtx->order, tsList[i]); + int32_t code = doHandleDiff(pDiffInfo, pInputCol->info.type, pv, pOutput, pos, pCtx->order, tsList[i]); + if (code != TSDB_CODE_SUCCESS) { + return code; + } // handle selectivity if (pCtx->subsidiaries.num > 0) { appendSelectivityValue(pCtx, i, pos); @@ -2708,7 +2793,10 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { numOfElems++; } else { - doSetPrevVal(pDiffInfo, pInputCol->info.type, pv, tsList[i]); + int32_t code = doSetPrevVal(pDiffInfo, pInputCol->info.type, pv, tsList[i]); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } pDiffInfo->hasPrev = true; @@ -2750,8 +2838,8 @@ static STopBotRes* getTopBotOutputInfo(SqlFunctionCtx* pCtx) { return pRes; } -static void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, - uint64_t uid, SResultRowEntryInfo* pEntryInfo, bool isTopQuery); +static int32_t doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, + uint64_t uid, SResultRowEntryInfo* pEntryInfo, bool isTopQuery); static void addResult(SqlFunctionCtx* pCtx, STopBotResItem* pSourceItem, int16_t type, bool isTopQuery); @@ -2773,11 +2861,17 @@ int32_t topFunction(SqlFunctionCtx* pCtx) { numOfElems++; char* data = colDataGetData(pCol, i); - doAddIntoResult(pCtx, data, i, pCtx->pSrcBlock, pRes->type, pInput->uid, pResInfo, true); + int32_t code = doAddIntoResult(pCtx, data, i, pCtx->pSrcBlock, pRes->type, pInput->uid, pResInfo, true); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pRes->nullTupleSaved) { - pRes->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL); + int32_t code = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, &pRes->nullTuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pRes->nullTupleSaved = true; } return TSDB_CODE_SUCCESS; @@ -2801,11 +2895,17 @@ int32_t bottomFunction(SqlFunctionCtx* pCtx) { numOfElems++; char* data = colDataGetData(pCol, i); - doAddIntoResult(pCtx, data, i, pCtx->pSrcBlock, pRes->type, pInput->uid, pResInfo, false); + int32_t code = doAddIntoResult(pCtx, data, i, pCtx->pSrcBlock, pRes->type, pInput->uid, pResInfo, false); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pRes->nullTupleSaved) { - pRes->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL); + int32_t code = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, &pRes->nullTuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pRes->nullTupleSaved = true; } @@ -2845,7 +2945,7 @@ static int32_t topBotResComparFn(const void* p1, const void* p2, const void* par return (val1->v.d > val2->v.d) ? 1 : -1; } -void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, +int32_t doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, uint64_t uid, SResultRowEntryInfo* pEntryInfo, bool isTopQuery) { STopBotRes* pRes = getTopBotOutputInfo(pCtx); @@ -2853,7 +2953,6 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); STopBotResItem* pItems = pRes->pItems; - assert(pItems != NULL); // not full yet if (pEntryInfo->numOfRes < pRes->maxSize) { @@ -2863,7 +2962,10 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData // save the data of this tuple if (pCtx->subsidiaries.num > 0) { - pItem->tuplePos = saveTupleData(pCtx, rowIndex, pSrcBlock, NULL); + int32_t code = saveTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } #ifdef BUF_PAGE_DEBUG qDebug("page_saveTuple i:%d, item:%p,pageId:%d, offset:%d\n", pEntryInfo->numOfRes, pItem, pItem->tuplePos.pageId, @@ -2889,7 +2991,10 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData // save the data of this tuple by over writing the old data if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); + int32_t code = updateTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } #ifdef BUF_PAGE_DEBUG qDebug("page_copyTuple pageId:%d, offset:%d", pItem->tuplePos.pageId, pItem->tuplePos.offset); @@ -2898,6 +3003,8 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData topBotResComparFn, NULL, !isTopQuery); } } + + return TSDB_CODE_SUCCESS; } /* @@ -2937,21 +3044,33 @@ void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsid return buf; } -static STuplePos doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, - const STupleKey* pKey) { +static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STupleKey key, + STuplePos* pPos) { STuplePos p = {0}; if (pHandle->pBuf != NULL) { SFilePage* pPage = NULL; if (pHandle->currentPage == -1) { pPage = getNewBufPage(pHandle->pBuf, &pHandle->currentPage); + if (pPage == NULL) { + terrno = TSDB_CODE_NO_AVAIL_DISK; + return terrno; + } pPage->num = sizeof(SFilePage); } else { pPage = getBufPage(pHandle->pBuf, pHandle->currentPage); + if (pPage == NULL) { + terrno = TSDB_CODE_NO_AVAIL_DISK; + return terrno; + } if (pPage->num + length > getBufPageSize(pHandle->pBuf)) { // current page is all used, let's prepare a new buffer page releaseBufPage(pHandle->pBuf, pPage); pPage = getNewBufPage(pHandle->pBuf, &pHandle->currentPage); + if (pPage == NULL) { + terrno = TSDB_CODE_NO_AVAIL_DISK; + return terrno; + } pPage->num = sizeof(SFilePage); } } @@ -2964,25 +3083,41 @@ static STuplePos doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf releaseBufPage(pHandle->pBuf, pPage); } else { // other tuple save policy - if (streamStateFuncPut(pHandle->pState, pKey, pBuf, length) < 0) { - ASSERT(0); + if (streamStateFuncPut(pHandle->pState, &key, pBuf, length) >= 0) { + p.streamTupleKey = key; } - p.streamTupleKey = *pKey; } - return p; + *pPos = p; + return TSDB_CODE_SUCCESS; } -STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, const STupleKey* pKey) { +int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { prepareBuf(pCtx); + STupleKey key; + if (pCtx->saveHandle.pBuf == NULL) { + SColumnInfoData* pColInfo = taosArrayGet(pSrcBlock->pDataBlock, 0); + if (pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) { + int64_t skey = *(int64_t*)colDataGetData(pColInfo, rowIndex); + + key.groupId = pSrcBlock->info.id.groupId; + key.ts = skey; + key.exprIdx = pCtx->exprIdx; + } + } + char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf); - return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pKey); + return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, key, pPos); } static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STuplePos* pPos) { if (pHandle->pBuf != NULL) { SFilePage* pPage = getBufPage(pHandle->pBuf, pPos->pageId); + if (pPage == NULL) { + terrno = TSDB_CODE_NO_AVAIL_DISK; + return terrno; + } memcpy(pPage->data + pPos->offset, pBuf, length); setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); @@ -2997,13 +3132,15 @@ int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBloc prepareBuf(pCtx); char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf); - doUpdateTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pPos); - return TSDB_CODE_SUCCESS; + return doUpdateTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pPos); } static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPos) { if (pHandle->pBuf != NULL) { SFilePage* pPage = getBufPage(pHandle->pBuf, pPos->pageId); + if (pPage == NULL) { + return NULL; + } char* p = pPage->data + pPos->offset; releaseBufPage(pHandle->pBuf, pPage); return p; @@ -3020,6 +3157,8 @@ const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos) { } int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; + SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); STopBotRes* pRes = getTopBotOutputInfo(pCtx); @@ -3032,8 +3171,8 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t currentRow = pBlock->info.rows; if (pEntryInfo->numOfRes <= 0) { colDataAppendNULL(pCol, currentRow); - setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, currentRow); - return pEntryInfo->numOfRes; + code = setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, currentRow); + return code; } for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) { STopBotResItem* pItem = &pRes->pItems[i]; @@ -3042,18 +3181,17 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { qDebug("page_finalize i:%d,item:%p,pageId:%d, offset:%d\n", i, pItem, pItem->tuplePos.pageId, pItem->tuplePos.offset); #endif - setSelectivityValue(pCtx, pBlock, &pRes->pItems[i].tuplePos, currentRow); + code = setSelectivityValue(pCtx, pBlock, &pRes->pItems[i].tuplePos, currentRow); currentRow += 1; } - return pEntryInfo->numOfRes; + return code; } void addResult(SqlFunctionCtx* pCtx, STopBotResItem* pSourceItem, int16_t type, bool isTopQuery) { SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); STopBotRes* pRes = getTopBotOutputInfo(pCtx); STopBotResItem* pItems = pRes->pItems; - assert(pItems != NULL); // not full yet if (pEntryInfo->numOfRes < pRes->maxSize) { @@ -3211,7 +3349,9 @@ static void spreadTransferInfo(SSpreadInfo* pInput, SSpreadInfo* pOutput) { int32_t spreadFunctionMerge(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; - ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + if (pCol->info.type != TSDB_DATA_TYPE_BINARY) { + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; + } SSpreadInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); @@ -3390,7 +3530,9 @@ static void elapsedTransferInfo(SElapsedInfo* pInput, SElapsedInfo* pOutput) { int32_t elapsedFunctionMerge(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; - ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + if (pCol->info.type != TSDB_DATA_TYPE_BINARY) { + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; + } SElapsedInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); @@ -3560,7 +3702,9 @@ static bool getHistogramBinDesc(SHistoFuncInfo* pInfo, char* binDescStr, int8_t intervals[0] = -INFINITY; intervals[numOfBins - 1] = INFINITY; // in case of desc bin orders, -inf/inf should be swapped - ASSERT(numOfBins >= 4); + if (numOfBins < 4) { + return false; + } if (intervals[1] > intervals[numOfBins - 2]) { TSWAP(intervals[0], intervals[numOfBins - 1]); } @@ -3623,7 +3767,7 @@ bool histogramFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultIn pInfo->totalCount = 0; pInfo->normalized = 0; - char *binTypeStr = strndup(varDataVal(pCtx->param[1].param.pz), varDataLen(pCtx->param[1].param.pz)); + char* binTypeStr = strndup(varDataVal(pCtx->param[1].param.pz), varDataLen(pCtx->param[1].param.pz)); int8_t binType = getHistogramBinType(binTypeStr); taosMemoryFree(binTypeStr); @@ -3703,7 +3847,9 @@ static void histogramTransferInfo(SHistoFuncInfo* pInput, SHistoFuncInfo* pOutpu int32_t histogramFunctionMerge(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; - ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + if (pCol->info.type != TSDB_DATA_TYPE_BINARY) { + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; + } SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); @@ -3947,7 +4093,6 @@ int32_t hllFunctionMerge(SqlFunctionCtx* pCtx) { int32_t start = pInput->startRowIndex; - for (int32_t i = start; i < start + pInput->numOfRows; ++i) { char* data = colDataGetData(pCol, i); SHLLInfo* pInputInfo = (SHLLInfo*)varDataVal(data); @@ -4086,7 +4231,7 @@ static bool checkStateOp(int8_t op, SColumnInfoData* pCol, int32_t index, SVaria break; } default: { - ASSERT(0); + return false; } } return false; @@ -4418,12 +4563,15 @@ static void sampleAssignResult(SSampleInfo* pInfo, char* data, int32_t index) { assignVal(pInfo->data + index * pInfo->colBytes, data, pInfo->colBytes, pInfo->colType); } -static void doReservoirSample(SqlFunctionCtx* pCtx, SSampleInfo* pInfo, char* data, int32_t index) { +static int32_t doReservoirSample(SqlFunctionCtx* pCtx, SSampleInfo* pInfo, char* data, int32_t index) { pInfo->totalPoints++; if (pInfo->numSampled < pInfo->samples) { sampleAssignResult(pInfo, data, pInfo->numSampled); if (pCtx->subsidiaries.num > 0) { - pInfo->tuplePos[pInfo->numSampled] = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); + int32_t code = saveTupleData(pCtx, index, pCtx->pSrcBlock, &pInfo->tuplePos[pInfo->numSampled]); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } pInfo->numSampled++; } else { @@ -4431,10 +4579,15 @@ static void doReservoirSample(SqlFunctionCtx* pCtx, SSampleInfo* pInfo, char* da if (j < pInfo->samples) { sampleAssignResult(pInfo, data, j); if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, index, pCtx->pSrcBlock, &pInfo->tuplePos[j]); + int32_t code = updateTupleData(pCtx, index, pCtx->pSrcBlock, &pInfo->tuplePos[j]); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } } + + return TSDB_CODE_SUCCESS; } int32_t sampleFunction(SqlFunctionCtx* pCtx) { @@ -4450,11 +4603,17 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) { } char* data = colDataGetData(pInputCol, i); - doReservoirSample(pCtx, pInfo, data, i); + int32_t code = doReservoirSample(pCtx, pInfo, data, i); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (pInfo->numSampled == 0 && pCtx->subsidiaries.num > 0 && !pInfo->nullTupleSaved) { - pInfo->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL); + int32_t code = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, &pInfo->nullTuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pInfo->nullTupleSaved = true; } @@ -4463,6 +4622,7 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) { } int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); SSampleInfo* pInfo = getSampleOutputInfo(pCtx); @@ -4474,15 +4634,15 @@ int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t currentRow = pBlock->info.rows; if (pInfo->numSampled == 0) { colDataAppendNULL(pCol, currentRow); - setSelectivityValue(pCtx, pBlock, &pInfo->nullTuplePos, currentRow); - return pInfo->numSampled; + code = setSelectivityValue(pCtx, pBlock, &pInfo->nullTuplePos, currentRow); + return code; } for (int32_t i = 0; i < pInfo->numSampled; ++i) { colDataAppend(pCol, currentRow + i, pInfo->data + i * pInfo->colBytes, false); - setSelectivityValue(pCtx, pBlock, &pInfo->tuplePos[i], currentRow + i); + code = setSelectivityValue(pCtx, pBlock, &pInfo->tuplePos[i], currentRow + i); } - return pInfo->numSampled; + return code; } bool getTailFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { @@ -4748,7 +4908,7 @@ bool modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { return true; } -static void doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCtx, char* data) { +static int32_t doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCtx, char* data) { int32_t hashKeyBytes = IS_STR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes; SModeItem** pHashItem = taosHashGet(pInfo->pHash, data, hashKeyBytes); if (pHashItem == NULL) { @@ -4758,7 +4918,10 @@ static void doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCtx, pItem->count += 1; if (pCtx->subsidiaries.num > 0) { - pItem->tuplePos = saveTupleData(pCtx, rowIndex, pCtx->pSrcBlock, NULL); + int32_t code = saveTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &pItem->tuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } taosHashPut(pInfo->pHash, data, hashKeyBytes, &pItem, sizeof(SModeItem*)); @@ -4766,9 +4929,14 @@ static void doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCtx, } else { (*pHashItem)->count += 1; if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &((*pHashItem)->tuplePos)); + int32_t code = updateTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &((*pHashItem)->tuplePos)); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } + + return TSDB_CODE_SUCCESS; } int32_t modeFunction(SqlFunctionCtx* pCtx) { @@ -4789,7 +4957,10 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) { numOfElems++; char* data = colDataGetData(pInputCol, i); - doModeAdd(pInfo, i, pCtx, data); + int32_t code = doModeAdd(pInfo, i, pCtx, data); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (sizeof(SModeInfo) + pInfo->numOfPoints * (sizeof(SModeItem) + pInfo->colBytes) >= MODE_MAX_RESULT_SIZE) { taosHashCleanup(pInfo->pHash); @@ -4798,7 +4969,10 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) { } if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pInfo->nullTupleSaved) { - pInfo->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL); + int32_t code = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, &pInfo->nullTuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pInfo->nullTupleSaved = true; } @@ -4808,6 +4982,7 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) { } int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SModeInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; @@ -4827,15 +5002,15 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (maxCount != 0) { SModeItem* pResItem = (SModeItem*)(pInfo->pItems + resIndex * (sizeof(SModeItem) + pInfo->colBytes)); colDataAppend(pCol, currentRow, pResItem->data, false); - setSelectivityValue(pCtx, pBlock, &pResItem->tuplePos, currentRow); + code = setSelectivityValue(pCtx, pBlock, &pResItem->tuplePos, currentRow); } else { colDataAppendNULL(pCol, currentRow); - setSelectivityValue(pCtx, pBlock, &pInfo->nullTuplePos, currentRow); + code = setSelectivityValue(pCtx, pBlock, &pInfo->nullTuplePos, currentRow); } taosHashCleanup(pInfo->pHash); - return pResInfo->numOfRes; + return code; } bool getTwaFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) { @@ -4887,11 +5062,7 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } int32_t i = pInput->startRowIndex; - if (pCtx->start.key != INT64_MIN) { - // ASSERT((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) || - // (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC)); - - ASSERT(last->key == INT64_MIN); + if (pCtx->start.key != INT64_MIN && last->key == INT64_MIN) { for (; i < pInput->numOfRows + pInput->startRowIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { continue; @@ -5111,7 +5282,7 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } default: - ASSERT(0); + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; } // the last interpolated time window value @@ -5138,7 +5309,6 @@ _twa_over: * is required, we simply copy to the resut ot interResBuffer. */ // void twa_function_copy(SQLFunctionCtx *pCtx) { -// assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); // SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); // // memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); @@ -5192,6 +5362,7 @@ int32_t blockDistFunction(SqlFunctionCtx* pCtx) { pDistInfo->numOfBlocks += p1.numOfBlocks; pDistInfo->numOfTables += p1.numOfTables; pDistInfo->numOfInmemRows += p1.numOfInmemRows; + pDistInfo->numOfVgroups += p1.numOfVgroups; pDistInfo->totalSize += p1.totalSize; pDistInfo->totalRows += p1.totalRows; pDistInfo->numOfFiles += p1.numOfFiles; @@ -5226,6 +5397,7 @@ int32_t tSerializeBlockDistInfo(void* buf, int32_t bufLen, const STableBlockDist if (tEncodeU16(&encoder, pInfo->numOfFiles) < 0) return -1; if (tEncodeU32(&encoder, pInfo->numOfBlocks) < 0) return -1; if (tEncodeU32(&encoder, pInfo->numOfTables) < 0) return -1; + if (tEncodeU32(&encoder, pInfo->numOfVgroups) < 0) return -1; if (tEncodeU64(&encoder, pInfo->totalSize) < 0) return -1; if (tEncodeU64(&encoder, pInfo->totalRows) < 0) return -1; @@ -5257,6 +5429,7 @@ int32_t tDeserializeBlockDistInfo(void* buf, int32_t bufLen, STableBlockDistInfo if (tDecodeU16(&decoder, &pInfo->numOfFiles) < 0) return -1; if (tDecodeU32(&decoder, &pInfo->numOfBlocks) < 0) return -1; if (tDecodeU32(&decoder, &pInfo->numOfTables) < 0) return -1; + if (tDecodeU32(&decoder, &pInfo->numOfVgroups) < 0) return -1; if (tDecodeU64(&decoder, &pInfo->totalSize) < 0) return -1; if (tDecodeU64(&decoder, &pInfo->totalRows) < 0) return -1; @@ -5299,7 +5472,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t len = sprintf(st + VARSTR_HEADER_SIZE, "Total_Blocks=[%d] Total_Size=[%.2f Kb] Average_size=[%.2f Kb] Compression_Ratio=[%.2f %c]", - pData->numOfBlocks, pData->totalSize / 1024.0, averageSize/1024.0, compRatio, '%'); + pData->numOfBlocks, pData->totalSize / 1024.0, averageSize / 1024.0, compRatio, '%'); varDataSetLen(st, len); colDataAppend(pColInfo, row++, st, false); @@ -5317,7 +5490,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { colDataAppend(pColInfo, row++, st, false); len = sprintf(st + VARSTR_HEADER_SIZE, "Total_Tables=[%d] Total_Files=[%d] Total_Vgroups=[%d]", pData->numOfTables, - pData->numOfFiles, 0); + pData->numOfFiles, pData->numOfVgroups); varDataSetLen(st, len); colDataAppend(pColInfo, row++, st, false); @@ -5684,7 +5857,10 @@ int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx) { TSKEY cts = getRowPTs(pInput->pPTS, i); if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - doSaveLastrow(pCtx, data, i, cts, pInfo); + int32_t code = doSaveLastrow(pCtx, data, i, cts, pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pResInfo->numOfRes = 1; } } diff --git a/source/libs/function/src/detail/tavgfunction.c b/source/libs/function/src/detail/tavgfunction.c index 8e010181d17ea12fe08491498217d21c48e562bb..3a70a65ec4f8d5888f264b3fd9f97a82932554dc 100644 --- a/source/libs/function/src/detail/tavgfunction.c +++ b/source/libs/function/src/detail/tavgfunction.c @@ -366,7 +366,6 @@ bool avgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { static int32_t calculateAvgBySMAInfo(SAvgRes* pRes, int32_t numOfRows, int32_t type, const SColumnDataAgg* pAgg) { int32_t numOfElem = numOfRows - pAgg->numOfNull; - ASSERT(numOfElem >= 0); pRes->count += numOfElem; if (IS_SIGNED_NUMERIC_TYPE(type)) { @@ -672,7 +671,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { break; } default: - ASSERT(0); + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; } } else { numOfElem = doAddNumericVector(pCol, type, pInput, pAvgRes); @@ -706,7 +705,9 @@ static void avgTransferInfo(SAvgRes* pInput, SAvgRes* pOutput) { int32_t avgFunctionMerge(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; - ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + if (pCol->info.type != TSDB_DATA_TYPE_BINARY) { + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; + } SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); diff --git a/source/libs/function/src/detail/tminmax.c b/source/libs/function/src/detail/tminmax.c index cb5cea3cc851692a2a348ff2319f00d96b360119..847c7386550c13cc93e2e9e118fa0b8045804702 100644 --- a/source/libs/function/src/detail/tminmax.c +++ b/source/libs/function/src/detail/tminmax.c @@ -64,7 +64,7 @@ static void calculateRounds(int32_t numOfRows, int32_t bytes, int32_t* remainder, int32_t* rounds, int32_t* width) { const int32_t bitWidth = 256; - *width = (bitWidth>>3u) / bytes; + *width = (bitWidth >> 3u) / bytes; *remainder = numOfRows % (*width); *rounds = numOfRows / (*width); } @@ -92,8 +92,7 @@ static void calculateRounds(int32_t numOfRows, int32_t bytes, int32_t* remainder (_v) = (_sec)[j]; \ } \ } - - + static int8_t i8VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFunc, bool signVal) { int8_t v = 0; const int8_t* p = pData; @@ -116,7 +115,7 @@ static int8_t i8VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFu const int8_t* q = (const int8_t*)&initVal; EXTRACT_MAX_VAL(q, p, width, remain, v) - } else { // unsigned value + } else { // unsigned value for (int32_t i = 0; i < rounds; ++i) { next = _mm256_lddqu_si256((__m256i*)p); initVal = _mm256_max_epu8(initVal, next); @@ -126,7 +125,7 @@ static int8_t i8VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFu const uint8_t* q = (const uint8_t*)&initVal; EXTRACT_MAX_VAL(q, p, width, remain, v) } - + } else { // min function if (signVal) { for (int32_t i = 0; i < rounds; ++i) { @@ -241,7 +240,7 @@ static int32_t i32VectorCmpAVX2(const int32_t* pData, int32_t numOfRows, bool is // let compare the final results const int32_t* q = (const int32_t*)&initVal; EXTRACT_MAX_VAL(q, p, width, remain, v) - } else { // unsigned value + } else { // unsigned value for (int32_t i = 0; i < rounds; ++i) { next = _mm256_lddqu_si256((__m256i*)p); initVal = _mm256_max_epi32(initVal, next); @@ -281,7 +280,7 @@ static int32_t i32VectorCmpAVX2(const int32_t* pData, int32_t numOfRows, bool is } static float floatVectorCmpAVX(const float* pData, int32_t numOfRows, bool isMinFunc) { - float v = 0; + float v = 0; const float* p = pData; int32_t width, remain, rounds; @@ -358,7 +357,7 @@ static double doubleVectorCmpAVX(const double* pData, int32_t numOfRows, bool is static int32_t findFirstValPosition(const SColumnInfoData* pCol, int32_t start, int32_t numOfRows) { int32_t i = start; - + while (i < (start + numOfRows) && (colDataIsNull_f(pCol->nullbitmap, i) == true)) { i += 1; } @@ -495,7 +494,8 @@ static void handleInt64Col(const void* data, int32_t start, int32_t numOfRows, S } } -static void handleFloatCol(SColumnInfoData* pCol, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc) { +static void handleFloatCol(SColumnInfoData* pCol, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, + bool isMinFunc) { float* pData = (float*)pCol->pData; float* val = (float*)&pBuf->v; @@ -525,7 +525,8 @@ static void handleFloatCol(SColumnInfoData* pCol, int32_t start, int32_t numOfRo pBuf->assign = true; } -static void handleDoubleCol(SColumnInfoData* pCol, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc) { +static void handleDoubleCol(SColumnInfoData* pCol, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, + bool isMinFunc) { double* pData = (double*)pCol->pData; double* val = (double*)&pBuf->v; @@ -699,7 +700,7 @@ static void doExtractVal(SColumnInfoData* pCol, int32_t i, int32_t end, SqlFunct } } -int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { +int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems) { int32_t numOfElems = 0; SInputColumnInfoData* pInput = &pCtx->input; @@ -720,7 +721,6 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { // data in current data block are qualified to the query if (pInput->colDataSMAIsSet) { numOfElems = pInput->numOfRows - pAgg->numOfNull; - ASSERT(pInput->numOfRows == pInput->totalRows && numOfElems >= 0); if (numOfElems == 0) { goto _over; @@ -736,11 +736,19 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { } if (!pBuf->assign) { - pBuf->v = *(int64_t*)tval; + if (type == TSDB_DATA_TYPE_FLOAT) { + GET_FLOAT_VAL(&pBuf->v) = GET_DOUBLE_VAL(tval); + } else { + pBuf->v = GET_INT64_VAL(tval); + } + if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); if (index >= 0) { - pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); + int32_t code = saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } } else { @@ -750,11 +758,14 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { int64_t val = GET_INT64_VAL(tval); if ((prev < val) ^ isMinFunc) { - *(int64_t*)&pBuf->v = val; + GET_INT64_VAL(&pBuf->v) = val; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); if (index >= 0) { - pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); + int32_t code = saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } } @@ -764,11 +775,14 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { uint64_t val = GET_UINT64_VAL(tval); if ((prev < val) ^ isMinFunc) { - *(uint64_t*)&pBuf->v = val; + GET_UINT64_VAL(&pBuf->v) = val; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); if (index >= 0) { - pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); + int32_t code = saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } } @@ -778,11 +792,14 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { double val = GET_DOUBLE_VAL(tval); if ((prev < val) ^ isMinFunc) { - *(double*)&pBuf->v = val; + GET_DOUBLE_VAL(&pBuf->v) = val; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); if (index >= 0) { - pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); + int32_t code = saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } } @@ -792,20 +809,23 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { float val = GET_DOUBLE_VAL(tval); if ((prev < val) ^ isMinFunc) { - *(float*)&pBuf->v = val; + GET_FLOAT_VAL(&pBuf->v) = val; } if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); if (index >= 0) { - pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); + int32_t code = saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } } } pBuf->assign = true; - return numOfElems; + return TSDB_CODE_SUCCESS; } int32_t start = pInput->startRowIndex; @@ -819,14 +839,16 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { memcpy(&pBuf->v, pCol->pData + (pCol->info.bytes * i), pCol->info.bytes); if (pCtx->subsidiaries.num > 0) { - pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL); + int32_t code = saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } pBuf->assign = true; numOfElems = 1; } if (i >= end) { - ASSERT(numOfElems == 0); goto _over; } @@ -883,9 +905,13 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { _over: if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pBuf->nullTupleSaved) { - pBuf->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL); + int32_t code = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, &pBuf->nullTuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pBuf->nullTupleSaved = true; } - return numOfElems; -} \ No newline at end of file + *nElems = numOfElems; + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/function/src/tfunctionInt.c b/source/libs/function/src/tfunctionInt.c index 70378df0c3315c03989ae673ec50ecf1a47d8278..edfd8660106fa8ce1141bb979a78cd4d9a771bbe 100644 --- a/source/libs/function/src/tfunctionInt.c +++ b/source/libs/function/src/tfunctionInt.c @@ -20,7 +20,6 @@ #include "ttypes.h" #include "function.h" -#include "tbuffer.h" #include "tcompression.h" #include "tdatablock.h" #include "tfunctionInt.h" @@ -41,8 +40,6 @@ int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock } } - assert(maxRows >= 0); - blockDataEnsureCapacity(pResBlock, maxRows); for (int32_t i = 0; i < num; ++i) { SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i); @@ -64,7 +61,6 @@ int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock } bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry) { - assert(pEntry != NULL); return pEntry->complete; } diff --git a/source/libs/function/src/thistogram.c b/source/libs/function/src/thistogram.c index b0f23f78df72afa23015a947632e024082cacd46..e7d631f638da769fe0d9eabb03762bb983410a56 100644 --- a/source/libs/function/src/thistogram.c +++ b/source/libs/function/src/thistogram.c @@ -73,7 +73,10 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { #if defined(USE_ARRAYLIST) int32_t idx = histoBinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val); - assert(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL); + if (ASSERTS(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL, "tHistogramAdd Error, idx:%d, maxEntries:%d, elems:%p", + idx, (*pHisto)->maxEntries, (*pHisto)->elems)) { + return -1; + } if ((*pHisto)->elems[idx].val == val && idx >= 0) { (*pHisto)->elems[idx].num += 1; @@ -84,15 +87,27 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { } else { /* insert a new slot */ if ((*pHisto)->numOfElems >= 1 && idx < (*pHisto)->numOfEntries) { if (idx > 0) { - assert((*pHisto)->elems[idx - 1].val <= val); + if (ASSERTS((*pHisto)->elems[idx - 1].val <= val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf", + idx - 1, (*pHisto)->elems[idx - 1].val, val)) { + return -1; + } } else { - assert((*pHisto)->elems[idx].val > val); + if (ASSERTS((*pHisto)->elems[idx].val > val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf", + idx, (*pHisto)->elems[idx].val, val)) { + return -1; + } } } else if ((*pHisto)->numOfElems > 0) { - assert((*pHisto)->elems[(*pHisto)->numOfEntries].val <= val); + if (ASSERTS((*pHisto)->elems[(*pHisto)->numOfEntries].val <= val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf", + (*pHisto)->numOfEntries, (*pHisto)->elems[idx].val, val)) { + return -1; + } } - histogramCreateBin(*pHisto, idx, val); + int32_t code = histogramCreateBin(*pHisto, idx, val); + if (code != 0) { + return code; + } } #else tSkipListKey key = tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &val, tDataTypes[TSDB_DATA_TYPE_DOUBLE].nSize); @@ -151,7 +166,6 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { if ((*pHisto)->numOfEntries >= (*pHisto)->maxEntries + 1) { // set the right value for loser-tree - assert((*pHisto)->pLoserTree != NULL); if (!(*pHisto)->ordered) { SSkipListPrint((*pHisto)->pList, 1); @@ -203,7 +217,10 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { tSkipListNode* pNext = pNode->pForward[0]; SHistBin* pNextEntry = (SHistBin*)pNext->pData; - assert(pNextEntry->val - pEntry->val == pEntry->delta); + if (ASSERTS(pNextEntry->val - pEntry->val == pEntry->delta, "tHistogramAdd Error, pNextEntry->val:%lf, pEntry->val:%lf, pEntry->delta:%lf", + pNextEntry->val, pEntry->val, pEntry->delta)) { + return -1; + } double newVal = (pEntry->val * pEntry->num + pNextEntry->val * pNextEntry->num) / (pEntry->num + pNextEntry->num); pEntry->val = newVal; @@ -253,7 +270,9 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { } else { SHistBin* pEntry = (SHistBin*)pResNode->pData; - assert(pEntry->val == val); + if (ASSERTS(pEntry->val == val, "tHistogramAdd Error, pEntry->val:%lf, val:%lf")) { + return -1; + } pEntry->num += 1; } @@ -329,7 +348,10 @@ int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val) { memmove(&pHisto->elems[index + 1], &pHisto->elems[index], sizeof(SHistBin) * remain); } - assert(index >= 0 && index <= pHisto->maxEntries); + if (ASSERTS(index >= 0 && index <= pHisto->maxEntries, "histogramCreateBin Error, index:%d, maxEntries:%d", + index, pHisto->maxEntries)) { + return -1; + } pHisto->elems[index].num = 1; pHisto->elems[index].val = val; @@ -343,7 +365,11 @@ int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val) { pHisto->elems[pHisto->maxEntries].num = 0; } #endif - assert(pHisto->numOfEntries <= pHisto->maxEntries); + if (ASSERTS(pHisto->numOfEntries <= pHisto->maxEntries, "histogramCreateBin Error, numOfEntries:%d, maxEntries:%d", + pHisto->numOfEntries, pHisto->maxEntries)) { + return -1; + } + return 0; } @@ -386,12 +412,14 @@ int64_t tHistogramSum(SHistogramInfo* pHisto, double v) { if (slotIdx < 0) { slotIdx = 0; - assert(v <= pHisto->elems[slotIdx].val); + ASSERTS(v <= pHisto->elems[slotIdx].val, "tHistogramSum Error, elems[%d].val:%lf, v:%lf", + slotIdx, pHisto->elems[slotIdx].val, v); } else { - assert(v >= pHisto->elems[slotIdx].val); - + ASSERTS(v >= pHisto->elems[slotIdx].val, "tHistogramSum Error, elems[%d].val:%lf, v:%lf", + slotIdx, pHisto->elems[slotIdx].val, v); if (slotIdx + 1 < pHisto->numOfEntries) { - assert(v < pHisto->elems[slotIdx + 1].val); + ASSERTS(v < pHisto->elems[slotIdx + 1].val, "tHistogramSum Error, elems[%d].val:%lf, v:%lf", + slotIdx + 1, pHisto->elems[slotIdx + 1].val, v); } } } @@ -445,7 +473,9 @@ double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) { j += 1; } - assert(total <= numOfElem && total + pHisto->elems[j + 1].num > numOfElem); + ASSERTS(total <= numOfElem && total + pHisto->elems[j + 1].num > numOfElem, + "tHistogramUniform Error, total:%d, numOfElem:%d, elems[%d].num:%d", + total, numOfElem, j + 1, pHisto->elems[j + 1].num); double delta = numOfElem - total; if (fabs(delta) < FLT_EPSILON) { @@ -502,7 +532,9 @@ double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) { j += 1; } - assert(total <= numOfElem && total + pEntry->num > numOfElem); + ASSERTS(total <= numOfElem && total + pEntry->num > numOfElem, + "tHistogramUniform Error, total:%d, numOfElem:%d, pEntry->num:%d", + total, numOfElem, pEntry->num); double delta = numOfElem - total; if (fabs(delta) < FLT_EPSILON) { diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 157ee08f1522118e0ab12daa5426c1f44c00adda..6577858ee693257b4ddf02f3ca1ab1d1234e40a6 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -33,13 +33,23 @@ static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) (SFilePage *)taosMemoryCalloc(1, pMemBucket->bytes * pMemBucket->pSlots[slotIdx].info.size + sizeof(SFilePage)); int32_t groupId = getGroupId(pMemBucket->numOfSlots, slotIdx, pMemBucket->times); - SArray *pIdList = *(SArray **)taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId)); + + SArray *pIdList; + void *p = taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId)); + if (p != NULL) { + pIdList = *(SArray **)p; + } else { + return NULL; + } int32_t offset = 0; for (int32_t i = 0; i < taosArrayGetSize(pIdList); ++i) { int32_t *pageId = taosArrayGet(pIdList, i); SFilePage *pg = getBufPage(pMemBucket->pBuffer, *pageId); + if (pg == NULL) { + return NULL; + } memcpy(buffer->data + offset, pg->data, (size_t)(pg->num * pMemBucket->bytes)); offset += (int32_t)(pg->num * pMemBucket->bytes); @@ -87,8 +97,9 @@ static void resetPosInfo(SSlotInfo *pInfo) { pInfo->data = NULL; } -double findOnlyResult(tMemBucket *pMemBucket) { - assert(pMemBucket->total == 1); +int32_t findOnlyResult(tMemBucket *pMemBucket, double *result) { + ASSERT(pMemBucket->total == 1); + terrno = 0; for (int32_t i = 0; i < pMemBucket->numOfSlots; ++i) { tMemBucketSlot *pSlot = &pMemBucket->pSlots[i]; @@ -100,19 +111,22 @@ double findOnlyResult(tMemBucket *pMemBucket) { SArray **pList = taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId)); if (pList != NULL) { SArray *list = *pList; - assert(list->size == 1); + ASSERT(list->size == 1); int32_t *pageId = taosArrayGet(list, 0); SFilePage *pPage = getBufPage(pMemBucket->pBuffer, *pageId); - assert(pPage->num == 1); + if (pPage == NULL) { + return TSDB_CODE_NO_AVAIL_DISK; + } + ASSERT(pPage->num == 1); - double v = 0; - GET_TYPED_DATA(v, double, pMemBucket->type, pPage->data); - return v; + GET_TYPED_DATA(*result, double, pMemBucket->type, pPage->data); + return TSDB_CODE_SUCCESS; } } - return 0; + *result = 0.0; + return TSDB_CODE_SUCCESS; } int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) { @@ -140,7 +154,8 @@ int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) { } } - assert(index >= 0 && index < pBucket->numOfSlots); + ASSERTS(index >= 0 && index < pBucket->numOfSlots, "tBucketIntHash Error, index:%d, numOfSlots:%d", + index, pBucket->numOfSlots); return index; } @@ -167,7 +182,7 @@ int32_t tBucketUintHash(tMemBucket *pBucket, const void *value) { } } - assert(index >= 0 && index < pBucket->numOfSlots); + ASSERT(index >= 0 && index < pBucket->numOfSlots); return index; } @@ -198,7 +213,7 @@ int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value) { } } - assert(index >= 0 && index < pBucket->numOfSlots); + ASSERT(index >= 0 && index < pBucket->numOfSlots); return index; } @@ -331,7 +346,7 @@ void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataT r->dMaxVal = v; } } else { - assert(0); + ASSERT(0); } } @@ -339,7 +354,7 @@ void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataT * in memory bucket, we only accept data array list */ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { - assert(pBucket != NULL && data != NULL && size > 0); + ASSERT(pBucket != NULL && data != NULL && size > 0); int32_t count = 0; int32_t bytes = pBucket->bytes; @@ -361,7 +376,7 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { if (pSlot->info.data == NULL || pSlot->info.data->num >= pBucket->elemPerPage) { if (pSlot->info.data != NULL) { - assert(pSlot->info.data->num >= pBucket->elemPerPage && pSlot->info.size > 0); + ASSERT(pSlot->info.data->num >= pBucket->elemPerPage && pSlot->info.size > 0); // keep the pointer in memory setBufPageDirty(pSlot->info.data, true); @@ -379,6 +394,9 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { } pSlot->info.data = getNewBufPage(pBucket->pBuffer, &pageId); + if (pSlot->info.data == NULL) { + return TSDB_CODE_NO_AVAIL_DISK; + } pSlot->info.pageId = pageId; taosArrayPush(pPageIdList, &pageId); } @@ -390,7 +408,7 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { } pBucket->total += count; - return 0; + return TSDB_CODE_SUCCESS; } //////////////////////////////////////////////////////////////////////////////////////////// @@ -407,14 +425,14 @@ static MinMaxEntry getMinMaxEntryOfNextSlotWithData(tMemBucket *pMemBucket, int3 ++j; } - assert(j < pMemBucket->numOfSlots); + ASSERT(j < pMemBucket->numOfSlots); return pMemBucket->pSlots[j].range; } static bool isIdenticalData(tMemBucket *pMemBucket, int32_t index); static double getIdenticalDataVal(tMemBucket *pMemBucket, int32_t slotIndex) { - assert(isIdenticalData(pMemBucket, slotIndex)); + ASSERT(isIdenticalData(pMemBucket, slotIndex)); tMemBucketSlot *pSlot = &pMemBucket->pSlots[slotIndex]; @@ -430,7 +448,7 @@ static double getIdenticalDataVal(tMemBucket *pMemBucket, int32_t slotIndex) { return finalResult; } -double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) { +int32_t getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction, double *result) { int32_t num = 0; for (int32_t i = 0; i < pMemBucket->numOfSlots; ++i) { @@ -461,15 +479,18 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) minOfNextSlot = (double)next.dMinVal; } - assert(minOfNextSlot > maxOfThisSlot); + ASSERT(minOfNextSlot > maxOfThisSlot); - double val = (1 - fraction) * maxOfThisSlot + fraction * minOfNextSlot; - return val; + *result = (1 - fraction) * maxOfThisSlot + fraction * minOfNextSlot; + return TSDB_CODE_SUCCESS; } if (pSlot->info.size <= pMemBucket->maxCapacity) { // data in buffer and file are merged together to be processed. SFilePage *buffer = loadDataFromFilePage(pMemBucket, i); + if (buffer == NULL) { + return TSDB_CODE_NO_AVAIL_DISK; + } int32_t currentIdx = count - num; char *thisVal = buffer->data + pMemBucket->bytes * currentIdx; @@ -479,13 +500,14 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) GET_TYPED_DATA(td, double, pMemBucket->type, thisVal); GET_TYPED_DATA(nd, double, pMemBucket->type, nextVal); - double val = (1 - fraction) * td + fraction * nd; + *result = (1 - fraction) * td + fraction * nd; taosMemoryFreeClear(buffer); - return val; + return TSDB_CODE_SUCCESS; } else { // incur a second round bucket split if (isIdenticalData(pMemBucket, i)) { - return getIdenticalDataVal(pMemBucket, i); + *result = getIdenticalDataVal(pMemBucket, i); + return TSDB_CODE_SUCCESS; } // try next round @@ -498,36 +520,53 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) resetSlotInfo(pMemBucket); int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times - 1); - SArray* list = *(SArray **)taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId)); - ASSERT(list != NULL && list->size > 0); + + SArray* list; + void *p = taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId)); + if (p != NULL) { + list = *(SArray **)p; + if (list == NULL || list->size <= 0) { + return -1; + } + } else { + return -1; + } for (int32_t f = 0; f < list->size; ++f) { int32_t *pageId = taosArrayGet(list, f); SFilePage *pg = getBufPage(pMemBucket->pBuffer, *pageId); - - tMemBucketPut(pMemBucket, pg->data, (int32_t)pg->num); + if (pg == NULL) { + return TSDB_CODE_NO_AVAIL_DISK; + } + + int32_t code = tMemBucketPut(pMemBucket, pg->data, (int32_t)pg->num); + if (code != TSDB_CODE_SUCCESS) { + return code; + } setBufPageDirty(pg, true); releaseBufPage(pMemBucket->pBuffer, pg); } - return getPercentileImpl(pMemBucket, count - num, fraction); + return getPercentileImpl(pMemBucket, count - num, fraction, result); } } else { num += pSlot->info.size; } } - return 0; + *result = 0; + return TSDB_CODE_SUCCESS; } -double getPercentile(tMemBucket *pMemBucket, double percent) { +int32_t getPercentile(tMemBucket *pMemBucket, double percent, double *result) { if (pMemBucket->total == 0) { - return 0.0; + *result = 0.0; + return TSDB_CODE_SUCCESS; } // if only one elements exists, return it if (pMemBucket->total == 1) { - return findOnlyResult(pMemBucket); + return findOnlyResult(pMemBucket, result); } percent = fabs(percent); @@ -537,21 +576,21 @@ double getPercentile(tMemBucket *pMemBucket, double percent) { MinMaxEntry *pRange = &pMemBucket->range; if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) { - double v = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->i64MaxVal : pRange->i64MinVal); - return v; + *result = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->i64MaxVal : pRange->i64MinVal); } else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) { - double v = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->u64MaxVal : pRange->u64MinVal); - return v; + *result = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->u64MaxVal : pRange->u64MinVal); } else { - return fabs(percent - 100) < DBL_EPSILON ? pRange->dMaxVal : pRange->dMinVal; + *result = fabs(percent - 100) < DBL_EPSILON ? pRange->dMaxVal : pRange->dMinVal; } + + return TSDB_CODE_SUCCESS; } double percentVal = (percent * (pMemBucket->total - 1)) / ((double)100.0); // do put data by using buckets int32_t orderIdx = (int32_t)percentVal; - return getPercentileImpl(pMemBucket, orderIdx, percentVal - orderIdx); + return getPercentileImpl(pMemBucket, orderIdx, percentVal - orderIdx, result); } /* diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 0b309bc8f56abd3870a07eebdb94e61f357ee23c..c9fa70ff11a6909022623affd3a0ebf1850837f5 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -841,36 +841,42 @@ int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SS for (int32_t i = 0; i < numOfCols; ++i) { numOfRows = (input[i].numOfRows > numOfRows) ? input[i].numOfRows : numOfRows; } - output->info.rows = numOfRows; - output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - for (int32_t i = 0; i < numOfCols; ++i) { - if ((input+i)->numOfRows < numOfRows) { - SColumnInfoData* pColInfoData = (input+i)->columnData; - int32_t startRow = (input+i)->numOfRows; - int32_t expandRows = numOfRows - startRow; - colInfoDataEnsureCapacity(pColInfoData, numOfRows, false); + + // create the basic block info structure + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pInfo = input[i].columnData; + SColumnInfoData d = {0}; + d.info = pInfo->info; + + blockDataAppendColInfo(output, &d); + } + + blockDataEnsureCapacity(output, numOfRows); + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pDest = taosArrayGet(output->pDataBlock, i); + + SColumnInfoData* pColInfoData = input[i].columnData; + colDataAssign(pDest, pColInfoData, input[i].numOfRows, &output->info); + + if (input[i].numOfRows < numOfRows) { + int32_t startRow = input[i].numOfRows; + int expandRows = numOfRows - startRow; bool isNull = colDataIsNull_s(pColInfoData, (input+i)->numOfRows - 1); if (isNull) { - colDataAppendNNULL(pColInfoData, startRow, expandRows); + colDataAppendNNULL(pDest, startRow, expandRows); } else { char* src = colDataGetData(pColInfoData, (input + i)->numOfRows - 1); - int32_t bytes = pColInfoData->info.bytes; - char* data = taosMemoryMalloc(bytes); - memcpy(data, src, bytes); for (int j = 0; j < expandRows; ++j) { - colDataAppend(pColInfoData, startRow+j, data, false); + colDataAppend(pDest, startRow+j, src, false); } //colDataAppendNItems(pColInfoData, startRow, data, expandRows); - taosMemoryFree(data); } } + } - taosArrayPush(output->pDataBlock, (input + i)->columnData); + output->info.rows = numOfRows; - if (IS_VAR_DATA_TYPE((input + i)->columnData->info.type)) { - output->info.hasVarCol = true; - } - } return 0; } @@ -1224,7 +1230,6 @@ int32_t udfcGetUdfTaskResultFromUvTask(SClientUdfTask *task, SClientUvTaskNode * if (uvTask->rspBuf.base != NULL) { SUdfResponse rsp = {0}; void *buf = decodeUdfResponse(uvTask->rspBuf.base, &rsp); - assert(uvTask->rspBuf.len == POINTER_DISTANCE(buf, uvTask->rspBuf.base)); task->errCode = rsp.code; switch (task->type) { @@ -1824,8 +1829,8 @@ int32_t doCallUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t convertDataBlockToScalarParm(&resultBlock, output); taosArrayDestroy(resultBlock.pDataBlock); } - - taosArrayDestroy(inputBlock.pDataBlock); + + blockDataFreeRes(&inputBlock); return err; } diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 40c75ce6baacbaad258ad10874af16f9466ad2d4..fe6ed7d785c7b6a6d99adbe9f930294bf8b950ef 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -400,7 +400,6 @@ void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { SUdfdRpcSendRecvInfo *msgInfo = (SUdfdRpcSendRecvInfo *)pMsg->info.ahandle; - ASSERT(pMsg->info.ahandle != NULL); if (pEpSet) { if (!isEpsetEqual(&global.mgmtEp.epSet, pEpSet)) { @@ -461,13 +460,14 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { #else snprintf(path, sizeof(path), "%s/lib%s.so", tsTempDir, pFuncInfo->name); #endif - TdFilePtr file = - taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL); + + TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); if (file == NULL) { fnError("udfd write udf shared library: %s failed, error: %d %s", path, errno, strerror(errno)); msgInfo->code = TSDB_CODE_FILE_CORRUPTED; goto _return; } + int64_t count = taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); if (count != pFuncInfo->codeSize) { fnError("udfd write udf shared library failed"); @@ -606,6 +606,7 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { } static bool udfdRpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_SYN_NOT_LEADER || + code == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) { if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || diff --git a/source/libs/index/inc/indexUtil.h b/source/libs/index/inc/indexUtil.h index dbaecaa9630b04b8b50f108c1a59e499f04899dc..148a521d5ac9ed12e566338da96f0755c2420eb4 100644 --- a/source/libs/index/inc/indexUtil.h +++ b/source/libs/index/inc/indexUtil.h @@ -36,7 +36,6 @@ extern "C" { #define SERIALIZE_VAR_TO_BUF(buf, var, type) \ do { \ type c = var; \ - assert(sizeof(type) == sizeof(c)); \ memcpy((void *)buf, (void *)&c, sizeof(c)); \ buf += sizeof(c); \ } while (0) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 76dc84ae42a3d38bea29b030be12a240bd3f510d..a99c87b7f9a5bef05c8e641ab4aa826e7c5e7dfc 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -226,7 +226,9 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { indexDebug("w suid:%" PRIu64 ", colName:%s, colType:%d", key.suid, key.colName, key.colType); IndexCache** cache = taosHashGet(index->colObj, buf, sz); - assert(*cache != NULL); + ASSERTS(*cache != NULL, "index-cache already release"); + if (*cache == NULL) return -1; + int ret = idxCachePut(*cache, p, uid); if (ret != 0) { return ret; diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index e3f140047ab1fa02b328813293300096386cc4d6..c2ac7f4478a6200d9408432e591132fed87a0208 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -170,7 +170,6 @@ TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t d } return tDoCompare(func, cmptype, &va, &vb); } - assert(0); return BREAK; #endif } @@ -367,7 +366,7 @@ int32_t idxConvertData(void* src, int8_t type, void** dst) { tlen = taosEncodeBinary(dst, src, strlen(src)); break; default: - ASSERT(0); + ASSERTS(0, "index invalid input type"); break; } *dst = (char*)*dst - tlen; @@ -459,7 +458,7 @@ int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) { *dst = (char*)*dst - tlen; break; default: - ASSERT(0); + ASSERTS(0, "index invalid input type"); break; } return tlen; diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 07696c38225796978cecb02aca2f65de5bf93558..5c14424b7ecb56a45405cd48daa018a5f8837da9 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -206,7 +206,9 @@ static FORCE_INLINE int32_t sifGetValueFromNode(SNode *node, char **value) { static FORCE_INLINE int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) { SOperatorNode *nd = (SOperatorNode *)node; - assert(nodeType(node) == QUERY_NODE_OPERATOR); + if (nodeType(node) != QUERY_NODE_OPERATOR) { + return -1; + } SColumnNode *l = (SColumnNode *)nd->pLeft; SValueNode *r = (SValueNode *)nd->pRight; diff --git a/source/libs/index/src/indexFst.c b/source/libs/index/src/indexFst.c index bbffcfa6c17aa628b0afc79ba96746d764add523..f3b7b2fbae9dc05db6645ba92b96bbaf04f212e2 100644 --- a/source/libs/index/src/indexFst.c +++ b/source/libs/index/src/indexFst.c @@ -65,10 +65,7 @@ void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes* nodes, bool isFinal) { taosArrayPush(nodes->stack, &un); } FstBuilderNode* fstUnFinishedNodesPopRoot(FstUnFinishedNodes* nodes) { - assert(taosArrayGetSize(nodes->stack) == 1); - FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); - assert(un->last == NULL); return un->node; } @@ -82,7 +79,6 @@ FstBuilderNode* fstUnFinishedNodesPopFreeze(FstUnFinishedNodes* nodes, CompiledA FstBuilderNode* fstUnFinishedNodesPopEmpty(FstUnFinishedNodes* nodes) { FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); - assert(un->last == NULL); return un->node; } void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes* nodes, Output out) { @@ -102,7 +98,8 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output } int32_t sz = taosArrayGetSize(nodes->stack) - 1; FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz); - assert(un->last == NULL); + ASSERTS(un->last == NULL, "index-fst meet unexpected node"); + if (un->last != NULL) return; // FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition)); // trn->inp = s->data[s->start]; @@ -247,7 +244,6 @@ void fstStateCompileForOneTrans(IdxFstFile* w, CompiledAddr addr, FstTransition* } void fstStateCompileForAnyTrans(IdxFstFile* w, CompiledAddr addr, FstBuilderNode* node) { int32_t sz = taosArrayGetSize(node->trans); - assert(sz <= 256); uint8_t tSize = 0; uint8_t oSize = packSize(node->finalOutput); @@ -322,7 +318,7 @@ void fstStateCompileForAnyTrans(IdxFstFile* w, CompiledAddr addr, FstBuilderNode // set_comm_input void fstStateSetCommInput(FstState* s, uint8_t inp) { - assert(s->state == OneTransNext || s->state == OneTrans); + ASSERT(s->state == OneTransNext || s->state == OneTrans); uint8_t val; COMMON_INDEX(inp, 0b111111, val); @@ -331,7 +327,7 @@ void fstStateSetCommInput(FstState* s, uint8_t inp) { // comm_input uint8_t fstStateCommInput(FstState* s, bool* null) { - assert(s->state == OneTransNext || s->state == OneTrans); + ASSERT(s->state == OneTransNext || s->state == OneTrans); uint8_t v = s->val & 0b00111111; if (v == 0) { *null = true; @@ -344,7 +340,7 @@ uint8_t fstStateCommInput(FstState* s, bool* null) { // input_len uint64_t fstStateInputLen(FstState* s) { - assert(s->state == OneTransNext || s->state == OneTrans); + ASSERT(s->state == OneTransNext || s->state == OneTrans); bool null = false; fstStateCommInput(s, &null); return null ? 1 : 0; @@ -352,11 +348,11 @@ uint64_t fstStateInputLen(FstState* s) { // end_addr uint64_t fstStateEndAddrForOneTransNext(FstState* s, FstSlice* data) { - assert(s->state == OneTransNext); + ASSERT(s->state == OneTransNext); return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s); } uint64_t fstStateEndAddrForOneTrans(FstState* s, FstSlice* data, PackSizes sizes) { - assert(s->state == OneTrans); + ASSERT(s->state == OneTrans); return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s) - 1 // pack size - FST_GET_TRANSITION_PACK_SIZE(sizes) - FST_GET_OUTPUT_PACK_SIZE(sizes); } @@ -370,7 +366,7 @@ uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* } // input uint8_t fstStateInput(FstState* s, FstNode* node) { - assert(s->state == OneTransNext || s->state == OneTrans); + ASSERT(s->state == OneTransNext || s->state == OneTrans); FstSlice* slice = &node->data; bool null = false; uint8_t inp = fstStateCommInput(s, &null); @@ -378,7 +374,7 @@ uint8_t fstStateInput(FstState* s, FstNode* node) { return null == false ? inp : data[node->start - 1]; } uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { - assert(s->state == AnyTrans); + ASSERT(s->state == AnyTrans); FstSlice* slice = &node->data; uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size @@ -390,7 +386,7 @@ uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { // trans_addr CompiledAddr fstStateTransAddr(FstState* s, FstNode* node) { - assert(s->state == OneTransNext || s->state == OneTrans); + ASSERT(s->state == OneTransNext || s->state == OneTrans); FstSlice* slice = &node->data; if (s->state == OneTransNext) { return (CompiledAddr)(node->end) - 1; @@ -406,7 +402,7 @@ CompiledAddr fstStateTransAddr(FstState* s, FstNode* node) { } } CompiledAddr fstStateTransAddrForAnyTrans(FstState* s, FstNode* node, uint64_t i) { - assert(s->state == AnyTrans); + ASSERT(s->state == AnyTrans); FstSlice* slice = &node->data; uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); @@ -418,7 +414,7 @@ CompiledAddr fstStateTransAddrForAnyTrans(FstState* s, FstNode* node, uint64_t i // sizes PackSizes fstStateSizes(FstState* s, FstSlice* slice) { - assert(s->state == OneTrans || s->state == AnyTrans); + ASSERT(s->state == OneTrans || s->state == AnyTrans); uint64_t i; if (s->state == OneTrans) { i = FST_SLICE_LEN(slice) - 1 - fstStateInputLen(s) - 1; @@ -431,7 +427,7 @@ PackSizes fstStateSizes(FstState* s, FstSlice* slice) { } // Output Output fstStateOutput(FstState* s, FstNode* node) { - assert(s->state == OneTrans); + ASSERT(s->state == OneTrans); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); if (oSizes == 0) { @@ -445,7 +441,7 @@ Output fstStateOutput(FstState* s, FstNode* node) { return unpackUint64(data + i, oSizes); } Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { - assert(s->state == AnyTrans); + ASSERT(s->state == AnyTrans); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); if (oSizes == 0) { @@ -462,19 +458,19 @@ Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { // anyTrans specify function void fstStateSetFinalState(FstState* s, bool yes) { - assert(s->state == AnyTrans); + ASSERT(s->state == AnyTrans); if (yes) { s->val |= 0b01000000; } return; } bool fstStateIsFinalState(FstState* s) { - assert(s->state == AnyTrans); + ASSERT(s->state == AnyTrans); return (s->val & 0b01000000) == 0b01000000; } void fstStateSetStateNtrans(FstState* s, uint8_t n) { - assert(s->state == AnyTrans); + ASSERT(s->state == AnyTrans); if (n <= 0b00111111) { s->val = (s->val & 0b11000000) | n; } @@ -482,7 +478,7 @@ void fstStateSetStateNtrans(FstState* s, uint8_t n) { } // state_ntrans uint8_t fstStateStateNtrans(FstState* s, bool* null) { - assert(s->state == AnyTrans); + ASSERT(s->state == AnyTrans); *null = false; uint8_t n = s->val & 0b00111111; @@ -492,16 +488,16 @@ uint8_t fstStateStateNtrans(FstState* s, bool* null) { return n; } uint64_t fstStateTotalTransSize(FstState* s, uint64_t version, PackSizes sizes, uint64_t nTrans) { - assert(s->state == AnyTrans); + ASSERT(s->state == AnyTrans); uint64_t idxSize = fstStateTransIndexSize(s, version, nTrans); return nTrans + (nTrans * FST_GET_TRANSITION_PACK_SIZE(sizes)) + idxSize; } uint64_t fstStateTransIndexSize(FstState* s, uint64_t version, uint64_t nTrans) { - assert(s->state == AnyTrans); + ASSERT(s->state == AnyTrans); return (version >= 2 && nTrans > TRANS_INDEX_THRESHOLD) ? 256 : 0; } uint64_t fstStateNtransLen(FstState* s) { - assert(s->state == AnyTrans); + ASSERT(s->state == AnyTrans); bool null = false; fstStateStateNtrans(s, &null); return null == true ? 1 : 0; @@ -530,7 +526,7 @@ Output fstStateFinalOutput(FstState* s, uint64_t version, FstSlice* slice, PackS return unpackUint64(data + at, (uint8_t)oSizes); } uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) { - assert(s->state == AnyTrans); + ASSERT(s->state == AnyTrans); FstSlice* slice = &node->data; if (node->version >= 2 && node->nTrans > TRANS_INDEX_THRESHOLD) { uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size @@ -676,17 +672,17 @@ bool fstNodeGetTransitionAddrAt(FstNode* node, uint64_t i, CompiledAddr* res) { bool s = true; FstState* st = &node->state; if (st->state == OneTransNext) { - assert(i == 0); + ASSERT(i == 0); fstStateTransAddr(st, node); } else if (st->state == OneTrans) { - assert(i == 0); + ASSERT(i == 0); fstStateTransAddr(st, node); } else if (st->state == AnyTrans) { fstStateTransAddrForAnyTrans(st, node, i); } else if (FST_STATE_EMPTY_FINAL(node)) { s = false; } else { - assert(0); + ASSERT(0); } return s; } @@ -722,7 +718,7 @@ bool fstNodeFindInput(FstNode* node, uint8_t b, uint64_t* res) { bool fstNodeCompile(FstNode* node, void* w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode* builderNode) { int32_t sz = taosArrayGetSize(builderNode->trans); - assert(sz < 256); + ASSERT(sz < 256); if (sz == 0 && builderNode->isFinal && builderNode->finalOutput == 0) { return true; } else if (sz != 1 || builderNode->isFinal) { @@ -804,7 +800,7 @@ void fstBuilderInsertOutput(FstBuilder* b, FstSlice bs, Output in) { uint64_t prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); if (prefixLen == FST_SLICE_LEN(s)) { - assert(out == 0); + ASSERT(out == 0); return; } @@ -848,7 +844,7 @@ void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate) { addr = fstBuilderCompile(b, bn); fstBuilderNodeDestroy(bn); - assert(addr != NONE_ADDRESS); + ASSERT(addr != NONE_ADDRESS); } fstUnFinishedNodesTopLastFreeze(b->unfinished, addr); return; diff --git a/source/libs/index/src/indexFstDfa.c b/source/libs/index/src/indexFstDfa.c index 8ce0ba1e69eb069cfe7b3c2009107d0eea5d5019..4d348e76f25eb9ec4575b536f2b5de73f1fabe6a 100644 --- a/source/libs/index/src/indexFstDfa.c +++ b/source/libs/index/src/indexFstDfa.c @@ -104,8 +104,9 @@ bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet DfaState *t = taosArrayGet(builder->dfa->states, state); for (int i = 0; i < taosArrayGetSize(t->insts); i++) { int32_t ip = *(int32_t *)taosArrayGet(t->insts, i); - bool succ = sparSetAdd(cur, ip, NULL); - assert(succ == true); + + bool succ = sparSetAdd(cur, ip, NULL); + if (succ == false) return false; } dfaRun(builder->dfa, cur, next, byte); diff --git a/source/libs/index/src/indexFstFile.c b/source/libs/index/src/indexFstFile.c index 4620af8694e538a3525adaed11a1277ec14cd0a9..5a9c8dfe3d3906bd452111bf885e62b65638f6e5 100644 --- a/source/libs/index/src/indexFstFile.c +++ b/source/libs/index/src/indexFstFile.c @@ -100,7 +100,7 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of do { char key[1024] = {0}; - assert(strlen(ctx->file.buf) + 1 + 64 < sizeof(key)); + ASSERT(strlen(ctx->file.buf) + 1 + 64 < sizeof(key)); idxGenLRUKey(key, ctx->file.buf, blkId); LRUHandle* h = taosLRUCacheLookup(ctx->lru, key, strlen(key)); @@ -114,7 +114,8 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of if (left < kBlockSize) { nread = TMIN(left, len); int32_t bytes = taosPReadFile(ctx->file.pFile, buf + total, nread, offset); - assert(bytes == nread); + ASSERTS(bytes == nread, "index read incomplete data"); + if (bytes != nread) break; total += bytes; return total; @@ -124,7 +125,8 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of SDataBlock* blk = taosMemoryCalloc(1, cacheMemSize); blk->blockId = blkId; blk->nread = taosPReadFile(ctx->file.pFile, blk->buf, kBlockSize, blkId * kBlockSize); - assert(blk->nread <= kBlockSize); + ASSERTS(blk->nread <= kBlockSize, "index read incomplete data"); + if (blk->nread > kBlockSize) break; if (blk->nread < kBlockSize && blk->nread < len) { taosMemoryFree(blk); @@ -275,7 +277,10 @@ int idxFileWrite(IdxFstFile* write, uint8_t* buf, uint32_t len) { // update checksum IFileCtx* ctx = write->wrt; int nWrite = ctx->write(ctx, buf, len); - assert(nWrite == len); + ASSERTS(nWrite == len, "index write incomplete data"); + if (nWrite != len) { + return -1; + } write->count += len; write->summer = taosCalcChecksum(write->summer, buf, len); @@ -302,7 +307,6 @@ int idxFileFlush(IdxFstFile* write) { } void idxFilePackUintIn(IdxFstFile* writer, uint64_t n, uint8_t nBytes) { - assert(1 <= nBytes && nBytes <= 8); uint8_t* buf = taosMemoryCalloc(8, sizeof(uint8_t)); for (uint8_t i = 0; i < nBytes; i++) { buf[i] = (uint8_t)n; diff --git a/source/libs/index/src/indexFstRegister.c b/source/libs/index/src/indexFstRegister.c index e0abcadc78a07b0f69ef92003d4304141551865e..adafecccb13dd6928b6ef4bd2ee2df0ce8aedf2a 100644 --- a/source/libs/index/src/indexFstRegister.c +++ b/source/libs/index/src/indexFstRegister.c @@ -57,8 +57,8 @@ static void fstRegistryCellPromote(SArray* arr, uint32_t start, uint32_t end) { if (start >= sz && end >= sz) { return; } - - assert(start >= end); + ASSERTS(start >= end, "index-fst start lower than end"); + if (start < end) return; int32_t s = (int32_t)start; int32_t e = (int32_t)end; diff --git a/source/libs/index/src/indexFstUtil.c b/source/libs/index/src/indexFstUtil.c index b1a919b365742791fc7daaae1f6ea47f9b012477..f1e8808cf522cd75525562971c3854de383c50f0 100644 --- a/source/libs/index/src/indexFstUtil.c +++ b/source/libs/index/src/indexFstUtil.c @@ -101,7 +101,6 @@ FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end) { int32_t slen; uint8_t* data = fstSliceData(s, &slen); - assert(tlen <= slen); uint8_t* buf = taosMemoryMalloc(sizeof(uint8_t) * tlen); memcpy(buf, data + start, tlen); diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index b34d05d297e26f03d5af40e8b440241cf55ed714..d921ca7103c55d51421a10818b2b23964854869c 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -122,7 +122,6 @@ TFileCache* tfileCacheCreate(SIndex* idx, const char* path) { char buf[128] = {0}; int32_t sz = idxSerialCacheKey(&key, buf); - assert(sz < sizeof(buf)); taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*)); tfileReaderRef(reader); } @@ -151,9 +150,8 @@ void tfileCacheDestroy(TFileCache* tcache) { } TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) { - char buf[128] = {0}; - int32_t sz = idxSerialCacheKey(key, buf); - assert(sz < sizeof(buf)); + char buf[128] = {0}; + int32_t sz = idxSerialCacheKey(key, buf); TFileReader** reader = taosHashGet(tcache->tableCache, buf, sz); if (reader == NULL || *reader == NULL) { return NULL; @@ -877,7 +875,7 @@ static int tfileWriteFooter(TFileWriter* write) { int nwrite = write->ctx->write(write->ctx, buf, (int32_t)strlen(buf)); indexInfo("tfile write footer size: %d", write->ctx->size(write->ctx)); - assert(nwrite == sizeof(FILE_MAGIC_NUMBER)); + ASSERTS(nwrite == sizeof(FILE_MAGIC_NUMBER), "index write incomplete data"); return nwrite; } static int tfileReaderLoadHeader(TFileReader* reader) { @@ -892,7 +890,6 @@ static int tfileReaderLoadHeader(TFileReader* reader) { } else { indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf), reader->ctx->file.buf); } - // assert(nread == sizeof(buf)); memcpy(&reader->header, buf, sizeof(buf)); return 0; @@ -914,7 +911,10 @@ static int tfileReaderLoadFst(TFileReader* reader) { indexInfo("nread = %d, and fst offset=%d, fst size: %d, filename: %s, file size: %d, time cost: %" PRId64 "us", nread, reader->header.fstOffset, fstSize, ctx->file.buf, size, cost); // we assuse fst size less than FST_MAX_SIZE - assert(nread > 0 && nread <= fstSize); + ASSERTS(nread > 0 && nread <= fstSize, "index read incomplete fst"); + if (nread <= 0 || nread > fstSize) { + return -1; + } FstSlice st = fstSliceCreate((uint8_t*)buf, nread); reader->fst = fstCreate(&st); @@ -929,7 +929,7 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* // add block cache char block[4096] = {0}; int32_t nread = ctx->readFrom(ctx, block, sizeof(block), offset); - assert(nread >= sizeof(uint32_t)); + ASSERT(nread >= sizeof(uint32_t)); char* p = block; int32_t nid = *(int32_t*)p; diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index b3ca0fa4528682f024cb2c101e2c99e1235ae164..b23a36d4df3acaef40116b2fd66559a661e3f8c1 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -20,6 +20,7 @@ #include "ttime.h" static SMonitor tsMonitor = {0}; +static char* tsMonUri = "/report"; void monRecordLog(int64_t ts, ELogLevel level, const char *content) { taosThreadMutexLock(&tsMonitor.lock); @@ -550,7 +551,7 @@ void monSendReport() { // uDebugL("report cont:%s\n", pCont); if (pCont != NULL) { EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; - if (taosSendHttpReport(tsMonitor.cfg.server, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { + if (taosSendHttpReport(tsMonitor.cfg.server, tsMonUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { uError("failed to send monitor msg"); } taosMemoryFree(pCont); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index fb9560341e3f9f2ff89deb9375cd2a28933a6334..12041bd2ae8557a1832c735b128fd4ca7861106f 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -409,6 +409,7 @@ static int32_t logicAggCopy(const SAggLogicNode* pSrc, SAggLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pGroupKeys); CLONE_NODE_LIST_FIELD(pAggFuncs); + COPY_SCALAR_FIELD(hasGroupKeyOptimized); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index e9ec2ce306078b6ea4cb49089d83e44cdf4ac412..d9b3237993732b21da75ab9eea5249e9e4019226 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -173,6 +173,10 @@ const char* nodesNodeName(ENodeType type) { return "BalanceVgroupStmt"; case QUERY_NODE_MERGE_VGROUP_STMT: return "MergeVgroupStmt"; + case QUERY_NODE_SHOW_DB_ALIVE_STMT: + return "ShowDbAliveStmt"; + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: + return "ShowClusterAliveStmt"; case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT: return "RedistributeVgroupStmt"; case QUERY_NODE_SPLIT_VGROUP_STMT: @@ -1914,6 +1918,7 @@ static const char* jkAggPhysiPlanExprs = "Exprs"; static const char* jkAggPhysiPlanGroupKeys = "GroupKeys"; static const char* jkAggPhysiPlanAggFuncs = "AggFuncs"; static const char* jkAggPhysiPlanMergeDataBlock = "MergeDataBlock"; +static const char* jkAggPhysiPlanGroupKeyOptimized = "GroupKeyOptimized"; static int32_t physiAggNodeToJson(const void* pObj, SJson* pJson) { const SAggPhysiNode* pNode = (const SAggPhysiNode*)pObj; @@ -1931,6 +1936,9 @@ static int32_t physiAggNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkAggPhysiPlanMergeDataBlock, pNode->mergeDataBlock); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkAggPhysiPlanGroupKeyOptimized, pNode->groupKeyOptimized); + } return code; } @@ -1951,6 +1959,9 @@ static int32_t jsonToPhysiAggNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkAggPhysiPlanMergeDataBlock, &pNode->mergeDataBlock); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkAggPhysiPlanGroupKeyOptimized, &pNode->groupKeyOptimized); + } return code; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index f5e4bc1ea4ddeff6b6e408c71856435b5db9845f..a4b77fad6928cdb46c52e44266f74c1f3ec6fb46 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2370,7 +2370,8 @@ enum { PHY_AGG_CODE_EXPR, PHY_AGG_CODE_GROUP_KEYS, PHY_AGG_CODE_AGG_FUNCS, - PHY_AGG_CODE_MERGE_DATA_BLOCK + PHY_AGG_CODE_MERGE_DATA_BLOCK, + PHY_AGG_CODE_GROUP_KEY_OPTIMIZE }; static int32_t physiAggNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { @@ -2389,6 +2390,9 @@ static int32_t physiAggNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeBool(pEncoder, PHY_AGG_CODE_MERGE_DATA_BLOCK, pNode->mergeDataBlock); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_AGG_CODE_GROUP_KEY_OPTIMIZE, pNode->groupKeyOptimized); + } return code; } @@ -2415,6 +2419,9 @@ static int32_t msgToPhysiAggNode(STlvDecoder* pDecoder, void* pObj) { case PHY_AGG_CODE_MERGE_DATA_BLOCK: code = tlvDecodeBool(pTlv, &pNode->mergeDataBlock); break; + case PHY_AGG_CODE_GROUP_KEY_OPTIMIZE: + code = tlvDecodeBool(pTlv, &pNode->groupKeyOptimized); + break; default: break; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 38203e61b0c95dfabcb8aa8d1a5f05bb73daa26b..61bb3778fa2d692816e99f43c5b09713b7673341 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -434,6 +434,9 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SShowDnodeVariablesStmt)); case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: return makeNode(type, sizeof(SShowCreateDatabaseStmt)); + case QUERY_NODE_SHOW_DB_ALIVE_STMT: + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: + return makeNode(type, sizeof(SShowAliveStmt)); case QUERY_NODE_SHOW_CREATE_TABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT: return makeNode(type, sizeof(SShowCreateTableStmt)); @@ -832,7 +835,8 @@ void nodesDestroyNode(SNode* pNode) { if (pStmt->freeArrayFunc) { pStmt->freeArrayFunc(pStmt->pVgDataBlocks); } - tdDestroySVCreateTbReq(&pStmt->createTblReq); + tdDestroySVCreateTbReq(pStmt->pCreateTblReq); + taosMemoryFreeClear(pStmt->pCreateTblReq); taosCloseFile(&pStmt->fp); break; } @@ -959,6 +963,8 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_SHOW_USERS_STMT: case QUERY_NODE_SHOW_LICENCES_STMT: case QUERY_NODE_SHOW_VGROUPS_STMT: + case QUERY_NODE_SHOW_DB_ALIVE_STMT: + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: case QUERY_NODE_SHOW_TOPICS_STMT: case QUERY_NODE_SHOW_CONSUMERS_STMT: case QUERY_NODE_SHOW_CONNECTIONS_STMT: diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index c74ec9c1479b872c11daf65bf1a0df0caa093a44..d202e60a4575c9ad3efcdbcccc938a4c8fe1ed31 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -176,6 +176,7 @@ SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type); SNode* createShowStmtWithCond(SAstCreateContext* pCxt, ENodeType type, SNode* pDbName, SNode* pTbName, EOperatorType tableCondType); SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName); +SNode* createShowAliveStmt(SAstCreateContext* pCxt, SNode* pDbName, ENodeType type); SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable); SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable); SNode* createShowDnodeVariablesStmt(SAstCreateContext* pCxt, SNode* pDnodeId, SNode* pLikePattern); @@ -187,6 +188,7 @@ SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName); SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort); SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, bool force); SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue); +SNode* createRealTableNodeForIndexName(SAstCreateContext* pCxt, SToken* pDbName, SToken* pIndexName); SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SNode* pIndexName, SNode* pRealTable, SNodeList* pCols, SNode* pOptions); SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInterval, SNode* pOffset, SNode* pSliding, @@ -194,15 +196,13 @@ SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInt SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pIndexName); SNode* createCreateComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId); SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId); -SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, - SNode* pQuery); -SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, - const SToken* pSubDbName, bool withMeta); -SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, - SNode* pRealTable, bool withMeta); -SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pTopicName); -SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pCGroupId, - const SToken* pTopicName); +SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pQuery); +SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SToken* pSubDbName, + bool withMeta); +SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pRealTable, + bool withMeta); +SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pTopicName); +SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pCGroupId, SToken* pTopicName); SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue); SNode* createDefaultExplainOptions(SAstCreateContext* pCxt); SNode* setExplainVerbose(SAstCreateContext* pCxt, SNode* pOptions, const SToken* pVal); @@ -215,9 +215,9 @@ SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool const SToken* pLibPath, SDataType dataType, int32_t bufSize); SNode* createDropFunctionStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pFuncName); SNode* createStreamOptions(SAstCreateContext* pCxt); -SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pStreamName, SNode* pRealTable, - SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery); -SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pStreamName); +SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pStreamName, SNode* pRealTable, + SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery, SNodeList* pCols); +SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pStreamName); SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId); SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId); SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt); diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index 5cc72f86923762454c8cae66ef9bbc7828b9ef05..7b816359f9c77b5fe370177fae2aedc67d6cc9a3 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -20,8 +20,6 @@ struct SToken; -#define IS_DATA_COL_ORDERED(spd) ((spd->orderStatus) == (int8_t)ORDER_STATUS_ORDERED) - #define NEXT_TOKEN(pSql, sToken) \ do { \ int32_t index = 0; \ @@ -37,108 +35,27 @@ struct SToken; } \ } while (0) -typedef enum EOrderStatus { - ORDER_STATUS_UNKNOWN = 0, - ORDER_STATUS_ORDERED = 1, - ORDER_STATUS_DISORDERED = 2, -} EOrderStatus; - -typedef enum EValStat { - VAL_STAT_HAS = 0x0, // 0 means has val - VAL_STAT_NONE = 0x01, // 1 means no val -} EValStat; - -typedef struct SBoundColumn { - int32_t offset; // all column offset value - int32_t toffset; // first part offset for SDataRow TODO: get offset from STSchema on future - uint8_t valStat; // EValStat. denote if current column bound or not(0 means has val, 1 means no val) -} SBoundColumn; - -typedef struct { - col_id_t schemaColIdx; - col_id_t boundIdx; - col_id_t finalIdx; -} SBoundIdxInfo; - -typedef struct SParsedDataColInfo { - col_id_t numOfCols; - col_id_t numOfBound; - uint16_t flen; // TODO: get from STSchema - uint16_t allNullLen; // TODO: get from STSchema(base on SDataRow) - uint16_t extendedVarLen; - uint16_t boundNullLen; // bound column len with all NULL value(without VarDataOffsetT/SColIdx part) - col_id_t *boundColumns; // bound column idx according to schema - SBoundColumn *cols; - SBoundIdxInfo *colIdxInfo; - int8_t orderStatus; // bound columns -} SParsedDataColInfo; - -typedef struct SInsertParseBaseContext { - SParseContext *pComCxt; - char *pSql; - SMsgBuf msg; -} SInsertParseBaseContext; - -typedef struct SInsertParseSyntaxCxt { - SParseContext *pComCxt; - char *pSql; - SMsgBuf msg; - SParseMetaCache *pMetaCache; -} SInsertParseSyntaxCxt; - -typedef struct SMemParam { - SRowBuilder *rb; - SSchema *schema; - int32_t toffset; - col_id_t colIdx; -} SMemParam; - -typedef struct { - uint8_t rowType; // default is 0, that is SDataRow - int32_t rowSize; -} SMemRowBuilder; - -typedef struct STableDataBlocks { - int8_t tsSource; // where does the UNIX timestamp come from, server or client - bool ordered; // if current rows are ordered or not - int32_t vgId; // virtual group id - int64_t prevTS; // previous timestamp, recorded to decide if the records array is ts ascending - int32_t numOfTables; // number of tables in current submit block - int32_t rowSize; // row size for current table - uint32_t nAllocSize; - uint32_t headerSize; // header for table info (uid, tid, submit metadata) - uint32_t size; - STableMeta *pTableMeta; // the tableMeta of current table, the table meta will be used during submit, keep a ref to - // avoid to be removed from cache - char *pData; - bool cloned; - int32_t createTbReqLen; - SParsedDataColInfo boundColumnInfo; - SRowBuilder rowBuilder; -} STableDataBlocks; +typedef struct SVgroupDataCxt { + int32_t vgId; + SSubmitReq2 *pData; +} SVgroupDataCxt; -int32_t insGetExtendedRowSize(STableDataBlocks *pBlock); -void insGetSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo *spd, col_id_t idx, int32_t *toffset, col_id_t *colIdx); -int32_t insSetBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks *dataBuf, int32_t numOfRows, SMsgBuf *pMsg); -int32_t insSchemaIdxCompar(const void *lhs, const void *rhs); -int32_t insBoundIdxCompar(const void *lhs, const void *rhs); -void insSetBoundColumnInfo(SParsedDataColInfo *pColList, SSchema *pSchema, col_id_t numOfCols); -void insDestroyBlockArrayList(SArray *pDataBlockList); -void insDestroyBlockHashmap(SHashObj *pDataBlockHash); -int32_t insInitRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo *pColInfo); -int32_t insGetDataBlockFromList(SHashObj *pHashList, void *id, int32_t idLen, int32_t size, int32_t startOffset, - int32_t rowSize, STableMeta *pTableMeta, STableDataBlocks **dataBlocks, - SArray *pBlockList, SVCreateTbReq *pCreateTbReq); -int32_t insMergeTableDataBlocks(SHashObj *pHashObj, SArray **pVgDataBlocks); -int32_t insBuildCreateTbMsg(STableDataBlocks *pBlocks, SVCreateTbReq *pCreateTbReq); -int32_t insAllocateMemForSize(STableDataBlocks *pDataBlock, int32_t allSize); int32_t insCreateSName(SName *pName, struct SToken *pTableName, int32_t acctId, const char *dbName, SMsgBuf *pMsgBuf); -int32_t insFindCol(struct SToken *pColname, int32_t start, int32_t end, SSchema *pSchema); +int16_t insFindCol(struct SToken *pColname, int16_t start, int16_t end, SSchema *pSchema); void insBuildCreateTbReq(SVCreateTbReq *pTbReq, const char *tname, STag *pTag, int64_t suid, const char *sname, SArray *tagName, uint8_t tagNum, int32_t ttl); -int32_t insMemRowAppend(SMsgBuf *pMsgBuf, const void *value, int32_t len, void *param); -int32_t insCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start); -int32_t insBuildOutput(SHashObj *pVgroupsHashObj, SArray *pVgDataBlocks, SArray **pDataBlocks); -void insDestroyDataBlock(STableDataBlocks *pDataBlock); +int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo *pInfo); +void insCheckTableDataOrder(STableDataCxt *pTableCxt, TSKEY tsKey); +int32_t insGetTableDataCxt(SHashObj *pHash, void *id, int32_t idLen, STableMeta *pTableMeta, + SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode); +int32_t initTableColSubmitData(STableDataCxt *pTableCxt); +int32_t insMergeTableDataCxt(SHashObj *pTableHash, SArray **pVgDataBlocks); +int32_t insBuildVgDataBlocks(SHashObj *pVgroupsHashObj, SArray *pVgDataBlocks, SArray **pDataBlocks); +void insDestroyTableDataCxtHashMap(SHashObj *pTableCxtHash); +void insDestroyVgroupDataCxt(SVgroupDataCxt *pVgCxt); +void insDestroyVgroupDataCxtList(SArray *pVgCxtList); +void insDestroyVgroupDataCxtHashMap(SHashObj *pVgCxtHash); +void insDestroyTableDataCxt(STableDataCxt *pTableCxt); +void insDestroyBoundColInfo(SBoundColInfo *pInfo); #endif // TDENGINE_PAR_INSERT_UTIL_H diff --git a/source/libs/parser/inc/parToken.h b/source/libs/parser/inc/parToken.h index fb4b46aa35094fd3ac65171c5499c038eb92d233..642979c471ab7b409bd622de26c2c2deb1879e83 100644 --- a/source/libs/parser/inc/parToken.h +++ b/source/libs/parser/inc/parToken.h @@ -175,8 +175,6 @@ _end: void taosCleanupKeywordsTable(); -SToken taosTokenDup(SToken *pToken, char *buf, int32_t len); - #ifdef __cplusplus } #endif diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index cd1260584e5ea6c4beea82088f0b28f6e7c40d33..abb34e7e806efa72da072f258e0352a0d1fdedc7 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -433,6 +433,9 @@ cmd ::= SHOW TAGS FROM table_name_cond(A) from_db_opt(B). cmd ::= SHOW TABLE TAGS tag_list_opt(C) FROM table_name_cond(A) from_db_opt(B). { pCxt->pRootNode = createShowTableTagsStmt(pCxt, A, B, C); } cmd ::= SHOW VNODES NK_INTEGER(A). { pCxt->pRootNode = createShowVnodesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &A), NULL); } cmd ::= SHOW VNODES NK_STRING(A). { pCxt->pRootNode = createShowVnodesStmt(pCxt, NULL, createValueNode(pCxt, TSDB_DATA_TYPE_VARCHAR, &A)); } +// show alive +cmd ::= SHOW db_name_cond_opt(A) ALIVE. { pCxt->pRootNode = createShowAliveStmt(pCxt, A, QUERY_NODE_SHOW_DB_ALIVE_STMT); } +cmd ::= SHOW CLUSTER ALIVE. { pCxt->pRootNode = createShowAliveStmt(pCxt, NULL, QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT); } db_name_cond_opt(A) ::= . { A = createDefaultDatabaseCondValue(pCxt); } db_name_cond_opt(A) ::= db_name(B) NK_DOT. { A = createIdentifierValueNode(pCxt, &B); } @@ -458,14 +461,19 @@ tag_item(A) ::= column_name(B) column_alias(C). tag_item(A) ::= column_name(B) AS column_alias(C). { A = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &B), &C); } /************************************************ create index ********************************************************/ -cmd ::= CREATE SMA INDEX not_exists_opt(D) - full_table_name(A) ON full_table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, D, A, B, NULL, C); } -cmd ::= DROP INDEX exists_opt(B) full_table_name(A). { pCxt->pRootNode = createDropIndexStmt(pCxt, B, A); } +cmd ::= CREATE SMA INDEX not_exists_opt(D) + full_index_name(A) ON full_table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, D, A, B, NULL, C); } +cmd ::= CREATE INDEX not_exists_opt(D) + full_index_name(A) ON full_table_name(B) NK_LP col_name_list(C) NK_RP. { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_NORMAL, D, A, B, C, NULL); } +cmd ::= DROP INDEX exists_opt(B) full_index_name(A). { pCxt->pRootNode = createDropIndexStmt(pCxt, B, A); } -index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL +full_index_name(A) ::= index_name(B). { A = createRealTableNodeForIndexName(pCxt, NULL, &B); } +full_index_name(A) ::= db_name(B) NK_DOT index_name(C). { A = createRealTableNodeForIndexName(pCxt, &B, &C); } + +index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL NK_LP duration_literal(C) NK_RP sliding_opt(D) sma_stream_opt(E). { A = createIndexOption(pCxt, B, releaseRawExprNode(pCxt, C), NULL, D, E); } -index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL - NK_LP duration_literal(C) NK_COMMA duration_literal(D) NK_RP sliding_opt(E) +index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL + NK_LP duration_literal(C) NK_COMMA duration_literal(D) NK_RP sliding_opt(E) sma_stream_opt(F). { A = createIndexOption(pCxt, B, releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, D), E, F); } %type func_list { SNodeList* } @@ -539,9 +547,15 @@ bufsize_opt(A) ::= BUFSIZE NK_INTEGER(B). /************************************************ create/drop stream **************************************************/ cmd ::= CREATE STREAM not_exists_opt(E) stream_name(A) stream_options(B) INTO - full_table_name(C) tags_def_opt(F) subtable_opt(G) AS query_or_subquery(D). { pCxt->pRootNode = createCreateStreamStmt(pCxt, E, &A, C, B, F, G, D); } + full_table_name(C) col_list_opt(H) tags_def_opt(F) subtable_opt(G) + AS query_or_subquery(D). { pCxt->pRootNode = createCreateStreamStmt(pCxt, E, &A, C, B, F, G, D, H); } cmd ::= DROP STREAM exists_opt(A) stream_name(B). { pCxt->pRootNode = createDropStreamStmt(pCxt, A, &B); } +%type col_list_opt { SNodeList* } +%destructor col_list_opt { nodesDestroyList($$); } +col_list_opt(A) ::= . { A = NULL; } +col_list_opt(A) ::= NK_LP col_name_list(B) NK_RP. { A = B; } + stream_options(A) ::= . { A = createStreamOptions(pCxt); } stream_options(A) ::= stream_options(B) TRIGGER AT_ONCE. { ((SStreamOptions*)B)->triggerType = STREAM_TRIGGER_AT_ONCE; A = B; } stream_options(A) ::= stream_options(B) TRIGGER WINDOW_CLOSE. { ((SStreamOptions*)B)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; A = B; } @@ -665,6 +679,10 @@ stream_name(A) ::= NK_ID(B). %destructor cgroup_name { } cgroup_name(A) ::= NK_ID(B). { A = B; } +%type index_name { SToken } +%destructor index_name { } +index_name(A) ::= NK_ID(B). { A = B; } + /************************************************ expression **********************************************************/ expr_or_subquery(A) ::= expression(B). { A = B; } //expr_or_subquery(A) ::= subquery(B). { A = createTempTableNode(pCxt, releaseRawExprNode(pCxt, B), NULL); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index db43fa5ed94c3431dca308439250c9b664ff0cf0..f582adb0c6fe7d75cd0ac68c035bac5dc5c71926 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -201,6 +201,24 @@ static bool checkIndexName(SAstCreateContext* pCxt, SToken* pIndexName) { return true; } +static bool checkTopicName(SAstCreateContext* pCxt, SToken* pTopicName) { + trimEscape(pTopicName); + if (pTopicName->n >= TSDB_TOPIC_NAME_LEN) { + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pTopicName->z); + return false; + } + return true; +} + +static bool checkStreamName(SAstCreateContext* pCxt, SToken* pStreamName) { + trimEscape(pStreamName); + if (pStreamName->n >= TSDB_STREAM_NAME_LEN) { + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pStreamName->z); + return false; + } + return true; +} + static bool checkComment(SAstCreateContext* pCxt, const SToken* pCommentToken, bool demand) { if (NULL == pCommentToken) { pCxt->errCode = demand ? TSDB_CODE_PAR_SYNTAX_ERROR : TSDB_CODE_SUCCESS; @@ -1370,6 +1388,38 @@ SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName) { return (SNode*)pStmt; } +SNode* createShowAliveStmt(SAstCreateContext* pCxt, SNode* pNode, ENodeType type) { + CHECK_PARSER_STATUS(pCxt); + SToken dbToken = {0}; + SToken* pDbToken = NULL; + + if (pNode) { + SValueNode* pDbName = (SValueNode*)pNode; + if (pDbName->literal) { + dbToken.z = pDbName->literal; + dbToken.n = strlen(pDbName->literal); + pDbToken = &dbToken; + } + } + + if (pDbToken && !checkDbName(pCxt, pDbToken, true)) { + nodesDestroyNode(pNode); + return NULL; + } + + SShowAliveStmt* pStmt = (SShowAliveStmt*)nodesMakeNode(type); + CHECK_OUT_OF_MEM(pStmt); + + if (pDbToken) { + COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbToken); + } + if (pNode) { + nodesDestroyNode(pNode); + } + + return (SNode*)pStmt; +} + SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable) { CHECK_PARSER_STATUS(pCxt); SShowCreateTableStmt* pStmt = (SShowCreateTableStmt*)nodesMakeNode(type); @@ -1523,6 +1573,13 @@ SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const return (SNode*)pStmt; } +SNode* createRealTableNodeForIndexName(SAstCreateContext* pCxt, SToken* pDbName, SToken* pIndexName) { + if (!checkIndexName(pCxt, pIndexName)) { + return NULL; + } + return createRealTableNode(pCxt, pDbName, pIndexName, NULL); +} + SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SNode* pIndexName, SNode* pRealTable, SNodeList* pCols, SNode* pOptions) { CHECK_PARSER_STATUS(pCxt); @@ -1581,9 +1638,11 @@ SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, cons return (SNode*)pStmt; } -SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, - SNode* pQuery) { +SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pQuery) { CHECK_PARSER_STATUS(pCxt); + if (!checkTopicName(pCxt, pTopicName)) { + return NULL; + } SCreateTopicStmt* pStmt = (SCreateTopicStmt*)nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT); CHECK_OUT_OF_MEM(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName); @@ -1592,9 +1651,12 @@ SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, return (SNode*)pStmt; } -SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, - const SToken* pSubDbName, bool withMeta) { +SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SToken* pSubDbName, + bool withMeta) { CHECK_PARSER_STATUS(pCxt); + if (!checkTopicName(pCxt, pTopicName) || !checkDbName(pCxt, pSubDbName, true)) { + return NULL; + } SCreateTopicStmt* pStmt = (SCreateTopicStmt*)nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT); CHECK_OUT_OF_MEM(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName); @@ -1604,9 +1666,12 @@ SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, co return (SNode*)pStmt; } -SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, - SNode* pRealTable, bool withMeta) { +SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pRealTable, + bool withMeta) { CHECK_PARSER_STATUS(pCxt); + if (!checkTopicName(pCxt, pTopicName)) { + return NULL; + } SCreateTopicStmt* pStmt = (SCreateTopicStmt*)nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT); CHECK_OUT_OF_MEM(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName); @@ -1618,8 +1683,11 @@ SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, return (SNode*)pStmt; } -SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pTopicName) { +SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pTopicName) { CHECK_PARSER_STATUS(pCxt); + if (!checkTopicName(pCxt, pTopicName)) { + return NULL; + } SDropTopicStmt* pStmt = (SDropTopicStmt*)nodesMakeNode(QUERY_NODE_DROP_TOPIC_STMT); CHECK_OUT_OF_MEM(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName); @@ -1628,8 +1696,11 @@ SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const } SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pCGroupId, - const SToken* pTopicName) { + SToken* pTopicName) { CHECK_PARSER_STATUS(pCxt); + if (!checkTopicName(pCxt, pTopicName)) { + return NULL; + } SDropCGroupStmt* pStmt = (SDropCGroupStmt*)nodesMakeNode(QUERY_NODE_DROP_CGROUP_STMT); CHECK_OUT_OF_MEM(pStmt); pStmt->ignoreNotExists = ignoreNotExists; @@ -1741,27 +1812,32 @@ SNode* createStreamOptions(SAstCreateContext* pCxt) { return (SNode*)pOptions; } -SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pStreamName, SNode* pRealTable, - SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery) { +SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pStreamName, SNode* pRealTable, + SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery, SNodeList* pCols) { CHECK_PARSER_STATUS(pCxt); + if (!checkStreamName(pCxt, pStreamName)) { + return NULL; + } SCreateStreamStmt* pStmt = (SCreateStreamStmt*)nodesMakeNode(QUERY_NODE_CREATE_STREAM_STMT); CHECK_OUT_OF_MEM(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->streamName, pStreamName); - if (NULL != pRealTable) { - strcpy(pStmt->targetDbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->targetTabName, ((SRealTableNode*)pRealTable)->table.tableName); - nodesDestroyNode(pRealTable); - } + strcpy(pStmt->targetDbName, ((SRealTableNode*)pRealTable)->table.dbName); + strcpy(pStmt->targetTabName, ((SRealTableNode*)pRealTable)->table.tableName); + nodesDestroyNode(pRealTable); pStmt->ignoreExists = ignoreExists; pStmt->pOptions = (SStreamOptions*)pOptions; pStmt->pQuery = pQuery; pStmt->pTags = pTags; pStmt->pSubtable = pSubtable; + pStmt->pCols = pCols; return (SNode*)pStmt; } -SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pStreamName) { +SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pStreamName) { CHECK_PARSER_STATUS(pCxt); + if (!checkStreamName(pCxt, pStreamName)) { + return NULL; + } SDropStreamStmt* pStmt = (SDropStreamStmt*)nodesMakeNode(QUERY_NODE_DROP_STREAM_STMT); CHECK_OUT_OF_MEM(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->streamName, pStreamName); diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 954e6c26fd6850ea8d870b09d09040828dd856dc..126027c78f0629cf5490cce490cc819fa4e2984f 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -166,7 +166,7 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const c code = reserveDnodeRequiredInCache(pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code && - (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS) || 0 == strcmp(pTable, TSDB_INS_TABLE_TABLES)) && + (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS) || 0 == strcmp(pTable, TSDB_INS_TABLE_TABLES) || 0 == strcmp(pTable, TSDB_INS_TABLE_COLS)) && QUERY_NODE_SELECT_STMT == nodeType(pCxt->pStmt)) { code = collectMetaKeyFromInsTags(pCxt); } @@ -355,7 +355,12 @@ static int32_t collectMetaKeyFromDescribe(SCollectMetaKeyCxt* pCxt, SDescribeStm } static int32_t collectMetaKeyFromCreateStream(SCollectMetaKeyCxt* pCxt, SCreateStreamStmt* pStmt) { - return collectMetaKeyFromQuery(pCxt, pStmt->pQuery); + int32_t code = + reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->targetDbName, pStmt->targetTabName, pCxt->pMetaCache); + if (TSDB_CODE_SUCCESS == code) { + code = collectMetaKeyFromQuery(pCxt, pStmt->pQuery); + } + return code; } static int32_t collectMetaKeyFromShowDnodes(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index d99d7d744579836def251114671c2c58b45259df..e4de60fd051d6f7f4e781160305ffb95a3ec7d3e 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -145,6 +145,8 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { case QUERY_NODE_SHOW_CLUSTER_STMT: case QUERY_NODE_SHOW_LICENCES_STMT: case QUERY_NODE_SHOW_VGROUPS_STMT: + case QUERY_NODE_SHOW_DB_ALIVE_STMT: + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: case QUERY_NODE_SHOW_VNODES_STMT: diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 358baa74cb3932887894bda79a6e26e9a888c797..4742921d083257b56d7095d0e8aca8fe705900c2 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -18,6 +18,16 @@ #include "parToken.h" #include "ttime.h" +static void clearColValArray(SArray* pCols) { + int32_t num = taosArrayGetSize(pCols); + for (int32_t i = 0; i < num; ++i) { + SColVal* pCol = taosArrayGet(pCols, i); + if (TSDB_DATA_TYPE_NCHAR == pCol->type) { + taosMemoryFreeClear(pCol->value.pData); + } + } +} + int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf, int32_t msgBufLen) { SMsgBuf msg = {.buf = msgBuf, .len = msgBufLen}; @@ -45,95 +55,46 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* return TSDB_CODE_SUCCESS; } -typedef struct SmlExecTableHandle { - SParsedDataColInfo tags; // each table - SVCreateTbReq createTblReq; // each table -} SmlExecTableHandle; - -typedef struct SmlExecHandle { - SHashObj* pBlockHash; - SmlExecTableHandle tableExecHandle; - SQuery* pQuery; -} SSmlExecHandle; - -static void smlDestroyTableHandle(void* pHandle) { - SmlExecTableHandle* handle = (SmlExecTableHandle*)pHandle; - destroyBoundColumnInfo(&handle->tags); - tdDestroySVCreateTbReq(&handle->createTblReq); -} - -static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SSchema* pSchema, bool isTag) { - col_id_t nCols = pColList->numOfCols; - - pColList->numOfBound = 0; - pColList->boundNullLen = 0; - memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols); - for (col_id_t i = 0; i < nCols; ++i) { - pColList->cols[i].valStat = VAL_STAT_NONE; +static int32_t smlBoundColumnData(SArray* cols, SBoundColInfo* pBoundInfo, SSchema* pSchema, bool isTag) { + bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool)); + if (NULL == pUseCols) { + return TSDB_CODE_OUT_OF_MEMORY; } - bool isOrdered = true; - col_id_t lastColIdx = -1; // last column found + pBoundInfo->numOfBound = 0; + int16_t lastColIdx = -1; // last column found + int32_t code = TSDB_CODE_SUCCESS; + for (int i = 0; i < taosArrayGetSize(cols); ++i) { - SSmlKv* kv = taosArrayGetP(cols, i); + SSmlKv* kv = taosArrayGet(cols, i); SToken sToken = {.n = kv->keyLen, .z = (char*)kv->key}; col_id_t t = lastColIdx + 1; - col_id_t index = ((t == 0 && !isTag) ? 0 : insFindCol(&sToken, t, nCols, pSchema)); - uDebug("SML, index:%d, t:%d, ncols:%d", index, t, nCols); + col_id_t index = ((t == 0 && !isTag) ? 0 : insFindCol(&sToken, t, pBoundInfo->numOfCols, pSchema)); + uDebug("SML, index:%d, t:%d, ncols:%d", index, t, pBoundInfo->numOfCols); if (index < 0 && t > 0) { index = insFindCol(&sToken, 0, t, pSchema); - isOrdered = false; } + if (index < 0) { uError("smlBoundColumnData. index:%d", index); - return TSDB_CODE_SML_INVALID_DATA; + code = TSDB_CODE_SML_INVALID_DATA; + goto end; } - if (pColList->cols[index].valStat == VAL_STAT_HAS) { + if (pUseCols[index]) { uError("smlBoundColumnData. already set. index:%d", index); - return TSDB_CODE_SML_INVALID_DATA; + code = TSDB_CODE_SML_INVALID_DATA; + goto end; } lastColIdx = index; - pColList->cols[index].valStat = VAL_STAT_HAS; - pColList->boundColumns[pColList->numOfBound] = index; - ++pColList->numOfBound; - switch (pSchema[t].type) { - case TSDB_DATA_TYPE_BINARY: - pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + CHAR_BYTES); - break; - case TSDB_DATA_TYPE_NCHAR: - pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); - break; - default: - pColList->boundNullLen += TYPE_BYTES[pSchema[t].type]; - break; - } + pUseCols[index] = true; + pBoundInfo->pColIndex[pBoundInfo->numOfBound] = index; + ++pBoundInfo->numOfBound; } - pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; - - if (!isOrdered) { - pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); - if (NULL == pColList->colIdxInfo) { - return TSDB_CODE_OUT_OF_MEMORY; - } - SBoundIdxInfo* pColIdx = pColList->colIdxInfo; - for (col_id_t i = 0; i < pColList->numOfBound; ++i) { - pColIdx[i].schemaColIdx = pColList->boundColumns[i]; - pColIdx[i].boundIdx = i; - } - taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insSchemaIdxCompar); - for (col_id_t i = 0; i < pColList->numOfBound; ++i) { - pColIdx[i].finalIdx = i; - } - taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insBoundIdxCompar); - } - - if (pColList->numOfCols > pColList->numOfBound) { - memset(&pColList->boundColumns[pColList->numOfBound], 0, - sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); - } +end: + taosMemoryFree(pUseCols); - return TSDB_CODE_SUCCESS; + return code; } /** @@ -146,7 +107,7 @@ static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SS * @param msg * @return int32_t */ -static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* pSchema, STag** ppTag, SArray** tagName, +static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchema, STag** ppTag, SArray** tagName, SMsgBuf* msg) { SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal)); if (!pTagArray) { @@ -159,8 +120,8 @@ static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* p int32_t code = TSDB_CODE_SUCCESS; for (int i = 0; i < tags->numOfBound; ++i) { - SSchema* pTagSchema = &pSchema[tags->boundColumns[i]]; - SSmlKv* kv = taosArrayGetP(cols, i); + SSchema* pTagSchema = &pSchema[tags->pColIndex[i]]; + SSmlKv* kv = taosArrayGet(cols, i); taosArrayPush(*tagName, pTagSchema->name); STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; @@ -207,153 +168,253 @@ end: return code; } -int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta, - char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int16_t msgBufLen) { - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; +STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) { + STableDataCxt* pTableCxt = NULL; + SVCreateTbReq* pCreateTbReq = NULL; + int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, + sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false); + if (ret != TSDB_CODE_SUCCESS) { + return NULL; + } - SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle; - smlDestroyTableHandle(&smlHandle->tableExecHandle); // free for each table - SSchema* pTagsSchema = getTableTagSchema(pTableMeta); - insSetBoundColumnInfo(&smlHandle->tableExecHandle.tags, pTagsSchema, getNumOfTags(pTableMeta)); - int ret = smlBoundColumnData(tags, &smlHandle->tableExecHandle.tags, pTagsSchema, true); + ret = initTableColSubmitData(pTableCxt); if (ret != TSDB_CODE_SUCCESS) { - buildInvalidOperationMsg(&pBuf, "bound tags error"); + return NULL; + } + return pTableCxt; +} + +int32_t smlBuildRow(STableDataCxt* pTableCxt) { + SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + int ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); + if (TSDB_CODE_SUCCESS != ret) { return ret; } - STag* pTag = NULL; - SArray* tagName = NULL; - ret = smlBuildTagRow(tags, &smlHandle->tableExecHandle.tags, pTagsSchema, &pTag, &tagName, &pBuf); + insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow)); + return TSDB_CODE_SUCCESS; +} + +int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32_t index) { + int ret = TSDB_CODE_SUCCESS; + SSchema* pColSchema = schema + index; + SColVal* pVal = taosArrayGet(pTableCxt->pValues, index); + SSmlKv* kv = (SSmlKv*)data; + if(kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || kv->type != pColSchema->type){ + ret = TSDB_CODE_SML_INVALID_DATA; + goto end; + } + if (kv->type == TSDB_DATA_TYPE_NCHAR) { + int32_t len = 0; + int64_t size = pColSchema->bytes - VARSTR_HEADER_SIZE; + if(size <= 0){ + ret = TSDB_CODE_SML_INVALID_DATA; + goto end; + } + char* pUcs4 = taosMemoryCalloc(1, size); + if (NULL == pUcs4) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, size, &len)) { + if (errno == E2BIG) { + taosMemoryFree(pUcs4); + ret = TSDB_CODE_PAR_VALUE_TOO_LONG; + goto end; + } + taosMemoryFree(pUcs4); + ret = TSDB_CODE_TSC_INVALID_VALUE; + goto end; + } + pVal->value.pData = pUcs4; + pVal->value.nData = len; + } else if (kv->type == TSDB_DATA_TYPE_BINARY) { + pVal->value.nData = kv->length; + pVal->value.pData = (uint8_t*)kv->value; + } else { + memcpy(&pVal->value.val, &(kv->value), kv->length); + } + pVal->flag = CV_FLAG_VALUE; + +end: + return ret; +} + +int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, + STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, + char* msgBuf, int16_t msgBufLen) { + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + + SSchema* pTagsSchema = getTableTagSchema(pTableMeta); + SBoundColInfo bindTags = {0}; + SVCreateTbReq* pCreateTblReq = NULL; + SArray* tagName = NULL; + + insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags); + int ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true); if (ret != TSDB_CODE_SUCCESS) { - taosArrayDestroy(tagName); - return ret; + buildInvalidOperationMsg(&pBuf, "bound tags error"); + goto end; } - insBuildCreateTbReq(&smlHandle->tableExecHandle.createTblReq, tableName, pTag, pTableMeta->suid, NULL, tagName, - pTableMeta->tableInfo.numOfTags, ttl); - taosArrayDestroy(tagName); + STag* pTag = NULL; + + ret = smlBuildTagRow(tags, &bindTags, pTagsSchema, &pTag, &tagName, &pBuf); + if (ret != TSDB_CODE_SUCCESS) { + goto end; + } - smlHandle->tableExecHandle.createTblReq.ctb.stbName = taosMemoryMalloc(sTableNameLen + 1); - memcpy(smlHandle->tableExecHandle.createTblReq.ctb.stbName, sTableName, sTableNameLen); - smlHandle->tableExecHandle.createTblReq.ctb.stbName[sTableNameLen] = 0; + pCreateTblReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == pCreateTblReq) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + insBuildCreateTbReq(pCreateTblReq, tableName, pTag, pTableMeta->suid, NULL, tagName, pTableMeta->tableInfo.numOfTags, + ttl); + + pCreateTblReq->ctb.stbName = taosMemoryCalloc(1, sTableNameLen + 1); + memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen); + + if (dataFormat) { + STableDataCxt** pTableCxt = (STableDataCxt**)taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, + &pTableMeta->uid, sizeof(pTableMeta->uid)); + if (NULL == pTableCxt) { + ret = buildInvalidOperationMsg(&pBuf, "dataformat true. get tableDataCtx error"); + goto end; + } + (*pTableCxt)->pData->flags |= SUBMIT_REQ_AUTO_CREATE_TABLE; + (*pTableCxt)->pData->pCreateTbReq = pCreateTblReq; + (*pTableCxt)->pMeta->uid = pTableMeta->uid; + (*pTableCxt)->pMeta->vgId = pTableMeta->vgId; + pCreateTblReq = NULL; + goto end; + } - STableDataBlocks* pDataBlock = NULL; - ret = insGetDataBlockFromList(smlHandle->pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid), - TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize, - pTableMeta, &pDataBlock, NULL, &smlHandle->tableExecHandle.createTblReq); + STableDataCxt* pTableCxt = NULL; + ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, + sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false); if (ret != TSDB_CODE_SUCCESS) { - buildInvalidOperationMsg(&pBuf, "create data block error"); - return ret; + buildInvalidOperationMsg(&pBuf, "insGetTableDataCxt error"); + goto end; } SSchema* pSchema = getTableColumnSchema(pTableMeta); - - ret = smlBoundColumnData(colsSchema, &pDataBlock->boundColumnInfo, pSchema, false); + ret = smlBoundColumnData(colsSchema, &pTableCxt->boundColsInfo, pSchema, false); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "bound cols error"); - return ret; + goto end; } - int32_t extendedRowSize = insGetExtendedRowSize(pDataBlock); - SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; - SRowBuilder* pBuilder = &pDataBlock->rowBuilder; - SMemParam param = {.rb = pBuilder}; - insInitRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo); + ret = initTableColSubmitData(pTableCxt); + if (ret != TSDB_CODE_SUCCESS) { + buildInvalidOperationMsg(&pBuf, "initTableColSubmitData error"); + goto end; + } int32_t rowNum = taosArrayGetSize(cols); if (rowNum <= 0) { - return buildInvalidOperationMsg(&pBuf, "cols size <= 0"); - } - ret = insAllocateMemForSize(pDataBlock, extendedRowSize * rowNum); - if (ret != TSDB_CODE_SUCCESS) { - buildInvalidOperationMsg(&pBuf, "allocate memory error"); - return ret; + ret = buildInvalidOperationMsg(&pBuf, "cols size <= 0"); + goto end; } + for (int32_t r = 0; r < rowNum; ++r) { - STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header - tdSRowResetBuf(pBuilder, row); - void* rowData = taosArrayGetP(cols, r); - size_t rowDataSize = 0; - if (format) { - rowDataSize = taosArrayGetSize(rowData); - } + void* rowData = taosArrayGetP(cols, r); // 1. set the parsed value from sql string - for (int c = 0, j = 0; c < spd->numOfBound; ++c) { - SSchema* pColSchema = &pSchema[spd->boundColumns[c]]; - - param.schema = pColSchema; - insGetSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); - - SSmlKv* kv = NULL; - if (format) { - if (j < rowDataSize) { - kv = taosArrayGetP(rowData, j); - if (rowDataSize != spd->numOfBound && j != 0 && - (kv->keyLen != strlen(pColSchema->name) || strncmp(kv->key, pColSchema->name, kv->keyLen) != 0)) { - kv = NULL; - } else { - j++; - } - } - } else { - void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); - if (p) kv = *p; + for (int c = 0; c < pTableCxt->boundColsInfo.numOfBound; ++c) { + SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]]; + SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]); + void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); + if (p == NULL) { + continue; } - - if (kv) { - int32_t colLen = kv->length; - if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { - uDebug("SML:data before:%" PRId64 ", precision:%d", kv->i, pTableMeta->tableInfo.precision); - kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision); - uDebug("SML:data after:%" PRId64 ", precision:%d", kv->i, pTableMeta->tableInfo.precision); + SSmlKv* kv = *(SSmlKv**)p; + if(kv->type != pColSchema->type){ + ret = buildInvalidOperationMsg(&pBuf, "kv type not equal to col type"); + goto end; + } + if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { + kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision); + } + if (kv->type == TSDB_DATA_TYPE_NCHAR) { + int32_t len = 0; + char* pUcs4 = taosMemoryCalloc(1, pColSchema->bytes - VARSTR_HEADER_SIZE); + if (NULL == pUcs4) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; } - - if (IS_VAR_DATA_TYPE(kv->type)) { - insMemRowAppend(&pBuf, kv->value, colLen, ¶m); - } else { - insMemRowAppend(&pBuf, &(kv->value), colLen, ¶m); + if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len)) { + if (errno == E2BIG) { + buildInvalidOperationMsg(&pBuf, "value too long"); + ret = TSDB_CODE_PAR_VALUE_TOO_LONG; + goto end; + } + ret = buildInvalidOperationMsg(&pBuf, strerror(errno)); + goto end; } + pVal->value.pData = pUcs4; + pVal->value.nData = len; + } else if (kv->type == TSDB_DATA_TYPE_BINARY) { + pVal->value.nData = kv->length; + pVal->value.pData = (uint8_t*)kv->value; } else { - pBuilder->hasNone = true; - } - - if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { - TSKEY tsKey = TD_ROW_KEY(row); - insCheckTimestamp(pDataBlock, (const char*)&tsKey); + memcpy(&pVal->value.val, &(kv->value), kv->length); } + pVal->flag = CV_FLAG_VALUE; } - // set the null value for the columns that do not assign values - if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - pBuilder->hasNone = true; + SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); + if (TSDB_CODE_SUCCESS != ret) { + buildInvalidOperationMsg(&pBuf, "tRowBuild error"); + goto end; } - - tdSRowEnd(pBuilder); - pDataBlock->size += extendedRowSize; + insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow)); + clearColValArray(pTableCxt->pValues); } - SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); - return insSetBlockInfo(pBlocks, pDataBlock, rowNum, &pBuf); +end: + insDestroyBoundColInfo(&bindTags); + tdDestroySVCreateTbReq(pCreateTblReq); + taosMemoryFree(pCreateTblReq); + taosArrayDestroy(tagName); + return ret; } -void* smlInitHandle(SQuery* pQuery) { - SSmlExecHandle* handle = taosMemoryCalloc(1, sizeof(SSmlExecHandle)); - if (!handle) return NULL; - handle->pBlockHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - handle->pQuery = pQuery; - - return handle; -} +SQuery* smlInitHandle() { + SQuery* pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); + if (NULL == pQuery) { + uError("create pQuery error"); + return NULL; + } + pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + pQuery->haveResultSet = false; + pQuery->msgType = TDMT_VND_SUBMIT; + SVnodeModifyOpStmt* stmt = (SVnodeModifyOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT); + if (NULL == stmt) { + uError("create SVnodeModifyOpStmt error"); + qDestroyQuery(pQuery); + return NULL; + } + stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + stmt->freeHashFunc = insDestroyTableDataCxtHashMap; + stmt->freeArrayFunc = insDestroyVgroupDataCxtList; -void smlDestroyHandle(void* pHandle) { - if (!pHandle) return; - SSmlExecHandle* handle = (SSmlExecHandle*)pHandle; - insDestroyBlockHashmap(handle->pBlockHash); - smlDestroyTableHandle(&handle->tableExecHandle); - taosMemoryFree(handle); + pQuery->pRoot = (SNode*)stmt; + return pQuery; } -int32_t smlBuildOutput(void* handle, SHashObj* pVgHash) { - SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle; - return qBuildStmtOutput(smlHandle->pQuery, pVgHash, smlHandle->pBlockHash); +int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) { + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)(handle)->pRoot; + // merge according to vgId + int32_t code = insMergeTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks); + if (code != TSDB_CODE_SUCCESS) { + uError("insMergeTableDataCxt failed"); + return code; + } + code = insBuildVgDataBlocks(pVgHash, pStmt->pVgDataBlocks, &pStmt->pDataBlocks); + if (code != TSDB_CODE_SUCCESS) { + uError("insBuildVgDataBlocks failed"); + return code; + } + return code; } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index f7415b9ba83a642f9fa579abdd406ade01c683f8..3c5d4d506a72d9a9e3d8bda482cbf1010c8acb3f 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -38,13 +38,13 @@ } while (TK_NK_SPACE == (token).type) typedef struct SInsertParseContext { - SParseContext* pComCxt; - SMsgBuf msg; - char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW]; - SParsedDataColInfo tags; // for stmt - bool missCache; - bool usingDuplicateTable; - bool forceUpdate; + SParseContext* pComCxt; + SMsgBuf msg; + char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW]; + SBoundColInfo tags; // for stmt + bool missCache; + bool usingDuplicateTable; + bool forceUpdate; } SInsertParseContext; typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param); @@ -173,21 +173,19 @@ static int32_t parseDuplicateUsingClause(SInsertParseContext* pCxt, SVnodeModify } // pStmt->pSql -> field1_name, ...) -static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, bool isTags, - SParsedDataColInfo* pColList, SSchema* pSchema) { - col_id_t nCols = pColList->numOfCols; - - pColList->numOfBound = 0; - pColList->boundNullLen = 0; - memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols); - for (col_id_t i = 0; i < nCols; ++i) { - pColList->cols[i].valStat = VAL_STAT_NONE; +static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, bool isTags, SSchema* pSchema, + SBoundColInfo* pBoundInfo) { + bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool)); + if (NULL == pUseCols) { + return TSDB_CODE_OUT_OF_MEMORY; } - SToken token; - bool isOrdered = true; - col_id_t lastColIdx = -1; // last column found - while (1) { + pBoundInfo->numOfBound = 0; + + int16_t lastColIdx = -1; // last column found + int32_t code = TSDB_CODE_SUCCESS; + while (TSDB_CODE_SUCCESS == code) { + SToken token; NEXT_TOKEN(*pSql, token); if (TK_NK_RP == token.type) { @@ -199,64 +197,30 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, b token.z = tmpTokenBuf; token.n = strdequote(token.z); - col_id_t t = lastColIdx + 1; - col_id_t index = insFindCol(&token, t, nCols, pSchema); + int16_t t = lastColIdx + 1; + int16_t index = insFindCol(&token, t, pBoundInfo->numOfCols, pSchema); if (index < 0 && t > 0) { index = insFindCol(&token, 0, t, pSchema); - isOrdered = false; } if (index < 0) { - return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMN, token.z); - } - if (pColList->cols[index].valStat == VAL_STAT_HAS) { - return buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", token.z); - } - lastColIdx = index; - pColList->cols[index].valStat = VAL_STAT_HAS; - pColList->boundColumns[pColList->numOfBound] = index; - ++pColList->numOfBound; - switch (pSchema[t].type) { - case TSDB_DATA_TYPE_BINARY: - pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + CHAR_BYTES); - break; - case TSDB_DATA_TYPE_NCHAR: - pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); - break; - default: - pColList->boundNullLen += TYPE_BYTES[pSchema[t].type]; - break; + code = generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMN, token.z); + } else if (pUseCols[index]) { + code = buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", token.z); + } else { + lastColIdx = index; + pUseCols[index] = true; + pBoundInfo->pColIndex[pBoundInfo->numOfBound] = index; + ++pBoundInfo->numOfBound; } } - if (!isTags && pColList->cols[0].valStat == VAL_STAT_NONE) { - return buildInvalidOperationMsg(&pCxt->msg, "primary timestamp column can not be null"); + if (TSDB_CODE_SUCCESS == code && !isTags && !pUseCols[0]) { + code = buildInvalidOperationMsg(&pCxt->msg, "primary timestamp column can not be null"); } - pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; - - if (!isOrdered) { - pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); - if (NULL == pColList->colIdxInfo) { - return TSDB_CODE_OUT_OF_MEMORY; - } - SBoundIdxInfo* pColIdx = pColList->colIdxInfo; - for (col_id_t i = 0; i < pColList->numOfBound; ++i) { - pColIdx[i].schemaColIdx = pColList->boundColumns[i]; - pColIdx[i].boundIdx = i; - } - taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insSchemaIdxCompar); - for (col_id_t i = 0; i < pColList->numOfBound; ++i) { - pColIdx[i].finalIdx = i; - } - taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insBoundIdxCompar); - } - - if (pColList->numOfCols > pColList->numOfBound) { - memset(&pColList->boundColumns[pColList->numOfBound], 0, - sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); - } + taosMemoryFree(pUseCols); - return TSDB_CODE_SUCCESS; + return code; } static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t* time, SMsgBuf* pMsgBuf) { @@ -519,8 +483,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, // input pStmt->pSql: [(tag1_name, ...)] TAGS (tag1_value, ...) ... // output pStmt->pSql: TAGS (tag1_value, ...) ... static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - SSchema* pTagsSchema = getTableTagSchema(pStmt->pTableMeta); - insSetBoundColumnInfo(&pCxt->tags, pTagsSchema, getNumOfTags(pStmt->pTableMeta)); + insInitBoundColsInfo(getNumOfTags(pStmt->pTableMeta), &pCxt->tags); SToken token; int32_t index = 0; @@ -530,7 +493,7 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStm } pStmt->pSql += index; - return parseBoundColumns(pCxt, &pStmt->pSql, true, &pCxt->tags, pTagsSchema); + return parseBoundColumns(pCxt, &pStmt->pSql, true, getTableTagSchema(pStmt->pTableMeta), &pCxt->tags); } static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SSchema* pTagSchema, SToken* pToken, @@ -561,10 +524,15 @@ static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStm return code; } -static void buildCreateTbReq(SVnodeModifyOpStmt* pStmt, STag* pTag, SArray* pTagName) { - insBuildCreateTbReq(&pStmt->createTblReq, pStmt->targetTableName.tname, pTag, pStmt->pTableMeta->suid, +static int32_t buildCreateTbReq(SVnodeModifyOpStmt* pStmt, STag* pTag, SArray* pTagName) { + pStmt->pCreateTblReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == pStmt->pCreateTblReq) { + return TSDB_CODE_OUT_OF_MEMORY; + } + insBuildCreateTbReq(pStmt->pCreateTblReq, pStmt->targetTableName.tname, pTag, pStmt->pTableMeta->suid, pStmt->usingTableName.tname, pTagName, pStmt->pTableMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); + return TSDB_CODE_SUCCESS; } static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf) { @@ -618,7 +586,7 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt break; } - SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i]]; + SSchema* pTagSchema = &pSchema[pCxt->tags.pColIndex[i]]; isJson = pTagSchema->type == TSDB_DATA_TYPE_JSON; code = checkAndTrimValue(&token, pCxt->tmpTokenBuf, &pCxt->msg); if (TSDB_CODE_SUCCESS == code) { @@ -631,7 +599,7 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt } if (TSDB_CODE_SUCCESS == code && !isParseBindParam) { - buildCreateTbReq(pStmt, pTag, pTagName); + code = buildCreateTbReq(pStmt, pTag, pTagName); pTag = NULL; } @@ -699,8 +667,8 @@ static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifyOpStmt* if (TK_NK_INTEGER != token.type) { return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", token.z); } - pStmt->createTblReq.ttl = taosStr2Int32(token.z, NULL, 10); - if (pStmt->createTblReq.ttl < 0) { + pStmt->pCreateTblReq->ttl = taosStr2Int32(token.z, NULL, 10); + if (pStmt->pCreateTblReq->ttl < 0) { return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", token.z); } } else if (TK_COMMENT == token.type) { @@ -713,11 +681,11 @@ static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifyOpStmt* return buildSyntaxErrMsg(&pCxt->msg, "comment too long", token.z); } int32_t len = trimString(token.z, token.n, pCxt->tmpTokenBuf, TSDB_TB_COMMENT_LEN); - pStmt->createTblReq.comment = strndup(pCxt->tmpTokenBuf, len); - if (NULL == pStmt->createTblReq.comment) { + pStmt->pCreateTblReq->comment = strndup(pCxt->tmpTokenBuf, len); + if (NULL == pStmt->pCreateTblReq->comment) { return TSDB_CODE_OUT_OF_MEMORY; } - pStmt->createTblReq.commentLen = len; + pStmt->pCreateTblReq->commentLen = len; } else { break; } @@ -975,26 +943,22 @@ static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModif return skipParentheses(pCxt, &pStmt->pSql); } -static int32_t getTableDataBlocks(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataBlocks** pDataBuf) { +static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt** pTableCxt) { if (pCxt->pComCxt->async) { - uint64_t uid = pStmt->pTableMeta->uid; - if (pStmt->usingTableProcessing) { - pStmt->pTableMeta->uid = 0; - } - - return insGetDataBlockFromList( - pStmt->pTableBlockHashObj, &uid, sizeof(pStmt->pTableMeta->uid), TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), - getTableInfo(pStmt->pTableMeta).rowSize, pStmt->pTableMeta, pDataBuf, NULL, &pStmt->createTblReq); + return insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), + pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false); } + char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(&pStmt->targetTableName, tbFName); - return insGetDataBlockFromList(pStmt->pTableBlockHashObj, tbFName, strlen(tbFName), TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), getTableInfo(pStmt->pTableMeta).rowSize, pStmt->pTableMeta, - pDataBuf, NULL, &pStmt->createTblReq); + if (pStmt->usingTableProcessing) { + pStmt->pTableMeta->uid = 0; + } + return insGetTableDataCxt(pStmt->pTableBlockHashObj, tbFName, strlen(tbFName), pStmt->pTableMeta, + &pStmt->pCreateTblReq, pTableCxt, NULL != pCxt->pComCxt->pStmtCb); } -static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, - STableDataBlocks* pDataBuf) { +static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { SToken token; int32_t index = 0; NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index); @@ -1004,13 +968,30 @@ static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOp return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } // pStmt->pSql -> field1_name, ...) - return parseBoundColumns(pCxt, &pStmt->pSql, false, &pDataBuf->boundColumnInfo, - getTableColumnSchema(pStmt->pTableMeta)); + return parseBoundColumns(pCxt, &pStmt->pSql, false, getTableColumnSchema(pStmt->pTableMeta), + &pTableCxt->boundColsInfo); } if (NULL != pStmt->pBoundCols) { - return parseBoundColumns(pCxt, &pStmt->pBoundCols, false, &pDataBuf->boundColumnInfo, - getTableColumnSchema(pStmt->pTableMeta)); + return parseBoundColumns(pCxt, &pStmt->pBoundCols, false, getTableColumnSchema(pStmt->pTableMeta), + &pTableCxt->boundColsInfo); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t initTableColSubmitData(STableDataCxt* pTableCxt) { + if (0 == (pTableCxt->pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT)) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < pTableCxt->boundColsInfo.numOfBound; ++i) { + SSchema* pSchema = &pTableCxt->pMeta->schema[pTableCxt->boundColsInfo.pColIndex[i]]; + SColData* pCol = taosArrayReserve(pTableCxt->pData->aCol, 1); + if (NULL == pCol) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tColDataInit(pCol, pSchema->colId, pSchema->type, 0); } return TSDB_CODE_SUCCESS; @@ -1021,13 +1002,16 @@ static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOp // 2. VALUES ... | FILE ... // output pStmt->pSql: VALUES ... | FILE ... static int32_t parseSchemaClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, - STableDataBlocks** pDataBuf) { + STableDataCxt** pTableCxt) { int32_t code = parseUsingClauseBottom(pCxt, pStmt); if (TSDB_CODE_SUCCESS == code) { - code = getTableDataBlocks(pCxt, pStmt, pDataBuf); + code = getTableDataCxt(pCxt, pStmt, pTableCxt); } if (TSDB_CODE_SUCCESS == code) { - code = parseBoundColumnsClause(pCxt, pStmt, *pDataBuf); + code = parseBoundColumnsClause(pCxt, pStmt, *pTableCxt); + } + if (TSDB_CODE_SUCCESS == code) { + code = initTableColSubmitData(*pTableCxt); } return code; } @@ -1050,108 +1034,88 @@ static int32_t parseSchemaClauseTop(SInsertParseContext* pCxt, SVnodeModifyOpStm } static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, SToken* pToken, SSchema* pSchema, - int16_t timePrec, _row_append_fn_t func, void* param) { - int64_t iv; - uint64_t uv; - char* endptr = NULL; - + int16_t timePrec, SColVal* pVal) { switch (pSchema->type) { case TSDB_DATA_TYPE_BOOL: { if ((pToken->type == TK_NK_BOOL || pToken->type == TK_NK_STRING) && (pToken->n != 0)) { if (strncmp(pToken->z, "true", pToken->n) == 0) { - return func(&pCxt->msg, &TRUE_VALUE, pSchema->bytes, param); + pVal->value.val = TRUE_VALUE; } else if (strncmp(pToken->z, "false", pToken->n) == 0) { - return func(&pCxt->msg, &FALSE_VALUE, pSchema->bytes, param); + pVal->value.val = FALSE_VALUE; } else { return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); } } else if (pToken->type == TK_NK_INTEGER) { - return func(&pCxt->msg, ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), - pSchema->bytes, param); + pVal->value.val = ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? FALSE_VALUE : TRUE_VALUE); } else if (pToken->type == TK_NK_FLOAT) { - return func(&pCxt->msg, ((taosStr2Double(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, - param); + pVal->value.val = ((taosStr2Double(pToken->z, NULL) == 0) ? FALSE_VALUE : TRUE_VALUE); } else { return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); } + break; } - case TSDB_DATA_TYPE_TINYINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z); - } else if (!IS_VALID_TINYINT(iv)) { + } else if (!IS_VALID_TINYINT(pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "tinyint data overflow", pToken->z); } - - uint8_t tmpVal = (uint8_t)iv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_UTINYINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z); - } else if (uv > UINT8_MAX) { + } else if (pVal->value.val > UINT8_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned tinyint data overflow", pToken->z); } - uint8_t tmpVal = (uint8_t)uv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_SMALLINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z); - } else if (!IS_VALID_SMALLINT(iv)) { + } else if (!IS_VALID_SMALLINT(pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "smallint data overflow", pToken->z); } - int16_t tmpVal = (int16_t)iv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_USMALLINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z); - } else if (uv > UINT16_MAX) { + } else if (pVal->value.val > UINT16_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned smallint data overflow", pToken->z); } - uint16_t tmpVal = (uint16_t)uv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_INT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z); - } else if (!IS_VALID_INT(iv)) { + } else if (!IS_VALID_INT(pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "int data overflow", pToken->z); } - int32_t tmpVal = (int32_t)iv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_UINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); - } else if (uv > UINT32_MAX) { + } else if (pVal->value.val > UINT32_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned int data overflow", pToken->z); } - uint32_t tmpVal = (uint32_t)uv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_BIGINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z); } - return func(&pCxt->msg, &iv, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_UBIGINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z); } - return func(&pCxt->msg, &uv, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_FLOAT: { + char* endptr = NULL; double dv; if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); @@ -1160,11 +1124,12 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, isnan(dv)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } - float tmpVal = (float)dv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + float f = dv; + memcpy(&pVal->value.val, &f, sizeof(f)); + break; } - case TSDB_DATA_TYPE_DOUBLE: { + char* endptr = NULL; double dv; if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); @@ -1172,49 +1137,77 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); } - return func(&pCxt->msg, &dv, pSchema->bytes, param); + pVal->value.val = *(int64_t*)&dv; + break; } - case TSDB_DATA_TYPE_BINARY: { // Too long values will raise the invalid sql error message if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } - - return func(&pCxt->msg, pToken->z, pToken->n, param); + pVal->value.pData = taosMemoryMalloc(pToken->n); + if (NULL == pVal->value.pData) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(pVal->value.pData, pToken->z, pToken->n); + pVal->value.nData = pToken->n; + break; } - case TSDB_DATA_TYPE_NCHAR: { - return func(&pCxt->msg, pToken->z, pToken->n, param); + // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' + int32_t len = 0; + char* pUcs4 = taosMemoryCalloc(1, pSchema->bytes - VARSTR_HEADER_SIZE); + if (NULL == pUcs4) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)pUcs4, pSchema->bytes - VARSTR_HEADER_SIZE, &len)) { + taosMemoryFree(pUcs4); + if (errno == E2BIG) { + return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + } + char buf[512] = {0}; + snprintf(buf, tListLen(buf), "%s", strerror(errno)); + return buildSyntaxErrMsg(&pCxt->msg, buf, pToken->z); + } + pVal->value.pData = pUcs4; + pVal->value.nData = len; + break; } case TSDB_DATA_TYPE_JSON: { if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { return buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", pToken->z); } - return func(&pCxt->msg, pToken->z, pToken->n, param); + pVal->value.pData = taosMemoryMalloc(pToken->n); + if (NULL == pVal->value.pData) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(pVal->value.pData, pToken->z, pToken->n); + pVal->value.nData = pToken->n; + break; } case TSDB_DATA_TYPE_TIMESTAMP: { - int64_t tmpVal; - if (parseTime(pSql, pToken, timePrec, &tmpVal, &pCxt->msg) != TSDB_CODE_SUCCESS) { + if (parseTime(pSql, pToken, timePrec, &pVal->value.val, &pCxt->msg) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp", pToken->z); } - - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } + default: + return TSDB_CODE_FAILED; } - return TSDB_CODE_FAILED; + pVal->flag = CV_FLAG_VALUE; + return TSDB_CODE_SUCCESS; } static int32_t parseValueToken(SInsertParseContext* pCxt, const char** pSql, SToken* pToken, SSchema* pSchema, - int16_t timePrec, _row_append_fn_t func, void* param) { + int16_t timePrec, SColVal* pVal) { int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg); if (TSDB_CODE_SUCCESS == code && isNullValue(pSchema->type, pToken)) { if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { return buildSyntaxErrMsg(&pCxt->msg, "primary timestamp should not be null", pToken->z); } - - return func(&pCxt->msg, NULL, 0, param); + pVal->flag = CV_FLAG_NULL; + return TSDB_CODE_SUCCESS; } if (TSDB_CODE_SUCCESS == code && IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { @@ -1222,26 +1215,34 @@ static int32_t parseValueToken(SInsertParseContext* pCxt, const char** pSql, STo } if (TSDB_CODE_SUCCESS == code) { - code = parseValueTokenImpl(pCxt, pSql, pToken, pSchema, timePrec, func, param); + code = parseValueTokenImpl(pCxt, pSql, pToken, pSchema, timePrec, pVal); } return code; } -static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataBlocks* pDataBuf, bool* pGotRow, +static void clearColValArray(SArray* pCols) { + int32_t num = taosArrayGetSize(pCols); + for (int32_t i = 0; i < num; ++i) { + SColVal* pCol = taosArrayGet(pCols, i); + if (IS_VAR_DATA_TYPE(pCol->type)) { + taosMemoryFreeClear(pCol->value.pData); + } + } +} + +static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataCxt* pTableCxt, bool* pGotRow, SToken* pToken) { - SRowBuilder* pBuilder = &pDataBuf->rowBuilder; - STSRow* row = (STSRow*)(pDataBuf->pData + pDataBuf->size); // skip the SSubmitBlk header - SParsedDataColInfo* pCols = &pDataBuf->boundColumnInfo; - bool isParseBindParam = false; - SSchema* pSchemas = getTableColumnSchema(pDataBuf->pTableMeta); - SMemParam param = {.rb = pBuilder}; - - int32_t code = tdSRowResetBuf(pBuilder, row); + SBoundColInfo* pCols = &pTableCxt->boundColsInfo; + bool isParseBindParam = false; + SSchema* pSchemas = getTableColumnSchema(pTableCxt->pMeta); + + int32_t code = TSDB_CODE_SUCCESS; // 1. set the parsed value from sql string for (int i = 0; i < pCols->numOfBound && TSDB_CODE_SUCCESS == code; ++i) { NEXT_TOKEN_WITH_PREV(*pSql, *pToken); - SSchema* pSchema = &pSchemas[pCols->boundColumns[i]]; + SSchema* pSchema = &pSchemas[pCols->pColIndex[i]]; + SColVal* pVal = taosArrayGet(pTableCxt->pValues, pCols->pColIndex[i]); if (pToken->type == TK_NK_QUESTION) { isParseBindParam = true; @@ -1260,10 +1261,7 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataB } if (TSDB_CODE_SUCCESS == code) { - param.schema = pSchema; - insGetSTSRowAppendInfo(pBuilder->rowType, pCols, i, ¶m.toffset, ¶m.colIdx); - code = parseValueToken(pCxt, pSql, pToken, pSchema, getTableInfo(pDataBuf->pTableMeta).precision, insMemRowAppend, - ¶m); + code = parseValueToken(pCxt, pSql, pToken, pSchema, getTableInfo(pTableCxt->pMeta).precision, pVal); } if (TSDB_CODE_SUCCESS == code && i < pCols->numOfBound - 1) { @@ -1274,65 +1272,28 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataB } } - if (TSDB_CODE_SUCCESS == code) { - TSKEY tsKey = TD_ROW_KEY(row); - code = insCheckTimestamp(pDataBuf, (const char*)&tsKey); - } - if (TSDB_CODE_SUCCESS == code && !isParseBindParam) { - // set the null value for the columns that do not assign values - if ((pCols->numOfBound < pCols->numOfCols) && TD_IS_TP_ROW(row)) { - pBuilder->hasNone = true; + SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + code = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); + if (TSDB_CODE_SUCCESS == code) { + insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow)); } + } - tdSRowEnd(pBuilder); - + if (TSDB_CODE_SUCCESS == code && !isParseBindParam) { *pGotRow = true; - -#ifdef TD_DEBUG_PRINT_ROW - STSchema* pSTSchema = tdGetSTSChemaFromSSChema(schema, spd->numOfCols, 1); - tdSRowPrint(row, pSTSchema, __func__); - taosMemoryFree(pSTSchema); -#endif } - return code; -} + clearColValArray(pTableCxt->pValues); -static int32_t allocateMemIfNeed(STableDataBlocks* pDataBlock, int32_t rowSize, int32_t* numOfRows) { - size_t remain = pDataBlock->nAllocSize - pDataBlock->size; - const int factor = 5; - uint32_t nAllocSizeOld = pDataBlock->nAllocSize; - - // expand the allocated size - if (remain < rowSize * factor) { - while (remain < rowSize * factor) { - pDataBlock->nAllocSize = (uint32_t)(pDataBlock->nAllocSize * 1.5); - remain = pDataBlock->nAllocSize - pDataBlock->size; - } - - char* tmp = taosMemoryRealloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize); - if (tmp != NULL) { - pDataBlock->pData = tmp; - memset(pDataBlock->pData + pDataBlock->size, 0, pDataBlock->nAllocSize - pDataBlock->size); - } else { - // do nothing, if allocate more memory failed - pDataBlock->nAllocSize = nAllocSizeOld; - *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize; - return TSDB_CODE_OUT_OF_MEMORY; - } - } - - *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize; - return TSDB_CODE_SUCCESS; + return code; } // pSql -> (field1_value, ...) [(field1_value2, ...) ...] -static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataBlocks* pDataBuf, - int32_t maxRows, int32_t* pNumOfRows, SToken* pToken) { - int32_t code = insInitRowBuilder(&pDataBuf->rowBuilder, pDataBuf->pTableMeta->sversion, &pDataBuf->boundColumnInfo); +static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, + int32_t* pNumOfRows, SToken* pToken) { + int32_t code = TSDB_CODE_SUCCESS; - int32_t extendedRowSize = insGetExtendedRowSize(pDataBuf); (*pNumOfRows) = 0; while (TSDB_CODE_SUCCESS == code) { int32_t index = 0; @@ -1342,13 +1303,9 @@ static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, } pStmt->pSql += index; - if ((*pNumOfRows) >= maxRows || pDataBuf->size + extendedRowSize >= pDataBuf->nAllocSize) { - code = allocateMemIfNeed(pDataBuf, extendedRowSize, &maxRows); - } - bool gotRow = false; if (TSDB_CODE_SUCCESS == code) { - code = parseOneRow(pCxt, &pStmt->pSql, pDataBuf, &gotRow, pToken); + code = parseOneRow(pCxt, &pStmt->pSql, pTableCxt, &gotRow, pToken); } if (TSDB_CODE_SUCCESS == code) { @@ -1361,7 +1318,6 @@ static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, } if (TSDB_CODE_SUCCESS == code && gotRow) { - pDataBuf->size += extendedRowSize; (*pNumOfRows)++; } } @@ -1374,19 +1330,11 @@ static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, } // VALUES (field1_value, ...) [(field1_value2, ...) ...] -static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataBlocks* pDataBuf, +static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, SToken* pToken) { - int32_t maxNumOfRows = 0; int32_t numOfRows = 0; - int32_t code = allocateMemIfNeed(pDataBuf, insGetExtendedRowSize(pDataBuf), &maxNumOfRows); + int32_t code = parseValues(pCxt, pStmt, pTableCxt, &numOfRows, pToken); if (TSDB_CODE_SUCCESS == code) { - code = parseValues(pCxt, pStmt, pDataBuf, maxNumOfRows, &numOfRows, pToken); - } - if (TSDB_CODE_SUCCESS == code) { - code = insSetBlockInfo((SSubmitBlk*)(pDataBuf->pData), pDataBuf, numOfRows, &pCxt->msg); - } - if (TSDB_CODE_SUCCESS == code) { - pDataBuf->numOfTables = 1; pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_INSERT); @@ -1394,11 +1342,9 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* return code; } -static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataBlocks* pDataBuf, - int maxRows, int32_t* pNumOfRows) { - int32_t code = insInitRowBuilder(&pDataBuf->rowBuilder, pDataBuf->pTableMeta->sversion, &pDataBuf->boundColumnInfo); - - int32_t extendedRowSize = insGetExtendedRowSize(pDataBuf); +static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, + int32_t* pNumOfRows) { + int32_t code = TSDB_CODE_SUCCESS; (*pNumOfRows) = 0; char* pLine = NULL; int64_t readLen = 0; @@ -1414,16 +1360,13 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt continue; } - if ((*pNumOfRows) >= maxRows || pDataBuf->size + extendedRowSize >= pDataBuf->nAllocSize) { - code = allocateMemIfNeed(pDataBuf, extendedRowSize, &maxRows); - } - bool gotRow = false; if (TSDB_CODE_SUCCESS == code) { SToken token; strtolower(pLine, pLine); const char* pRow = pLine; - code = parseOneRow(pCxt, (const char**)&pRow, pDataBuf, &gotRow, &token); + + code = parseOneRow(pCxt, (const char**)&pRow, pTableCxt, &gotRow, &token); if (code && firstLine) { firstLine = false; code = 0; @@ -1432,11 +1375,10 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt } if (TSDB_CODE_SUCCESS == code && gotRow) { - pDataBuf->size += extendedRowSize; (*pNumOfRows)++; } - if (TSDB_CODE_SUCCESS == code && pDataBuf->nAllocSize > tsMaxMemUsedByInsert * 1024 * 1024) { + if (TSDB_CODE_SUCCESS == code && (*pNumOfRows) > tsMaxMemUsedByInsert * 1024 * 1024) { pStmt->fileProcessing = true; break; } @@ -1452,18 +1394,10 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt return code; } -static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataBlocks* pDataBuf) { - int32_t maxNumOfRows = 0; +static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { int32_t numOfRows = 0; - int32_t code = allocateMemIfNeed(pDataBuf, insGetExtendedRowSize(pDataBuf), &maxNumOfRows); + int32_t code = parseCsvFile(pCxt, pStmt, pTableCxt, &numOfRows); if (TSDB_CODE_SUCCESS == code) { - code = parseCsvFile(pCxt, pStmt, pDataBuf, maxNumOfRows, &numOfRows); - } - if (TSDB_CODE_SUCCESS == code) { - code = insSetBlockInfo((SSubmitBlk*)(pDataBuf->pData), pDataBuf, numOfRows, &pCxt->msg); - } - if (TSDB_CODE_SUCCESS == code) { - pDataBuf->numOfTables = 1; pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_FILE_INSERT); @@ -1477,7 +1411,7 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpSt } static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pFilePath, - STableDataBlocks* pDataBuf) { + STableDataCxt* pTableCxt) { char filePathStr[TSDB_FILENAME_LEN] = {0}; if (TK_NK_STRING == pFilePath->type) { trimString(pFilePath->z, pFilePath->n, filePathStr, sizeof(filePathStr)); @@ -1489,10 +1423,10 @@ static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* return TAOS_SYSTEM_ERROR(errno); } - return parseDataFromFileImpl(pCxt, pStmt, pDataBuf); + return parseDataFromFileImpl(pCxt, pStmt, pTableCxt); } -static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataBlocks* pDataBuf, +static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, SToken* pToken) { if (tsUseAdapter) { return buildInvalidOperationMsg(&pCxt->msg, "proxy mode does not support csv loading"); @@ -1502,18 +1436,18 @@ static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS if (0 == pToken->n || (TK_NK_STRING != pToken->type && TK_NK_ID != pToken->type)) { return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", pToken->z); } - return parseDataFromFile(pCxt, pStmt, pToken, pDataBuf); + return parseDataFromFile(pCxt, pStmt, pToken, pTableCxt); } // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path -static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataBlocks* pDataBuf) { +static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { SToken token; NEXT_TOKEN(pStmt->pSql, token); switch (token.type) { case TK_VALUES: - return parseValuesClause(pCxt, pStmt, pDataBuf, &token); + return parseValuesClause(pCxt, pStmt, pTableCxt, &token); case TK_FILE: - return parseFileClause(pCxt, pStmt, pDataBuf, &token); + return parseFileClause(pCxt, pStmt, pTableCxt, &token); default: break; } @@ -1524,18 +1458,19 @@ static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS // 1. [(tag1_name, ...)] ... // 2. VALUES ... | FILE ... static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - STableDataBlocks* pDataBuf = NULL; - int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pDataBuf); + STableDataCxt* pTableCxt = NULL; + int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pTableCxt); if (TSDB_CODE_SUCCESS == code) { - code = parseDataClause(pCxt, pStmt, pDataBuf); + code = parseDataClause(pCxt, pStmt, pTableCxt); } return code; } static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - destroyBoundColumnInfo(&pCxt->tags); + insDestroyBoundColInfo(&pCxt->tags); taosMemoryFreeClear(pStmt->pTableMeta); - tdDestroySVCreateTbReq(&pStmt->createTblReq); + tdDestroySVCreateTbReq(pStmt->pCreateTblReq); + taosMemoryFreeClear(pStmt->pCreateTblReq); pCxt->missCache = false; pCxt->usingDuplicateTable = false; pStmt->pBoundCols = NULL; @@ -1593,7 +1528,7 @@ static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModif } static int32_t setStmtInfo(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); + SBoundColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); if (NULL == tags) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1616,13 +1551,11 @@ static int32_t parseInsertBodyBottom(SInsertParseContext* pCxt, SVnodeModifyOpSt } // merge according to vgId - int32_t code = TSDB_CODE_SUCCESS; - if (taosHashGetSize(pStmt->pTableBlockHashObj) > 0) { - code = insMergeTableDataBlocks(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks); - } + int32_t code = insMergeTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks); if (TSDB_CODE_SUCCESS == code) { - code = insBuildOutput(pStmt->pVgroupsHashObj, pStmt->pVgDataBlocks, &pStmt->pDataBlocks); + code = insBuildVgDataBlocks(pStmt->pVgroupsHashObj, pStmt->pVgDataBlocks, &pStmt->pDataBlocks); } + return code; } @@ -1663,8 +1596,8 @@ static int32_t createVnodeModifOpStmt(SInsertParseContext* pCxt, bool reentry, S TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT); } pStmt->pSql = pCxt->pComCxt->pSql; - pStmt->freeHashFunc = insDestroyBlockHashmap; - pStmt->freeArrayFunc = insDestroyBlockArrayList; + pStmt->freeHashFunc = insDestroyTableDataCxtHashMap; + pStmt->freeArrayFunc = insDestroyVgroupDataCxtList; if (!reentry) { pStmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); @@ -1878,10 +1811,10 @@ static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifyOp } static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - STableDataBlocks* pDataBuf = NULL; - int32_t code = getTableDataBlocks(pCxt, pStmt, &pDataBuf); + STableDataCxt* pTableCxt = NULL; + int32_t code = getTableDataCxt(pCxt, pStmt, &pTableCxt); if (TSDB_CODE_SUCCESS == code) { - code = parseDataFromFileImpl(pCxt, pStmt, pDataBuf); + code = parseDataFromFileImpl(pCxt, pStmt, pTableCxt); } if (TSDB_CODE_SUCCESS == code) { @@ -2005,6 +1938,6 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal QUERY_EXEC_STAGE_SCHEDULE == (*pQuery)->execStage) { code = setRefreshMate(*pQuery); } - destroyBoundColumnInfo(&context.tags); + insDestroyBoundColInfo(&context.tags); return code; } diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index e10b195d34dd3b1be5774294703a2155fc93aa81..01a635e4b2d34bbb7982d45400429ac323e9547e 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -29,26 +29,53 @@ typedef struct SKvParam { char buf[TSDB_MAX_TAGS_LEN]; } SKvParam; +int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData** pData) { + *pData = taosMemoryCalloc(1, sizeof(SSubmitTbData)); + if (NULL == *pData) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SSubmitTbData* pNew = *pData; + + *pNew = *pDataBlock->pData; + + cloneSVreateTbReq(pDataBlock->pData->pCreateTbReq, &pNew->pCreateTbReq); + pNew->aCol = taosArrayDup(pDataBlock->pData->aCol, NULL); + + int32_t colNum = taosArrayGetSize(pNew->aCol); + for (int32_t i = 0; i < colNum; ++i) { + SColData* pCol = (SColData*)taosArrayGet(pNew->aCol, i); + tColDataDeepClear(pCol); + } + + return TSDB_CODE_SUCCESS; +} + int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash) { - int32_t code = TSDB_CODE_SUCCESS; - SArray* pVgDataBlocks = NULL; + int32_t code = TSDB_CODE_SUCCESS; + SArray* pVgDataBlocks = NULL; + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; + // merge according to vgId if (taosHashGetSize(pBlockHash) > 0) { - code = insMergeTableDataBlocks(pBlockHash, &pVgDataBlocks); + code = insMergeTableDataCxt(pBlockHash, &pVgDataBlocks); } if (TSDB_CODE_SUCCESS == code) { - code = insBuildOutput(pVgHash, pVgDataBlocks, &((SVnodeModifyOpStmt*)pQuery->pRoot)->pDataBlocks); + code = insBuildVgDataBlocks(pVgHash, pVgDataBlocks, &pStmt->pDataBlocks); + } + + if (pStmt->freeArrayFunc) { + pStmt->freeArrayFunc(pVgDataBlocks); } - insDestroyBlockArrayList(pVgDataBlocks); return code; } int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; - int32_t code = TSDB_CODE_SUCCESS; - SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + int32_t code = TSDB_CODE_SUCCESS; + SBoundColInfo* tags = (SBoundColInfo*)boundTags; if (NULL == tags) { return TSDB_CODE_APP_ERROR; } @@ -64,7 +91,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch goto end; } - SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); + SSchema* pSchema = getTableTagSchema(pDataBlock->pMeta); bool isJson = false; STag* pTag = NULL; @@ -74,7 +101,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch continue; } - SSchema* pTagSchema = &pSchema[tags->boundColumns[c]]; + SSchema* pTagSchema = &pSchema[tags->pColIndex[c]]; int32_t colLen = pTagSchema->bytes; if (IS_VAR_DATA_TYPE(pTagSchema->type)) { colLen = bind[c].length[0]; @@ -136,11 +163,16 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch goto end; } - SVCreateTbReq tbReq = {0}; - insBuildCreateTbReq(&tbReq, tName, pTag, suid, sTableName, tagName, pDataBlock->pTableMeta->tableInfo.numOfTags, - TSDB_DEFAULT_TABLE_TTL); - code = insBuildCreateTbMsg(pDataBlock, &tbReq); - tdDestroySVCreateTbReq(&tbReq); + if (NULL == pDataBlock->pData->pCreateTbReq) { + pDataBlock->pData->pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == pDataBlock->pData->pCreateTbReq) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + } + + insBuildCreateTbReq(pDataBlock->pData->pCreateTbReq, tName, pTag, suid, sTableName, tagName, + pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); end: for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) { @@ -155,199 +187,178 @@ end: return code; } -int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - int32_t extendedRowSize = insGetExtendedRowSize(pDataBlock); - SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; - SRowBuilder* pBuilder = &pDataBlock->rowBuilder; - SMemParam param = {.rb = pBuilder}; - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; - int32_t rowNum = bind->num; - - CHECK_CODE( - insInitRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo)); +int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND* src, TAOS_MULTI_BIND* dst) { + int32_t output = 0; + int32_t newBuflen = (pSchema->bytes - VARSTR_HEADER_SIZE) * src->num; + if (dst->buffer_length < newBuflen) { + dst->buffer = taosMemoryRealloc(dst->buffer, newBuflen); + if (NULL == dst->buffer) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } - CHECK_CODE(insAllocateMemForSize(pDataBlock, extendedRowSize * bind->num)); + if (NULL == dst->length) { + dst->length = taosMemoryRealloc(dst->length, sizeof(int32_t) * src->num); + if (NULL == dst->buffer) { + taosMemoryFreeClear(dst->buffer); + return TSDB_CODE_OUT_OF_MEMORY; + } + } - for (int32_t r = 0; r < bind->num; ++r) { - STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header - tdSRowResetBuf(pBuilder, row); + dst->buffer_length = pSchema->bytes - VARSTR_HEADER_SIZE; - for (int c = 0; c < spd->numOfBound; ++c) { - SSchema* pColSchema = &pSchema[spd->boundColumns[c]]; + for (int32_t i = 0; i < src->num; ++i) { + if (src->is_null && src->is_null[i]) { + continue; + } - if (bind[c].num != rowNum) { - return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); + if (!taosMbsToUcs4(((char*)src->buffer) + src->buffer_length * i, src->length[i], + (TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output)) { + if (errno == E2BIG) { + return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } + char buf[512] = {0}; + snprintf(buf, tListLen(buf), "%s", strerror(errno)); + return buildSyntaxErrMsg(pMsgBuf, buf, NULL); + } - param.schema = pColSchema; - insGetSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); - - if (bind[c].is_null && bind[c].is_null[r]) { - if (pColSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - return buildInvalidOperationMsg(&pBuf, "primary timestamp should not be NULL"); - } - - CHECK_CODE(insMemRowAppend(&pBuf, NULL, 0, ¶m)); - } else { - if (bind[c].buffer_type != pColSchema->type) { - return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); - } + dst->length[i] = output; + } - int32_t colLen = pColSchema->bytes; - if (IS_VAR_DATA_TYPE(pColSchema->type)) { - colLen = bind[c].length[r]; - } + dst->buffer_type = src->buffer_type; + dst->is_null = src->is_null; + dst->num = src->num; - CHECK_CODE(insMemRowAppend(&pBuf, (char*)bind[c].buffer + bind[c].buffer_length * r, colLen, ¶m)); - } + return TSDB_CODE_SUCCESS; +} - if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { - TSKEY tsKey = TD_ROW_KEY(row); - insCheckTimestamp(pDataBlock, (const char*)&tsKey); - } - } - // set the null value for the columns that do not assign values - if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - pBuilder->hasNone = true; +int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); + SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + int32_t rowNum = bind->num; + TAOS_MULTI_BIND ncharBind = {0}; + TAOS_MULTI_BIND* pBind = NULL; + int32_t code = 0; + + for (int c = 0; c < boundInfo->numOfBound; ++c) { + SSchema* pColSchema = &pSchema[boundInfo->pColIndex[c]]; + SColData* pCol = taosArrayGet(pDataBlock->pData->aCol, c); + + if (bind[c].num != rowNum) { + code = buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); + goto _return; } - tdSRowEnd(pBuilder); -#ifdef TD_DEBUG_PRINT_ROW - STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1); - tdSRowPrint(row, pSTSchema, __func__); - taosMemoryFree(pSTSchema); -#endif - pDataBlock->size += extendedRowSize; - } - SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); - return insSetBlockInfo(pBlocks, pDataBlock, bind->num, &pBuf); -} + if (bind[c].buffer_type != pColSchema->type) { + code = buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); + goto _return; + } -int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, - int32_t rowNum) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - int32_t extendedRowSize = insGetExtendedRowSize(pDataBlock); - SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; - SRowBuilder* pBuilder = &pDataBlock->rowBuilder; - SMemParam param = {.rb = pBuilder}; - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; - bool rowStart = (0 == colIdx); - bool rowEnd = ((colIdx + 1) == spd->numOfBound); - - if (rowStart) { - CHECK_CODE( - insInitRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo)); - CHECK_CODE(insAllocateMemForSize(pDataBlock, extendedRowSize * bind->num)); - } - - for (int32_t r = 0; r < bind->num; ++r) { - STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size + extendedRowSize * r); // skip the SSubmitBlk header - if (rowStart) { - tdSRowResetBuf(pBuilder, row); + if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { + code = convertStmtNcharCol(&pBuf, pColSchema, bind + c, &ncharBind); + if (code) { + goto _return; + } + pBind = &ncharBind; } else { - tdSRowGetBuf(pBuilder, row); + pBind = bind + c; } - SSchema* pColSchema = &pSchema[spd->boundColumns[colIdx]]; + tColDataAddValueByBind(pCol, pBind); + } - if (bind->num != rowNum) { - return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); - } + qDebug("stmt all %d columns bind %d rows data", boundInfo->numOfBound, rowNum); - param.schema = pColSchema; - insGetSTSRowAppendInfo(pBuilder->rowType, spd, colIdx, ¶m.toffset, ¶m.colIdx); +_return: - if (bind->is_null && bind->is_null[r]) { - if (pColSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - return buildInvalidOperationMsg(&pBuf, "primary timestamp should not be NULL"); - } + taosMemoryFree(ncharBind.buffer); + taosMemoryFree(ncharBind.length); - CHECK_CODE(insMemRowAppend(&pBuf, NULL, 0, ¶m)); - } else { - if (bind->buffer_type != pColSchema->type) { - return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); - } - - int32_t colLen = pColSchema->bytes; - if (IS_VAR_DATA_TYPE(pColSchema->type)) { - colLen = bind->length[r]; - } + return code; +} - CHECK_CODE(insMemRowAppend(&pBuf, (char*)bind->buffer + bind->buffer_length * r, colLen, ¶m)); - } +int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, + int32_t rowNum) { + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); + SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + SSchema* pColSchema = &pSchema[boundInfo->pColIndex[colIdx]]; + SColData* pCol = taosArrayGet(pDataBlock->pData->aCol, colIdx); + TAOS_MULTI_BIND ncharBind = {0}; + TAOS_MULTI_BIND* pBind = NULL; + int32_t code = 0; + + if (bind->num != rowNum) { + return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); + } - if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { - TSKEY tsKey = TD_ROW_KEY(row); - insCheckTimestamp(pDataBlock, (const char*)&tsKey); - } + if (bind->buffer_type != pColSchema->type) { + return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); + } - // set the null value for the columns that do not assign values - if (rowEnd && (spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - pBuilder->hasNone = true; - } - if (rowEnd) { - tdSRowEnd(pBuilder); - } -#ifdef TD_DEBUG_PRINT_ROW - if (rowEnd) { - STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1); - tdSRowPrint(row, pSTSchema, __func__); - taosMemoryFree(pSTSchema); + if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { + code = convertStmtNcharCol(&pBuf, pColSchema, bind, &ncharBind); + if (code) { + goto _return; } -#endif + pBind = &ncharBind; + } else { + pBind = bind; } - if (rowEnd) { - pDataBlock->size += extendedRowSize * bind->num; + tColDataAddValueByBind(pCol, pBind); - SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); - CHECK_CODE(insSetBlockInfo(pBlocks, pDataBlock, bind->num, &pBuf)); - } + qDebug("stmt col %d bind %d rows data", colIdx, rowNum); - return TSDB_CODE_SUCCESS; +_return: + + taosMemoryFree(ncharBind.buffer); + taosMemoryFree(ncharBind.length); + + return code; } -int32_t buildBoundFields(SParsedDataColInfo* boundInfo, SSchema* pSchema, int32_t* fieldNum, TAOS_FIELD_E** fields, - uint8_t timePrec) { +int32_t buildBoundFields(int32_t numOfBound, int16_t* boundColumns, SSchema* pSchema, int32_t* fieldNum, + TAOS_FIELD_E** fields, uint8_t timePrec) { if (fields) { - *fields = taosMemoryCalloc(boundInfo->numOfBound, sizeof(TAOS_FIELD)); + *fields = taosMemoryCalloc(numOfBound, sizeof(TAOS_FIELD_E)); if (NULL == *fields) { return TSDB_CODE_OUT_OF_MEMORY; } - SSchema* schema = &pSchema[boundInfo->boundColumns[0]]; + SSchema* schema = &pSchema[boundColumns[0]]; if (TSDB_DATA_TYPE_TIMESTAMP == schema->type) { (*fields)[0].precision = timePrec; } - for (int32_t i = 0; i < boundInfo->numOfBound; ++i) { - schema = &pSchema[boundInfo->boundColumns[i]]; + for (int32_t i = 0; i < numOfBound; ++i) { + schema = &pSchema[boundColumns[i]]; strcpy((*fields)[i].name, schema->name); (*fields)[i].type = schema->type; (*fields)[i].bytes = schema->bytes; } } - *fieldNum = boundInfo->numOfBound; + *fieldNum = numOfBound; return TSDB_CODE_SUCCESS; } int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + SBoundColInfo* tags = (SBoundColInfo*)boundTags; if (NULL == tags) { return TSDB_CODE_APP_ERROR; } - if (pDataBlock->pTableMeta->tableType != TSDB_SUPER_TABLE && pDataBlock->pTableMeta->tableType != TSDB_CHILD_TABLE) { + if (pDataBlock->pMeta->tableType != TSDB_SUPER_TABLE && pDataBlock->pMeta->tableType != TSDB_CHILD_TABLE) { return TSDB_CODE_TSC_STMT_API_ERROR; } - SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); + SSchema* pSchema = getTableTagSchema(pDataBlock->pMeta); if (tags->numOfBound <= 0) { *fieldNum = 0; *fields = NULL; @@ -355,15 +366,15 @@ int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TA return TSDB_CODE_SUCCESS; } - CHECK_CODE(buildBoundFields(tags, pSchema, fieldNum, fields, 0)); + CHECK_CODE(buildBoundFields(tags->numOfBound, tags->pColIndex, pSchema, fieldNum, fields, 0)); return TSDB_CODE_SUCCESS; } int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fields) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - if (pDataBlock->boundColumnInfo.numOfBound <= 0) { + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); + if (pDataBlock->boundColsInfo.numOfBound <= 0) { *fieldNum = 0; if (fields) { *fields = NULL; @@ -372,107 +383,125 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fiel return TSDB_CODE_SUCCESS; } - CHECK_CODE(buildBoundFields(&pDataBlock->boundColumnInfo, pSchema, fieldNum, fields, - pDataBlock->pTableMeta->tableInfo.precision)); + CHECK_CODE(buildBoundFields(pDataBlock->boundColsInfo.numOfBound, pDataBlock->boundColsInfo.pColIndex, pSchema, + fieldNum, fields, pDataBlock->pMeta->tableInfo.precision)); return TSDB_CODE_SUCCESS; } -int32_t qResetStmtDataBlock(void* block, bool keepBuf) { - STableDataBlocks* pBlock = (STableDataBlocks*)block; +int32_t qResetStmtDataBlock(STableDataCxt* block, bool deepClear) { + STableDataCxt* pBlock = (STableDataCxt*)block; + int32_t colNum = taosArrayGetSize(pBlock->pData->aCol); - if (keepBuf) { - taosMemoryFreeClear(pBlock->pData); - pBlock->pData = taosMemoryMalloc(TSDB_PAYLOAD_SIZE); - if (NULL == pBlock->pData) { - return TSDB_CODE_OUT_OF_MEMORY; + for (int32_t i = 0; i < colNum; ++i) { + SColData* pCol = (SColData*)taosArrayGet(pBlock->pData->aCol, i); + if (deepClear) { + tColDataDeepClear(pCol); + } else { + tColDataClear(pCol); } - memset(pBlock->pData, 0, sizeof(SSubmitBlk)); - } else { - pBlock->pData = NULL; } - pBlock->ordered = true; - pBlock->prevTS = INT64_MIN; - pBlock->size = sizeof(SSubmitBlk); - pBlock->tsSource = -1; - pBlock->numOfTables = 1; - pBlock->nAllocSize = TSDB_PAYLOAD_SIZE; - pBlock->headerSize = pBlock->size; - pBlock->createTbReqLen = 0; - - memset(&pBlock->rowBuilder, 0, sizeof(pBlock->rowBuilder)); - return TSDB_CODE_SUCCESS; } -int32_t qCloneStmtDataBlock(void** pDst, void* pSrc) { - *pDst = taosMemoryMalloc(sizeof(STableDataBlocks)); +int32_t qCloneStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, bool reset) { + int32_t code = 0; + + *pDst = taosMemoryCalloc(1, sizeof(STableDataCxt)); if (NULL == *pDst) { return TSDB_CODE_OUT_OF_MEMORY; } - memcpy(*pDst, pSrc, sizeof(STableDataBlocks)); - ((STableDataBlocks*)(*pDst))->cloned = true; + STableDataCxt* pNewCxt = (STableDataCxt*)*pDst; + STableDataCxt* pCxt = (STableDataCxt*)pSrc; + pNewCxt->pSchema = NULL; + pNewCxt->pValues = NULL; - STableDataBlocks* pBlock = (STableDataBlocks*)(*pDst); - if (pBlock->pTableMeta) { - void* pNewMeta = taosMemoryMalloc(TABLE_META_SIZE(pBlock->pTableMeta)); + if (pCxt->pMeta) { + void* pNewMeta = taosMemoryMalloc(TABLE_META_SIZE(pCxt->pMeta)); if (NULL == pNewMeta) { - taosMemoryFreeClear(*pDst); + insDestroyTableDataCxt(*pDst); + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(pNewMeta, pCxt->pMeta, TABLE_META_SIZE(pCxt->pMeta)); + pNewCxt->pMeta = pNewMeta; + } + + memcpy(&pNewCxt->boundColsInfo, &pCxt->boundColsInfo, sizeof(pCxt->boundColsInfo)); + pNewCxt->boundColsInfo.pColIndex = NULL; + + if (pCxt->boundColsInfo.pColIndex) { + void* pNewColIdx = taosMemoryMalloc(pCxt->boundColsInfo.numOfBound * sizeof(*pCxt->boundColsInfo.pColIndex)); + if (NULL == pNewColIdx) { + insDestroyTableDataCxt(*pDst); return TSDB_CODE_OUT_OF_MEMORY; } - memcpy(pNewMeta, pBlock->pTableMeta, TABLE_META_SIZE(pBlock->pTableMeta)); - pBlock->pTableMeta = pNewMeta; + memcpy(pNewColIdx, pCxt->boundColsInfo.pColIndex, + pCxt->boundColsInfo.numOfBound * sizeof(*pCxt->boundColsInfo.pColIndex)); + pNewCxt->boundColsInfo.pColIndex = pNewColIdx; + } + + if (pCxt->pData) { + SSubmitTbData* pNewTb = (SSubmitTbData*)taosMemoryMalloc(sizeof(SSubmitTbData)); + if (NULL == pNewTb) { + insDestroyTableDataCxt(*pDst); + return TSDB_CODE_OUT_OF_MEMORY; + } + + memcpy(pNewTb, pCxt->pData, sizeof(*pCxt->pData)); + pNewTb->pCreateTbReq = NULL; + + pNewTb->aCol = taosArrayDup(pCxt->pData->aCol, NULL); + if (NULL == pNewTb) { + insDestroyTableDataCxt(*pDst); + return TSDB_CODE_OUT_OF_MEMORY; + } + + pNewCxt->pData = pNewTb; + + if (reset) { + code = qResetStmtDataBlock(*pDst, true); + } } - return qResetStmtDataBlock(*pDst, false); + return code; } -int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgId) { - int32_t code = qCloneStmtDataBlock(pDst, pSrc); +int32_t qRebuildStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_t uid, uint64_t suid, int32_t vgId, + bool rebuildCreateTb) { + int32_t code = qCloneStmtDataBlock(pDst, pSrc, false); if (code) { return code; } - STableDataBlocks* pBlock = (STableDataBlocks*)*pDst; - pBlock->pData = taosMemoryMalloc(pBlock->nAllocSize); - if (NULL == pBlock->pData) { - qFreeStmtDataBlock(pBlock); - return TSDB_CODE_OUT_OF_MEMORY; + STableDataCxt* pBlock = (STableDataCxt*)*pDst; + if (pBlock->pMeta) { + pBlock->pMeta->uid = uid; + pBlock->pMeta->vgId = vgId; + pBlock->pMeta->suid = suid; } - pBlock->vgId = vgId; + pBlock->pData->suid = suid; + pBlock->pData->uid = uid; - if (pBlock->pTableMeta) { - pBlock->pTableMeta->uid = uid; - pBlock->pTableMeta->vgId = vgId; + if (rebuildCreateTb && NULL == pBlock->pData->pCreateTbReq) { + pBlock->pData->pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == pBlock->pData->pCreateTbReq) { + return TSDB_CODE_OUT_OF_MEMORY; + } } - memset(pBlock->pData, 0, sizeof(SSubmitBlk)); - return TSDB_CODE_SUCCESS; } -STableMeta* qGetTableMetaInDataBlock(void* pDataBlock) { return ((STableDataBlocks*)pDataBlock)->pTableMeta; } +STableMeta* qGetTableMetaInDataBlock(STableDataCxt* pDataBlock) { return ((STableDataCxt*)pDataBlock)->pMeta; } -void qFreeStmtDataBlock(void* pDataBlock) { - if (pDataBlock == NULL) { - return; - } - - taosMemoryFreeClear(((STableDataBlocks*)pDataBlock)->pTableMeta); - taosMemoryFreeClear(((STableDataBlocks*)pDataBlock)->pData); - taosMemoryFreeClear(pDataBlock); -} - -void qDestroyStmtDataBlock(void* pBlock) { +void qDestroyStmtDataBlock(STableDataCxt* pBlock) { if (pBlock == NULL) { return; } - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - - pDataBlock->cloned = false; - insDestroyDataBlock(pDataBlock); + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + insDestroyTableDataCxt(pDataBlock); } diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 73cedfeb3d4109622062477214cd148bdd441391..b8ef7d970df5dd74176f6dcb0b7a8ab478e34129 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -20,938 +20,671 @@ #include "parUtil.h" #include "querynodes.h" #include "tRealloc.h" +#include "tdatablock.h" -typedef struct SBlockKeyTuple { - TSKEY skey; - void* payloadAddr; - int16_t index; -} SBlockKeyTuple; - -typedef struct SBlockKeyInfo { - int32_t maxBytesAlloc; - SBlockKeyTuple* pKeyTuple; -} SBlockKeyInfo; +void qDestroyBoundColInfo(void* pInfo) { + if (NULL == pInfo) { + return; + } -typedef struct { - int32_t index; - SArray* rowArray; // array of merged rows(mem allocated by tRealloc/free by tFree) - STSchema* pSchema; - int64_t tbUid; // suid for child table, uid for normal table -} SBlockRowMerger; + SBoundColInfo* pBoundInfo = (SBoundColInfo*)pInfo; -static FORCE_INLINE void tdResetSBlockRowMerger(SBlockRowMerger* pMerger) { - if (pMerger) { - pMerger->index = -1; - } + taosMemoryFreeClear(pBoundInfo->pColIndex); } -static void tdFreeSBlockRowMerger(SBlockRowMerger* pMerger) { - if (pMerger) { - int32_t size = taosArrayGetSize(pMerger->rowArray); - for (int32_t i = 0; i < size; ++i) { - tFree(*(void**)taosArrayGet(pMerger->rowArray, i)); +static char* tableNameGetPosition(SToken* pToken, char target) { + bool inEscape = false; + bool inQuote = false; + char quotaStr = 0; + + for (uint32_t i = 0; i < pToken->n; ++i) { + if (*(pToken->z + i) == target && (!inEscape) && (!inQuote)) { + return pToken->z + i; } - taosArrayDestroy(pMerger->rowArray); - taosMemoryFreeClear(pMerger->pSchema); - taosMemoryFree(pMerger); - } -} + if (*(pToken->z + i) == TS_ESCAPE_CHAR) { + if (!inQuote) { + inEscape = !inEscape; + } + } -static int32_t rowDataCompar(const void* lhs, const void* rhs) { - TSKEY left = *(TSKEY*)lhs; - TSKEY right = *(TSKEY*)rhs; - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; + if (*(pToken->z + i) == '\'' || *(pToken->z + i) == '"') { + if (!inEscape) { + if (!inQuote) { + quotaStr = *(pToken->z + i); + inQuote = !inQuote; + } else if (quotaStr == *(pToken->z + i)) { + inQuote = !inQuote; + } + } + } } -} -static int32_t rowDataComparStable(const void* lhs, const void* rhs) { - TSKEY left = *(TSKEY*)lhs; - TSKEY right = *(TSKEY*)rhs; - if (left == right) { - return ((SBlockKeyTuple*)lhs)->index - ((SBlockKeyTuple*)rhs)->index; - } else { - return left > right ? 1 : -1; - } + return NULL; } -int32_t insGetExtendedRowSize(STableDataBlocks* pBlock) { - STableComInfo* pTableInfo = &pBlock->pTableMeta->tableInfo; - ASSERT(pBlock->rowSize == pTableInfo->rowSize); - return pBlock->rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + pBlock->boundColumnInfo.extendedVarLen + - (int32_t)TD_BITMAP_BYTES(pTableInfo->numOfColumns - 1); -} +int32_t insCreateSName(SName* pName, SToken* pTableName, int32_t acctId, const char* dbName, SMsgBuf* pMsgBuf) { + const char* msg1 = "name too long"; + const char* msg2 = "invalid database name"; + const char* msg3 = "db is not specified"; + const char* msg4 = "invalid table name"; -void insGetSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo* spd, col_id_t idx, int32_t* toffset, - col_id_t* colIdx) { - col_id_t schemaIdx = 0; - if (IS_DATA_COL_ORDERED(spd)) { - schemaIdx = spd->boundColumns[idx]; - if (TD_IS_TP_ROW_T(rowType)) { - *toffset = (spd->cols + schemaIdx)->toffset; // the offset of firstPart - *colIdx = schemaIdx; - } else { - *toffset = idx * sizeof(SKvRowIdx); // the offset of SKvRowIdx - *colIdx = idx; + int32_t code = TSDB_CODE_SUCCESS; + char* p = tableNameGetPosition(pTableName, TS_PATH_DELIMITER[0]); + + if (p != NULL) { // db has been specified in sql string so we ignore current db path + int32_t dbLen = p - pTableName->z; + if (dbLen <= 0) { + return buildInvalidOperationMsg(pMsgBuf, msg2); } - } else { - ASSERT(idx == (spd->colIdxInfo + idx)->boundIdx); - schemaIdx = (spd->colIdxInfo + idx)->schemaColIdx; - if (TD_IS_TP_ROW_T(rowType)) { - *toffset = (spd->cols + schemaIdx)->toffset; - *colIdx = schemaIdx; - } else { - *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SKvRowIdx); - *colIdx = (spd->colIdxInfo + idx)->finalIdx; + char name[TSDB_DB_FNAME_LEN] = {0}; + strncpy(name, pTableName->z, dbLen); + int32_t actualDbLen = strdequote(name); + + code = tNameSetDbName(pName, acctId, name, actualDbLen); + if (code != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg1); } - } -} -int32_t insSetBlockInfo(SSubmitBlk* pBlocks, STableDataBlocks* dataBuf, int32_t numOfRows, SMsgBuf* pMsg) { - pBlocks->suid = (TSDB_NORMAL_TABLE == dataBuf->pTableMeta->tableType ? 0 : dataBuf->pTableMeta->suid); - pBlocks->uid = dataBuf->pTableMeta->uid; - pBlocks->sversion = dataBuf->pTableMeta->sversion; - pBlocks->schemaLen = dataBuf->createTbReqLen; + int32_t tbLen = pTableName->n - dbLen - 1; + if (tbLen <= 0) { + return buildInvalidOperationMsg(pMsgBuf, msg4); + } - if (pBlocks->numOfRows + numOfRows >= INT32_MAX) { - return buildInvalidOperationMsg(pMsg, "too many rows in sql, total number of rows should be less than INT32_MAX"); - } - pBlocks->numOfRows += numOfRows; - return TSDB_CODE_SUCCESS; -} + char tbname[TSDB_TABLE_FNAME_LEN] = {0}; + strncpy(tbname, p + 1, tbLen); + /*tbLen = */ strdequote(tbname); -void insSetBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, col_id_t numOfCols) { - pColList->numOfCols = numOfCols; - pColList->numOfBound = numOfCols; - pColList->orderStatus = ORDER_STATUS_ORDERED; // default is ORDERED for non-bound mode - pColList->boundColumns = taosMemoryCalloc(pColList->numOfCols, sizeof(col_id_t)); - pColList->cols = taosMemoryCalloc(pColList->numOfCols, sizeof(SBoundColumn)); - pColList->colIdxInfo = NULL; - pColList->flen = 0; - pColList->allNullLen = 0; - - int32_t nVar = 0; - for (int32_t i = 0; i < pColList->numOfCols; ++i) { - uint8_t type = pSchema[i].type; - if (i > 0) { - pColList->cols[i].offset = pColList->cols[i - 1].offset + pSchema[i - 1].bytes; - pColList->cols[i].toffset = pColList->flen; - pColList->flen += TYPE_BYTES[type]; + code = tNameFromString(pName, tbname, T_NAME_TABLE); + if (code != 0) { + return buildInvalidOperationMsg(pMsgBuf, msg1); } - switch (type) { - case TSDB_DATA_TYPE_BINARY: - pColList->allNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES); - ++nVar; - break; - case TSDB_DATA_TYPE_NCHAR: - pColList->allNullLen += (VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); - ++nVar; - break; - default: - break; + } else { // get current DB name first, and then set it into path + if (pTableName->n >= TSDB_TABLE_NAME_LEN) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + strncpy(name, pTableName->z, pTableName->n); + strdequote(name); + + if (dbName == NULL) { + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + + code = tNameSetDbName(pName, acctId, dbName, strlen(dbName)); + if (code != TSDB_CODE_SUCCESS) { + code = buildInvalidOperationMsg(pMsgBuf, msg2); + return code; + } + + code = tNameFromString(pName, name, T_NAME_TABLE); + if (code != 0) { + code = buildInvalidOperationMsg(pMsgBuf, msg1); } - pColList->boundColumns[i] = i; } - pColList->allNullLen += pColList->flen; - pColList->boundNullLen = pColList->allNullLen; // default set allNullLen - pColList->extendedVarLen = (uint16_t)(nVar * sizeof(VarDataOffsetT)); -} -int32_t insSchemaIdxCompar(const void* lhs, const void* rhs) { - uint16_t left = *(uint16_t*)lhs; - uint16_t right = *(uint16_t*)rhs; + if (NULL != strchr(pName->tname, '.')) { + code = generateSyntaxErrMsgExt(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, "The table name cannot contain '.'"); + } + + return code; +} - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; +int16_t insFindCol(SToken* pColname, int16_t start, int16_t end, SSchema* pSchema) { + while (start < end) { + if (strlen(pSchema[start].name) == pColname->n && strncmp(pColname->z, pSchema[start].name, pColname->n) == 0) { + return start; + } + ++start; } + return -1; } -int32_t insBoundIdxCompar(const void* lhs, const void* rhs) { - uint16_t left = *(uint16_t*)POINTER_SHIFT(lhs, sizeof(uint16_t)); - uint16_t right = *(uint16_t*)POINTER_SHIFT(rhs, sizeof(uint16_t)); +void insBuildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, int64_t suid, const char* sname, + SArray* tagName, uint8_t tagNum, int32_t ttl) { + pTbReq->type = TD_CHILD_TABLE; + pTbReq->name = strdup(tname); + pTbReq->ctb.suid = suid; + pTbReq->ctb.tagNum = tagNum; + if (sname) pTbReq->ctb.stbName = strdup(sname); + pTbReq->ctb.pTag = (uint8_t*)pTag; + pTbReq->ctb.tagName = taosArrayDup(tagName, NULL); + pTbReq->ttl = ttl; + pTbReq->commentLen = -1; + + return; +} - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; +static void initBoundCols(int32_t ncols, int16_t* pBoundCols) { + for (int32_t i = 0; i < ncols; ++i) { + pBoundCols[i] = i; } } -void destroyBoundColumnInfo(void* pBoundInfo) { - if (NULL == pBoundInfo) { - return; +static void initColValues(STableMeta* pTableMeta, SArray* pValues) { + SSchema* pSchemas = getTableColumnSchema(pTableMeta); + for (int32_t i = 0; i < pTableMeta->tableInfo.numOfColumns; ++i) { + SColVal val = COL_VAL_NONE(pSchemas[i].colId, pSchemas[i].type); + taosArrayPush(pValues, &val); } - - SParsedDataColInfo* pColList = (SParsedDataColInfo*)pBoundInfo; - - taosMemoryFreeClear(pColList->boundColumns); - taosMemoryFreeClear(pColList->cols); - taosMemoryFreeClear(pColList->colIdxInfo); } -static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, STableMeta* pTableMeta, - STableDataBlocks** dataBlocks) { - STableDataBlocks* dataBuf = (STableDataBlocks*)taosMemoryCalloc(1, sizeof(STableDataBlocks)); - if (dataBuf == NULL) { +int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) { + pInfo->numOfCols = numOfBound; + pInfo->numOfBound = numOfBound; + pInfo->pColIndex = taosMemoryCalloc(numOfBound, sizeof(int16_t)); + if (NULL == pInfo->pColIndex) { return TSDB_CODE_OUT_OF_MEMORY; } + initBoundCols(numOfBound, pInfo->pColIndex); + return TSDB_CODE_SUCCESS; +} - dataBuf->nAllocSize = (uint32_t)defaultSize; - dataBuf->headerSize = startOffset; +void insCheckTableDataOrder(STableDataCxt* pTableCxt, TSKEY tsKey) { + // once the data block is disordered, we do NOT keep last timestamp any more + if (!pTableCxt->ordered) { + return; + } - // the header size will always be the startOffset value, reserved for the subumit block header - if (dataBuf->nAllocSize <= dataBuf->headerSize) { - dataBuf->nAllocSize = dataBuf->headerSize * 2; + if (tsKey < pTableCxt->lastTs) { + pTableCxt->ordered = false; } - dataBuf->pData = taosMemoryMalloc(dataBuf->nAllocSize); - if (dataBuf->pData == NULL) { - taosMemoryFreeClear(dataBuf); - return TSDB_CODE_OUT_OF_MEMORY; + if (tsKey == pTableCxt->lastTs) { + pTableCxt->duplicateTs = true; } - memset(dataBuf->pData, 0, sizeof(SSubmitBlk)); - dataBuf->pTableMeta = tableMetaDup(pTableMeta); + pTableCxt->lastTs = tsKey; + return; +} - SParsedDataColInfo* pColInfo = &dataBuf->boundColumnInfo; - SSchema* pSchema = getTableColumnSchema(dataBuf->pTableMeta); - insSetBoundColumnInfo(pColInfo, pSchema, dataBuf->pTableMeta->tableInfo.numOfColumns); +void insDestroyBoundColInfo(SBoundColInfo* pInfo) { taosMemoryFreeClear(pInfo->pColIndex); } - dataBuf->ordered = true; - dataBuf->prevTS = INT64_MIN; - dataBuf->rowSize = rowSize; - dataBuf->size = startOffset; - dataBuf->vgId = dataBuf->pTableMeta->vgId; +static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreateTbReq, STableDataCxt** pOutput, + bool colMode) { + STableDataCxt* pTableCxt = taosMemoryCalloc(1, sizeof(STableDataCxt)); + if (NULL == pTableCxt) { + return TSDB_CODE_OUT_OF_MEMORY; + } - assert(defaultSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); + int32_t code = TSDB_CODE_SUCCESS; - *dataBlocks = dataBuf; - return TSDB_CODE_SUCCESS; -} + pTableCxt->lastTs = 0; + pTableCxt->ordered = true; + pTableCxt->duplicateTs = false; -int32_t insBuildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) { - SEncoder coder = {0}; - char* pBuf; - int32_t len; - - int32_t ret = 0; - tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len, ret); - if (pBlocks->nAllocSize - pBlocks->size < len) { - pBlocks->nAllocSize += len + pBlocks->rowSize; - char* pTmp = taosMemoryRealloc(pBlocks->pData, pBlocks->nAllocSize); - if (pTmp != NULL) { - pBlocks->pData = pTmp; - memset(pBlocks->pData + pBlocks->size, 0, pBlocks->nAllocSize - pBlocks->size); + pTableCxt->pMeta = tableMetaDup(pTableMeta); + if (NULL == pTableCxt->pMeta) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + if (TSDB_CODE_SUCCESS == code) { + pTableCxt->pSchema = + tBuildTSchema(getTableColumnSchema(pTableMeta), pTableMeta->tableInfo.numOfColumns, pTableMeta->sversion); + if (NULL == pTableCxt->pSchema) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + if (TSDB_CODE_SUCCESS == code) { + code = insInitBoundColsInfo(pTableMeta->tableInfo.numOfColumns, &pTableCxt->boundColsInfo); + } + if (TSDB_CODE_SUCCESS == code) { + pTableCxt->pValues = taosArrayInit(pTableMeta->tableInfo.numOfColumns, sizeof(SColVal)); + if (NULL == pTableCxt->pValues) { + code = TSDB_CODE_OUT_OF_MEMORY; } else { - pBlocks->nAllocSize -= len + pBlocks->rowSize; - return TSDB_CODE_OUT_OF_MEMORY; + initColValues(pTableMeta, pTableCxt->pValues); + } + } + if (TSDB_CODE_SUCCESS == code) { + pTableCxt->pData = taosMemoryCalloc(1, sizeof(SSubmitTbData)); + if (NULL == pTableCxt->pData) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pTableCxt->pData->flags = NULL != *pCreateTbReq ? SUBMIT_REQ_AUTO_CREATE_TABLE : 0; + pTableCxt->pData->flags |= colMode ? SUBMIT_REQ_COLUMN_DATA_FORMAT : 0; + pTableCxt->pData->suid = pTableMeta->suid; + pTableCxt->pData->uid = pTableMeta->uid; + pTableCxt->pData->sver = pTableMeta->sversion; + pTableCxt->pData->pCreateTbReq = *pCreateTbReq; + *pCreateTbReq = NULL; + if (pTableCxt->pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + pTableCxt->pData->aCol = taosArrayInit(128, sizeof(SColData)); + if (NULL == pTableCxt->pData->aCol) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } else { + pTableCxt->pData->aRowP = taosArrayInit(128, POINTER_BYTES); + if (NULL == pTableCxt->pData->aRowP) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } } } - pBuf = pBlocks->pData + pBlocks->size; - - tEncoderInit(&coder, pBuf, len); - int32_t code = tEncodeSVCreateTbReq(&coder, pCreateTbReq); - tEncoderClear(&coder); - pBlocks->size += len; - pBlocks->createTbReqLen = len; + if (TSDB_CODE_SUCCESS == code) { + *pOutput = pTableCxt; + qDebug("tableDataCxt created, uid:%" PRId64 ", vgId:%d", pTableMeta->uid, pTableMeta->vgId); + } else { + taosMemoryFree(pTableCxt); + } return code; } -void insDestroyDataBlock(STableDataBlocks* pDataBlock) { - if (pDataBlock == NULL) { - return; +static void resetColValues(SArray* pValues) { + int32_t num = taosArrayGetSize(pValues); + for (int32_t i = 0; i < num; ++i) { + SColVal* pVal = taosArrayGet(pValues, i); + pVal->flag = CV_FLAG_NONE; } +} - taosMemoryFreeClear(pDataBlock->pData); - taosMemoryFreeClear(pDataBlock->pTableMeta); - destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); - taosMemoryFreeClear(pDataBlock); +int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta* pTableMeta, + SVCreateTbReq** pCreateTbReq, STableDataCxt** pTableCxt, bool colMode) { + STableDataCxt** tmp = (STableDataCxt**)taosHashGet(pHash, id, idLen); + if (NULL != tmp) { + *pTableCxt = *tmp; + resetColValues((*pTableCxt)->pValues); + return TSDB_CODE_SUCCESS; + } + int32_t code = createTableDataCxt(pTableMeta, pCreateTbReq, pTableCxt, colMode); + if (TSDB_CODE_SUCCESS == code) { + code = taosHashPut(pHash, id, idLen, pTableCxt, POINTER_BYTES); + } + return code; } -int32_t insGetDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, int32_t size, int32_t startOffset, - int32_t rowSize, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, - SArray* pBlockList, SVCreateTbReq* pCreateTbReq) { - *dataBlocks = NULL; - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)id, idLen); - if (t1 != NULL) { - *dataBlocks = *t1; +static void destroyColVal(void* p) { + SColVal* pVal = p; + if (TSDB_DATA_TYPE_NCHAR == pVal->type) { + taosMemoryFree(pVal->value.pData); } +} - if (*dataBlocks == NULL) { - int32_t ret = createDataBlock((size_t)size, rowSize, startOffset, pTableMeta, dataBlocks); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } +void insDestroyTableDataCxt(STableDataCxt* pTableCxt) { + if (NULL == pTableCxt) { + return; + } - if (NULL != pCreateTbReq && NULL != pCreateTbReq->ctb.pTag) { - ret = insBuildCreateTbMsg(*dataBlocks, pCreateTbReq); - if (ret != TSDB_CODE_SUCCESS) { - insDestroyDataBlock(*dataBlocks); - return ret; - } - } + taosMemoryFreeClear(pTableCxt->pMeta); + tDestroyTSchema(pTableCxt->pSchema); + insDestroyBoundColInfo(&pTableCxt->boundColsInfo); + taosArrayDestroyEx(pTableCxt->pValues, destroyColVal); + if (pTableCxt->pData) { + tDestroySSubmitTbData(pTableCxt->pData, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pTableCxt->pData); + } + taosMemoryFree(pTableCxt); +} - // converting to 'const char*' is to handle coverity scan errors - taosHashPut(pHashList, (const char*)id, idLen, (const char*)dataBlocks, POINTER_BYTES); - if (pBlockList) { - taosArrayPush(pBlockList, dataBlocks); - } +void insDestroyVgroupDataCxt(SVgroupDataCxt* pVgCxt) { + if (NULL == pVgCxt) { + return; } - return TSDB_CODE_SUCCESS; + tDestroySSubmitReq2(pVgCxt->pData, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pVgCxt->pData); + taosMemoryFree(pVgCxt); } -void insDestroyBlockArrayList(SArray* pDataBlockList) { - if (pDataBlockList == NULL) { +void insDestroyVgroupDataCxtList(SArray* pVgCxtList) { + if (NULL == pVgCxtList) { return; } - size_t size = taosArrayGetSize(pDataBlockList); + size_t size = taosArrayGetSize(pVgCxtList); for (int32_t i = 0; i < size; i++) { - void* p = taosArrayGetP(pDataBlockList, i); - insDestroyDataBlock(p); + void* p = taosArrayGetP(pVgCxtList, i); + insDestroyVgroupDataCxt(p); } - taosArrayDestroy(pDataBlockList); + taosArrayDestroy(pVgCxtList); } -void insDestroyBlockHashmap(SHashObj* pDataBlockHash) { - if (pDataBlockHash == NULL) { +void insDestroyVgroupDataCxtHashMap(SHashObj* pVgCxtHash) { + if (NULL == pVgCxtHash) { return; } - void** p1 = taosHashIterate(pDataBlockHash, NULL); - while (p1) { - STableDataBlocks* pBlocks = *p1; - insDestroyDataBlock(pBlocks); + void** p = taosHashIterate(pVgCxtHash, NULL); + while (p) { + insDestroyVgroupDataCxt(*(SVgroupDataCxt**)p); - p1 = taosHashIterate(pDataBlockHash, p1); + p = taosHashIterate(pVgCxtHash, p); } - taosHashCleanup(pDataBlockHash); + taosHashCleanup(pVgCxtHash); } -// data block is disordered, sort it in ascending order -static int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo) { - SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData; - int16_t nRows = pBlocks->numOfRows; - - // size is less than the total size, since duplicated rows may be removed yet. - - // allocate memory - size_t nAlloc = nRows * sizeof(SBlockKeyTuple); - if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { - char* tmp = taosMemoryRealloc(pBlkKeyInfo->pKeyTuple, nAlloc); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple*)tmp; - pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc; +void insDestroyTableDataCxtHashMap(SHashObj* pTableCxtHash) { + if (NULL == pTableCxtHash) { + return; } - memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc); - int32_t extendedRowSize = insGetExtendedRowSize(dataBuf); - SBlockKeyTuple* pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - char* pBlockData = pBlocks->data + pBlocks->schemaLen; - int n = 0; - while (n < nRows) { - pBlkKeyTuple->skey = TD_ROW_KEY((STSRow*)pBlockData); - pBlkKeyTuple->payloadAddr = pBlockData; - pBlkKeyTuple->index = n; + void** p = taosHashIterate(pTableCxtHash, NULL); + while (p) { + insDestroyTableDataCxt(*(STableDataCxt**)p); - // next loop - pBlockData += extendedRowSize; - ++pBlkKeyTuple; - ++n; + p = taosHashIterate(pTableCxtHash, p); } - if (!dataBuf->ordered) { - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - - // todo. qsort is unstable, if timestamp is same, should get the last one - taosSort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataComparStable); - - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - int32_t i = 0; - int32_t j = 1; - while (j < nRows) { - TSKEY ti = (pBlkKeyTuple + i)->skey; - TSKEY tj = (pBlkKeyTuple + j)->skey; - - if (ti == tj) { - ++j; - continue; - } + taosHashCleanup(pTableCxtHash); +} - int32_t nextPos = (++i); - if (nextPos != j) { - memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + j, sizeof(SBlockKeyTuple)); - } - ++j; +static int32_t fillVgroupDataCxt(STableDataCxt* pTableCxt, SVgroupDataCxt* pVgCxt) { + if (NULL == pVgCxt->pData->aSubmitTbData) { + pVgCxt->pData->aSubmitTbData = taosArrayInit(128, sizeof(SSubmitTbData)); + if (NULL == pVgCxt->pData->aSubmitTbData) { + return TSDB_CODE_OUT_OF_MEMORY; } - - dataBuf->ordered = true; - pBlocks->numOfRows = i + 1; } + taosArrayPush(pVgCxt->pData->aSubmitTbData, pTableCxt->pData); + taosMemoryFreeClear(pTableCxt->pData); - dataBuf->size = sizeof(SSubmitBlk) + pBlocks->numOfRows * extendedRowSize; - dataBuf->prevTS = INT64_MIN; + qDebug("add tableDataCxt uid:%" PRId64 " to vgId:%d", pTableCxt->pMeta->uid, pVgCxt->vgId); - return 0; -} - -static void* tdGetCurRowFromBlockMerger(SBlockRowMerger* pBlkRowMerger) { - if (pBlkRowMerger && (pBlkRowMerger->index >= 0)) { - ASSERT(pBlkRowMerger->index < taosArrayGetSize(pBlkRowMerger->rowArray)); - return *(void**)taosArrayGet(pBlkRowMerger->rowArray, pBlkRowMerger->index); - } - return NULL; + return TSDB_CODE_SUCCESS; } -static int32_t tdBlockRowMerge(STableMeta* pTableMeta, SBlockKeyTuple* pEndKeyTp, int32_t nDupRows, - SBlockRowMerger** pBlkRowMerger, int32_t rowSize) { - ASSERT(nDupRows > 1); - SBlockKeyTuple* pStartKeyTp = pEndKeyTp - (nDupRows - 1); - ASSERT(pStartKeyTp->skey == pEndKeyTp->skey); - - // TODO: optimization if end row is all normal -#if 0 - STSRow* pEndRow = (STSRow*)pEndKeyTp->payloadAddr; - if(isNormal(pEndRow)) { // set the end row if it is normal and return directly - pStartKeyTp->payloadAddr = pEndKeyTp->payloadAddr; - return TSDB_CODE_SUCCESS; - } -#endif - - if (!(*pBlkRowMerger)) { - (*pBlkRowMerger) = taosMemoryCalloc(1, sizeof(**pBlkRowMerger)); - if (!(*pBlkRowMerger)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - (*pBlkRowMerger)->index = -1; - if (!(*pBlkRowMerger)->rowArray) { - (*pBlkRowMerger)->rowArray = taosArrayInit(1, sizeof(void*)); - if (!(*pBlkRowMerger)->rowArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - } - } - - if ((*pBlkRowMerger)->pSchema) { - if ((*pBlkRowMerger)->pSchema->version != pTableMeta->sversion) { - taosMemoryFreeClear((*pBlkRowMerger)->pSchema); - } else { - if ((*pBlkRowMerger)->tbUid != (pTableMeta->suid > 0 ? pTableMeta->suid : pTableMeta->uid)) { - taosMemoryFreeClear((*pBlkRowMerger)->pSchema); - } - } +static int32_t createVgroupDataCxt(STableDataCxt* pTableCxt, SHashObj* pVgroupHash, SArray* pVgroupList, + SVgroupDataCxt** pOutput) { + SVgroupDataCxt* pVgCxt = taosMemoryCalloc(1, sizeof(SVgroupDataCxt)); + if (NULL == pVgCxt) { + return TSDB_CODE_OUT_OF_MEMORY; } - - if (!(*pBlkRowMerger)->pSchema) { - (*pBlkRowMerger)->pSchema = - tdGetSTSChemaFromSSChema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->sversion); - - if (!(*pBlkRowMerger)->pSchema) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - (*pBlkRowMerger)->tbUid = pTableMeta->suid > 0 ? pTableMeta->suid : pTableMeta->uid; + pVgCxt->pData = taosMemoryCalloc(1, sizeof(SSubmitReq2)); + if (NULL == pVgCxt->pData) { + insDestroyVgroupDataCxt(pVgCxt); + return TSDB_CODE_OUT_OF_MEMORY; } - void* pDestRow = NULL; - ++((*pBlkRowMerger)->index); - if ((*pBlkRowMerger)->index < taosArrayGetSize((*pBlkRowMerger)->rowArray)) { - void** pAlloc = (void**)taosArrayGet((*pBlkRowMerger)->rowArray, (*pBlkRowMerger)->index); - if (tRealloc((uint8_t**)pAlloc, rowSize) != 0) { - return TSDB_CODE_FAILED; - } - pDestRow = *pAlloc; + pVgCxt->vgId = pTableCxt->pMeta->vgId; + int32_t code = taosHashPut(pVgroupHash, &pVgCxt->vgId, sizeof(pVgCxt->vgId), &pVgCxt, POINTER_BYTES); + if (TSDB_CODE_SUCCESS == code) { + taosArrayPush(pVgroupList, &pVgCxt); + *pOutput = pVgCxt; } else { - if (tRealloc((uint8_t**)&pDestRow, rowSize) != 0) { - return TSDB_CODE_FAILED; - } - taosArrayPush((*pBlkRowMerger)->rowArray, &pDestRow); - } - - // merge rows to pDestRow - STSchema* pSchema = (*pBlkRowMerger)->pSchema; - SArray* pArray = taosArrayInit(pSchema->numOfCols, sizeof(SColVal)); - for (int32_t i = 0; i < pSchema->numOfCols; ++i) { - SColVal colVal = {0}; - for (int32_t j = 0; j < nDupRows; ++j) { - tTSRowGetVal((pEndKeyTp - j)->payloadAddr, pSchema, i, &colVal); - if (!COL_VAL_IS_NONE(&colVal)) { - break; - } - } - taosArrayPush(pArray, &colVal); + insDestroyVgroupDataCxt(pVgCxt); } - if (tdSTSRowNew(pArray, pSchema, (STSRow**)&pDestRow) < 0) { - taosArrayDestroy(pArray); - return TSDB_CODE_FAILED; - } - - taosArrayDestroy(pArray); - return TSDB_CODE_SUCCESS; + return code; } -// data block is disordered, sort it in ascending order, and merge dup rows if exists -static int sortMergeDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo, - SBlockRowMerger** ppBlkRowMerger) { - SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData; - STableMeta* pTableMeta = dataBuf->pTableMeta; - int32_t nRows = pBlocks->numOfRows; - - // size is less than the total size, since duplicated rows may be removed. - - // allocate memory - size_t nAlloc = nRows * sizeof(SBlockKeyTuple); - if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { - char* tmp = taosMemoryRealloc(pBlkKeyInfo->pKeyTuple, nAlloc); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple*)tmp; - pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc; +int insColDataComp(const void* lp, const void* rp) { + SColData* pLeft = (SColData*)lp; + SColData* pRight = (SColData*)rp; + if (pLeft->cid < pRight->cid) { + return -1; + } else if (pLeft->cid > pRight->cid) { + return 1; } - memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc); - tdResetSBlockRowMerger(*ppBlkRowMerger); - - int32_t extendedRowSize = insGetExtendedRowSize(dataBuf); - SBlockKeyTuple* pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - char* pBlockData = pBlocks->data + pBlocks->schemaLen; - int32_t n = 0; - while (n < nRows) { - pBlkKeyTuple->skey = TD_ROW_KEY((STSRow*)pBlockData); - pBlkKeyTuple->payloadAddr = pBlockData; - pBlkKeyTuple->index = n; + return 0; +} - // next loop - pBlockData += extendedRowSize; - ++pBlkKeyTuple; - ++n; +int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks) { + SHashObj* pVgroupHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); + SArray* pVgroupList = taosArrayInit(8, POINTER_BYTES); + if (NULL == pVgroupHash || NULL == pVgroupList) { + taosHashCleanup(pVgroupHash); + taosArrayDestroy(pVgroupList); + return TSDB_CODE_OUT_OF_MEMORY; } - if (!dataBuf->ordered) { - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - - taosSort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataComparStable); - - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - bool hasDup = false; - int32_t nextPos = 0; - int32_t i = 0; - int32_t j = 1; + int32_t code = TSDB_CODE_SUCCESS; + bool colFormat = false; - while (j < nRows) { - TSKEY ti = (pBlkKeyTuple + i)->skey; - TSKEY tj = (pBlkKeyTuple + j)->skey; + void* p = taosHashIterate(pTableHash, NULL); + if (p) { + STableDataCxt* pTableCxt = *(STableDataCxt**)p; + colFormat = (0 != (pTableCxt->pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT)); + } - if (ti == tj) { - ++j; + while (TSDB_CODE_SUCCESS == code && NULL != p) { + STableDataCxt* pTableCxt = *(STableDataCxt**)p; + if (colFormat) { + SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, 0); + if (pCol->nVal <= 0) { + p = taosHashIterate(pTableHash, p); continue; } - if ((j - i) > 1) { - if (tdBlockRowMerge(pTableMeta, (pBlkKeyTuple + j - 1), j - i, ppBlkRowMerger, extendedRowSize) < 0) { - return TSDB_CODE_FAILED; - } - (pBlkKeyTuple + nextPos)->payloadAddr = tdGetCurRowFromBlockMerger(*ppBlkRowMerger); - if (!hasDup) { - hasDup = true; - } - i = j; - } else { - if (hasDup) { - memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + i, sizeof(SBlockKeyTuple)); - } - ++i; + if (pTableCxt->pData->pCreateTbReq) { + pTableCxt->pData->flags |= SUBMIT_REQ_AUTO_CREATE_TABLE; } - ++nextPos; - ++j; - } + taosArraySort(pTableCxt->pData->aCol, insColDataComp); - if ((j - i) > 1) { - ASSERT((pBlkKeyTuple + i)->skey == (pBlkKeyTuple + j - 1)->skey); - if (tdBlockRowMerge(pTableMeta, (pBlkKeyTuple + j - 1), j - i, ppBlkRowMerger, extendedRowSize) < 0) { - return TSDB_CODE_FAILED; + tColDataSortMerge(pTableCxt->pData->aCol); + } else { + if (!pTableCxt->ordered) { + tRowSort(pTableCxt->pData->aRowP); + } + if (!pTableCxt->ordered || pTableCxt->duplicateTs) { + code = tRowMerge(pTableCxt->pData->aRowP, pTableCxt->pSchema, 0); } - (pBlkKeyTuple + nextPos)->payloadAddr = tdGetCurRowFromBlockMerger(*ppBlkRowMerger); - } else if (hasDup) { - memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + i, sizeof(SBlockKeyTuple)); } - dataBuf->ordered = true; - pBlocks->numOfRows = nextPos + 1; - } - - dataBuf->size = sizeof(SSubmitBlk) + pBlocks->numOfRows * extendedRowSize; - dataBuf->prevTS = INT64_MIN; - - return TSDB_CODE_SUCCESS; -} - -// Erase the empty space reserved for binary data -static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SBlockKeyTuple* blkKeyTuple) { - // TODO: optimize this function, handle the case while binary is not presented - int32_t nonDataLen = sizeof(SSubmitBlk) + pTableDataBlock->createTbReqLen; - SSubmitBlk* pBlock = pDataBlock; - memcpy(pDataBlock, pTableDataBlock->pData, nonDataLen); - pDataBlock = (char*)pDataBlock + nonDataLen; - - pBlock->schemaLen = pTableDataBlock->createTbReqLen; - pBlock->dataLen = 0; - - int32_t numOfRows = pBlock->numOfRows; - for (int32_t i = 0; i < numOfRows; ++i) { - void* payload = (blkKeyTuple + i)->payloadAddr; - TDRowLenT rowTLen = TD_ROW_LEN((STSRow*)payload); - memcpy(pDataBlock, payload, rowTLen); - pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen); - pBlock->dataLen += rowTLen; - } - - return pBlock->dataLen + pBlock->schemaLen; -} - -int32_t insMergeTableDataBlocks(SHashObj* pHashObj, SArray** pVgDataBlocks) { - const int INSERT_HEAD_SIZE = sizeof(SSubmitReq); - int code = 0; - SHashObj* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); - SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); - - STableDataBlocks** p = taosHashIterate(pHashObj, NULL); - STableDataBlocks* pOneTableBlock = *p; - SBlockKeyInfo blkKeyInfo = {0}; // share by pOneTableBlock - SBlockRowMerger* pBlkRowMerger = NULL; - - while (pOneTableBlock) { - SSubmitBlk* pBlocks = (SSubmitBlk*)pOneTableBlock->pData; - if (pBlocks->numOfRows > 0) { - STableDataBlocks* dataBuf = NULL; - pOneTableBlock->pTableMeta->vgId = pOneTableBlock->vgId; // for schemaless, restore origin vgId - int32_t ret = insGetDataBlockFromList(pVnodeDataBlockHashList, &pOneTableBlock->vgId, - sizeof(pOneTableBlock->vgId), TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, - pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList, NULL); - if (ret != TSDB_CODE_SUCCESS) { - tdFreeSBlockRowMerger(pBlkRowMerger); - taosHashCleanup(pVnodeDataBlockHashList); - insDestroyBlockArrayList(pVnodeDataBlockList); - taosMemoryFreeClear(blkKeyInfo.pKeyTuple); - return ret; - } - ASSERT(pOneTableBlock->pTableMeta->tableInfo.rowSize > 0); - // the maximum expanded size in byte when a row-wise data is converted to SDataRow format - int64_t destSize = dataBuf->size + pOneTableBlock->size + - sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta) + - pOneTableBlock->createTbReqLen; - - if (dataBuf->nAllocSize < destSize) { - dataBuf->nAllocSize = (uint32_t)(destSize * 1.5); - char* tmp = taosMemoryRealloc(dataBuf->pData, dataBuf->nAllocSize); - if (tmp != NULL) { - dataBuf->pData = tmp; - } else { // failed to allocate memory, free already allocated memory and return error code - tdFreeSBlockRowMerger(pBlkRowMerger); - taosHashCleanup(pVnodeDataBlockHashList); - insDestroyBlockArrayList(pVnodeDataBlockList); - taosMemoryFreeClear(dataBuf->pData); - taosMemoryFreeClear(blkKeyInfo.pKeyTuple); - return TSDB_CODE_OUT_OF_MEMORY; - } + if (TSDB_CODE_SUCCESS == code) { + SVgroupDataCxt* pVgCxt = NULL; + int32_t vgId = pTableCxt->pMeta->vgId; + void** p = taosHashGet(pVgroupHash, &vgId, sizeof(vgId)); + if (NULL == p) { + code = createVgroupDataCxt(pTableCxt, pVgroupHash, pVgroupList, &pVgCxt); + } else { + pVgCxt = *(SVgroupDataCxt**)p; } - - if ((code = sortMergeDataBlockDupRows(pOneTableBlock, &blkKeyInfo, &pBlkRowMerger)) != 0) { - tdFreeSBlockRowMerger(pBlkRowMerger); - taosHashCleanup(pVnodeDataBlockHashList); - insDestroyBlockArrayList(pVnodeDataBlockList); - taosMemoryFreeClear(dataBuf->pData); - taosMemoryFreeClear(blkKeyInfo.pKeyTuple); - return code; + if (TSDB_CODE_SUCCESS == code) { + code = fillVgroupDataCxt(pTableCxt, pVgCxt); } - ASSERT(blkKeyInfo.pKeyTuple != NULL && pBlocks->numOfRows > 0); - - // erase the empty space reserved for binary data - int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, blkKeyInfo.pKeyTuple); - - dataBuf->size += (finalLen + sizeof(SSubmitBlk)); - assert(dataBuf->size <= dataBuf->nAllocSize); - dataBuf->numOfTables += 1; } - - p = taosHashIterate(pHashObj, p); - if (p == NULL) { - break; + if (TSDB_CODE_SUCCESS == code) { + p = taosHashIterate(pTableHash, p); } + } - pOneTableBlock = *p; + taosHashCleanup(pVgroupHash); + if (TSDB_CODE_SUCCESS == code) { + *pVgDataBlocks = pVgroupList; + } else { + insDestroyVgroupDataCxtList(pVgroupList); } - // free the table data blocks; - tdFreeSBlockRowMerger(pBlkRowMerger); - taosHashCleanup(pVnodeDataBlockHashList); - taosMemoryFreeClear(blkKeyInfo.pKeyTuple); - *pVgDataBlocks = pVnodeDataBlockList; - return TSDB_CODE_SUCCESS; + return code; } -int32_t insAllocateMemForSize(STableDataBlocks* pDataBlock, int32_t allSize) { - size_t remain = pDataBlock->nAllocSize - pDataBlock->size; - uint32_t nAllocSizeOld = pDataBlock->nAllocSize; - - // expand the allocated size - if (remain < allSize) { - pDataBlock->nAllocSize = (pDataBlock->size + allSize) * 1.5; - - char* tmp = taosMemoryRealloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize); - if (tmp != NULL) { - pDataBlock->pData = tmp; - memset(pDataBlock->pData + pDataBlock->size, 0, pDataBlock->nAllocSize - pDataBlock->size); - } else { - // do nothing, if allocate more memory failed - pDataBlock->nAllocSize = nAllocSizeOld; +static int32_t buildSubmitReq(int32_t vgId, SSubmitReq2* pReq, void** pData, uint32_t* pLen) { + int32_t code = TSDB_CODE_SUCCESS; + uint32_t len = 0; + void* pBuf = NULL; + tEncodeSize(tEncodeSSubmitReq2, pReq, len, code); + if (TSDB_CODE_SUCCESS == code) { + SEncoder encoder; + len += sizeof(SMsgHead); + pBuf = taosMemoryMalloc(len); + if (NULL == pBuf) { return TSDB_CODE_OUT_OF_MEMORY; } + ((SMsgHead*)pBuf)->vgId = htonl(vgId); + ((SMsgHead*)pBuf)->contLen = htonl(len); + tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SMsgHead)), len - sizeof(SMsgHead)); + code = tEncodeSSubmitReq2(&encoder, pReq); + tEncoderClear(&encoder); } - return TSDB_CODE_SUCCESS; + if (TSDB_CODE_SUCCESS == code) { + *pData = pBuf; + *pLen = len; + } else { + taosMemoryFree(pBuf); + } + return code; } -int32_t insInitRowBuilder(SRowBuilder* pBuilder, int16_t schemaVer, SParsedDataColInfo* pColInfo) { - ASSERT(pColInfo->numOfCols > 0 && (pColInfo->numOfBound <= pColInfo->numOfCols)); - tdSRowInit(pBuilder, schemaVer); - tdSRowSetExtendedInfo(pBuilder, pColInfo->numOfCols, pColInfo->numOfBound, pColInfo->flen, pColInfo->allNullLen, - pColInfo->boundNullLen); - return TSDB_CODE_SUCCESS; +static void destroyVgDataBlocks(void* p) { + SVgDataBlocks* pVg = p; + taosMemoryFree(pVg->pData); + taosMemoryFree(pVg); } -static char* tableNameGetPosition(SToken* pToken, char target) { - bool inEscape = false; - bool inQuote = false; - char quotaStr = 0; - - for (uint32_t i = 0; i < pToken->n; ++i) { - if (*(pToken->z + i) == target && (!inEscape) && (!inQuote)) { - return pToken->z + i; - } - - if (*(pToken->z + i) == TS_ESCAPE_CHAR) { - if (!inQuote) { - inEscape = !inEscape; - } - } - - if (*(pToken->z + i) == '\'' || *(pToken->z + i) == '"') { - if (!inEscape) { - if (!inQuote) { - quotaStr = *(pToken->z + i); - inQuote = !inQuote; - } else if (quotaStr == *(pToken->z + i)) { - inQuote = !inQuote; - } - } - } +int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList, SArray** pVgDataBlocks) { + size_t numOfVg = taosArrayGetSize(pVgDataCxtList); + SArray* pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); + if (NULL == pDataBlocks) { + return TSDB_CODE_OUT_OF_MEMORY; } - return NULL; -} - -int32_t insCreateSName(SName* pName, SToken* pTableName, int32_t acctId, const char* dbName, SMsgBuf* pMsgBuf) { - const char* msg1 = "name too long"; - const char* msg2 = "invalid database name"; - const char* msg3 = "db is not specified"; - const char* msg4 = "invalid table name"; - int32_t code = TSDB_CODE_SUCCESS; - char* p = tableNameGetPosition(pTableName, TS_PATH_DELIMITER[0]); - - if (p != NULL) { // db has been specified in sql string so we ignore current db path - assert(*p == TS_PATH_DELIMITER[0]); - - int32_t dbLen = p - pTableName->z; - if (dbLen <= 0) { - return buildInvalidOperationMsg(pMsgBuf, msg2); + for (size_t i = 0; TSDB_CODE_SUCCESS == code && i < numOfVg; ++i) { + SVgroupDataCxt* src = taosArrayGetP(pVgDataCxtList, i); + SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); + if (NULL == dst) { + code = TSDB_CODE_OUT_OF_MEMORY; } - char name[TSDB_DB_FNAME_LEN] = {0}; - strncpy(name, pTableName->z, dbLen); - int32_t actualDbLen = strdequote(name); - - code = tNameSetDbName(pName, acctId, name, actualDbLen); - if (code != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg1); + if (TSDB_CODE_SUCCESS == code) { + dst->numOfTables = taosArrayGetSize(src->pData->aSubmitTbData); + code = taosHashGetDup(pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); } - - int32_t tbLen = pTableName->n - dbLen - 1; - if (tbLen <= 0) { - return buildInvalidOperationMsg(pMsgBuf, msg4); + if (TSDB_CODE_SUCCESS == code) { + code = buildSubmitReq(src->vgId, src->pData, &dst->pData, &dst->size); } - - char tbname[TSDB_TABLE_FNAME_LEN] = {0}; - strncpy(tbname, p + 1, tbLen); - /*tbLen = */ strdequote(tbname); - - code = tNameFromString(pName, tbname, T_NAME_TABLE); - if (code != 0) { - return buildInvalidOperationMsg(pMsgBuf, msg1); - } - } else { // get current DB name first, and then set it into path - if (pTableName->n >= TSDB_TABLE_NAME_LEN) { - return buildInvalidOperationMsg(pMsgBuf, msg1); + if (TSDB_CODE_SUCCESS == code) { + code = (NULL == taosArrayPush(pDataBlocks, &dst) ? TSDB_CODE_OUT_OF_MEMORY : TSDB_CODE_SUCCESS); } + } - assert(pTableName->n < TSDB_TABLE_FNAME_LEN); + if (TSDB_CODE_SUCCESS == code) { + *pVgDataBlocks = pDataBlocks; + } else { + taosArrayDestroyP(pDataBlocks, destroyVgDataBlocks); + } - char name[TSDB_TABLE_FNAME_LEN] = {0}; - strncpy(name, pTableName->z, pTableName->n); - strdequote(name); + return code; +} - if (dbName == NULL) { - return buildInvalidOperationMsg(pMsgBuf, msg3); - } +static int bindFileds(SBoundColInfo* pBoundInfo, SSchema* pSchema, TAOS_FIELD* fields, int numFields) { + bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool)); + if (NULL == pUseCols) { + return TSDB_CODE_OUT_OF_MEMORY; + } - code = tNameSetDbName(pName, acctId, dbName, strlen(dbName)); - if (code != TSDB_CODE_SUCCESS) { - code = buildInvalidOperationMsg(pMsgBuf, msg2); - return code; - } + pBoundInfo->numOfBound = 0; - code = tNameFromString(pName, name, T_NAME_TABLE); - if (code != 0) { - code = buildInvalidOperationMsg(pMsgBuf, msg1); + int16_t lastColIdx = -1; // last column found + int32_t code = TSDB_CODE_SUCCESS; + for (int i = 0; i < numFields; i++) { + SToken token; + token.z = fields[i].name; + token.n = strlen(fields[i].name); + + int16_t t = lastColIdx + 1; + int16_t index = insFindCol(&token, t, pBoundInfo->numOfCols, pSchema); + if (index < 0 && t > 0) { + index = insFindCol(&token, 0, t, pSchema); + } + if (index < 0) { + uError("can not find column name:%s", token.z); + code = TSDB_CODE_PAR_INVALID_COLUMN; + break; + } else if (pUseCols[index]) { + code = TSDB_CODE_PAR_INVALID_COLUMN; + uError("duplicated column name:%s", token.z); + break; + } else { + lastColIdx = index; + pUseCols[index] = true; + pBoundInfo->pColIndex[pBoundInfo->numOfBound] = index; + ++pBoundInfo->numOfBound; } } - if (NULL != strchr(pName->tname, '.')) { - code = generateSyntaxErrMsgExt(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, "The table name cannot contain '.'"); + if (TSDB_CODE_SUCCESS == code && !pUseCols[0]) { + uError("primary timestamp column can not be null:"); + code = TSDB_CODE_PAR_INVALID_COLUMN; } + taosMemoryFree(pUseCols); return code; } -int32_t insFindCol(SToken* pColname, int32_t start, int32_t end, SSchema* pSchema) { - while (start < end) { - if (strlen(pSchema[start].name) == pColname->n && strncmp(pColname->z, pSchema[start].name, pColname->n) == 0) { - return start; +int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD* tFields, + int numFields) { + STableDataCxt* pTableCxt = NULL; + int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, + sizeof(pTableMeta->uid), pTableMeta, &pCreateTb, &pTableCxt, true); + if (ret != TSDB_CODE_SUCCESS) { + uError("insGetTableDataCxt error"); + goto end; + } + if (tFields != NULL) { + ret = bindFileds(&pTableCxt->boundColsInfo, getTableColumnSchema(pTableMeta), tFields, numFields); + if (ret != TSDB_CODE_SUCCESS) { + uError("bindFileds error"); + goto end; } - ++start; } - return -1; -} + // no need to bind, because select * get all fields + ret = initTableColSubmitData(pTableCxt); + if (ret != TSDB_CODE_SUCCESS) { + uError("initTableColSubmitData error"); + goto end; + } -void insBuildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, int64_t suid, const char* sname, - SArray* tagName, uint8_t tagNum, int32_t ttl) { - pTbReq->type = TD_CHILD_TABLE; - pTbReq->name = strdup(tname); - pTbReq->ctb.suid = suid; - pTbReq->ctb.tagNum = tagNum; - if (sname) pTbReq->ctb.stbName = strdup(sname); - pTbReq->ctb.pTag = (uint8_t*)pTag; - pTbReq->ctb.tagName = taosArrayDup(tagName, NULL); - pTbReq->ttl = ttl; - pTbReq->commentLen = -1; + char* p = (char*)data; + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column + // length | + p += sizeof(int32_t); + p += sizeof(int32_t); - return; -} + int32_t numOfRows = *(int32_t*)p; + p += sizeof(int32_t); -int32_t insMemRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param) { - SMemParam* pa = (SMemParam*)param; - SRowBuilder* rb = pa->rb; + int32_t numOfCols = *(int32_t*)p; + p += sizeof(int32_t); - if (value == NULL) { // it is a null data - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, pa->colIdx); - return TSDB_CODE_SUCCESS; - } + p += sizeof(int32_t); + p += sizeof(uint64_t); - if (TSDB_DATA_TYPE_BINARY == pa->schema->type) { - const char* rowEnd = tdRowEnd(rb->pBuf); - STR_WITH_SIZE_TO_VARSTR(rowEnd, value, len); - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, false, pa->toffset, pa->colIdx); - } else if (TSDB_DATA_TYPE_NCHAR == pa->schema->type) { - // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' - int32_t output = 0; - const char* rowEnd = tdRowEnd(rb->pBuf); - if (!taosMbsToUcs4(value, len, (TdUcs4*)varDataVal(rowEnd), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { - if (errno == E2BIG) { - return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pa->schema->name); - } - char buf[512] = {0}; - snprintf(buf, tListLen(buf), "%s", strerror(errno)); - return buildSyntaxErrMsg(pMsgBuf, buf, value); - } - varDataSetLen(rowEnd, output); - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, false, pa->toffset, pa->colIdx); - } else { - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, value, false, pa->toffset, pa->colIdx); - } + int8_t* fields = p; + p += numOfCols * (sizeof(int8_t) + sizeof(int32_t)); - return TSDB_CODE_SUCCESS; -} + int32_t* colLength = (int32_t*)p; + p += sizeof(int32_t) * numOfCols; -int32_t insCheckTimestamp(STableDataBlocks* pDataBlocks, const char* start) { - // once the data block is disordered, we do NOT keep previous timestamp any more - if (!pDataBlocks->ordered) { - return TSDB_CODE_SUCCESS; - } - - TSKEY k = *(TSKEY*)start; - if (k <= pDataBlocks->prevTS) { - pDataBlocks->ordered = false; - } + char* pStart = p; - pDataBlocks->prevTS = k; - return TSDB_CODE_SUCCESS; -} + SSchema* pSchema = getTableColumnSchema(pTableCxt->pMeta); + SBoundColInfo* boundInfo = &pTableCxt->boundColsInfo; -static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { - SSubmitReq* submit = (SSubmitReq*)blocks->pData; - submit->header.vgId = htonl(blocks->vg.vgId); - submit->header.contLen = htonl(blocks->size); - submit->length = submit->header.contLen; - submit->numOfBlocks = htonl(blocks->numOfTables); - SSubmitBlk* blk = (SSubmitBlk*)(submit + 1); - int32_t numOfBlocks = blocks->numOfTables; - while (numOfBlocks--) { - int32_t dataLen = blk->dataLen; - int32_t schemaLen = blk->schemaLen; - blk->uid = htobe64(blk->uid); - blk->suid = htobe64(blk->suid); - blk->sversion = htonl(blk->sversion); - blk->dataLen = htonl(blk->dataLen); - blk->schemaLen = htonl(blk->schemaLen); - blk->numOfRows = htonl(blk->numOfRows); - blk = (SSubmitBlk*)(blk->data + schemaLen + dataLen); + if (boundInfo->numOfBound != numOfCols) { + uError("boundInfo->numOfBound:%d != numOfCols:%d", boundInfo->numOfBound, numOfCols); + ret = TSDB_CODE_INVALID_PARA; + goto end; } -} + for (int c = 0; c < boundInfo->numOfBound; ++c) { + SSchema* pColSchema = &pSchema[boundInfo->pColIndex[c]]; + SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, c); -int32_t insBuildOutput(SHashObj* pVgroupsHashObj, SArray* pVgDataBlocks, SArray** pDataBlocks) { - size_t numOfVg = taosArrayGetSize(pVgDataBlocks); - *pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); - if (NULL == *pDataBlocks) { - return TSDB_CODE_OUT_OF_MEMORY; - } - for (size_t i = 0; i < numOfVg; ++i) { - STableDataBlocks* src = taosArrayGetP(pVgDataBlocks, i); - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - return TSDB_CODE_OUT_OF_MEMORY; + if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { + uError("type or bytes not equal"); + ret = TSDB_CODE_INVALID_PARA; + goto end; + } + + colLength[c] = htonl(colLength[c]); + int8_t* offset = pStart; + if (IS_VAR_DATA_TYPE(pColSchema->type)) { + pStart += numOfRows * sizeof(int32_t); + } else { + pStart += BitmapLen(numOfRows); } - taosHashGetDup(pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); - dst->numOfTables = src->numOfTables; - dst->size = src->size; - TSWAP(dst->pData, src->pData); - buildMsgHeader(src, dst); - taosArrayPush(*pDataBlocks, &dst); + char* pData = pStart; + + tColDataAddValueByDataBlock(pCol, pColSchema->type, pColSchema->bytes, numOfRows, offset, pData); + fields += sizeof(int8_t) + sizeof(int32_t); + pStart += colLength[c]; } - return TSDB_CODE_SUCCESS; -} + +end: + return ret; +} \ No newline at end of file diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 94b32a3de219d52a1a90e36531979c3487ca28d3..93bfe179f486775111d0f5aa9781b585cb583057 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -269,6 +269,7 @@ static SKeyword keywordTable[] = { {"_WDURATION", TK_WDURATION}, {"_WEND", TK_WEND}, {"_WSTART", TK_WSTART}, + {"ALIVE", TK_ALIVE}, }; // clang-format on @@ -714,14 +715,3 @@ void taosCleanupKeywordsTable() { taosHashCleanup(m); } } - -SToken taosTokenDup(SToken* pToken, char* buf, int32_t len) { - assert(pToken != NULL && buf != NULL && len > pToken->n); - - strncpy(buf, pToken->z, pToken->n); - buf[pToken->n] = 0; - - SToken token = *pToken; - token.z = buf; - return token; -} diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 45eacad8557477f8c3dcc6654f8134ee2a3118b8..a9c6d42cd456ecfc1f832e1e61b9e72c1d73f25e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -352,7 +352,7 @@ static int32_t getTableMetaImpl(STranslateContext* pCxt, const SName* pName, STa code = catalogGetTableMeta(pParCxt->pCatalog, &conn, pName, pMeta); } } - if (TSDB_CODE_SUCCESS != code && TSDB_CODE_TSC_INVALID_TABLE_NAME != code) { + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_PAR_TABLE_NOT_EXIST != code) { parserError("0x%" PRIx64 " catalogGetTableMeta error, code:%s, dbName:%s, tbName:%s", pCxt->pParseCxt->requestId, tstrerror(code), pName->dbname, pName->tname); } @@ -2210,7 +2210,7 @@ static int32_t dnodeToVgroupsInfo(SArray* pDnodes, SVgroupsInfo** pVgsInfo) { } static bool sysTableFromVnode(const char* pTable) { - return ((0 == strcmp(pTable, TSDB_INS_TABLE_TABLES)) || (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS))); + return ((0 == strcmp(pTable, TSDB_INS_TABLE_TABLES)) || (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS)) || (0 == strcmp(pTable, TSDB_INS_TABLE_COLS))); } static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES); } @@ -2278,7 +2278,8 @@ static int32_t setVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, ((SSelectStmt*)pCxt->pCurrStmt)->isEmptyResult = true; } - if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) && !hasUserDbCond) { + if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) && !hasUserDbCond) || + 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_COLS)) { code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &pVgs); } @@ -2376,7 +2377,8 @@ static bool isSingleTable(SRealTableNode* pRealTable) { int8_t tableType = pRealTable->pMeta->tableType; if (TSDB_SYSTEM_TABLE == tableType) { return 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) && - 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS); + 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS) && + 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_COLS); } return (TSDB_CHILD_TABLE == tableType || TSDB_NORMAL_TABLE == tableType); } @@ -3902,7 +3904,8 @@ static int32_t checkDbKeepOption(STranslateContext* pCxt, SDatabaseOptions* pOpt if (pOptions->keep[0] < TSDB_MIN_KEEP || pOptions->keep[1] < TSDB_MIN_KEEP || pOptions->keep[2] < TSDB_MIN_KEEP || pOptions->keep[0] > tsdbMaxKeep || pOptions->keep[1] > tsdbMaxKeep || pOptions->keep[2] > tsdbMaxKeep) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, - "Invalid option keep: %" PRId64 ", %" PRId64 ", %" PRId64 " valid range: [%dm, %dm]", + "Invalid option keep: %" PRId64 ", %" PRId64 ", %" PRId64 + " valid range: [%dm, %" PRId64 "m]", pOptions->keep[0], pOptions->keep[1], pOptions->keep[2], TSDB_MIN_KEEP, tsdbMaxKeep); } @@ -4569,58 +4572,109 @@ typedef struct SSampleAstInfo { SNode* pSliding; SNodeList* pPartitionByList; STableMeta* pRollupTableMeta; + bool createSmaIndex; } SSampleAstInfo; -static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, char** pAst, int32_t* pLen, char** pExpr, - int32_t* pExprLen) { - SSelectStmt* pSelect = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); - if (NULL == pSelect) { - return TSDB_CODE_OUT_OF_MEMORY; - } - sprintf(pSelect->stmtName, "%p", pSelect); - +static int32_t buildTableForSampleAst(SSampleAstInfo* pInfo, SNode** pOutput) { SRealTableNode* pTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); if (NULL == pTable) { - nodesDestroyNode((SNode*)pSelect); return TSDB_CODE_OUT_OF_MEMORY; } snprintf(pTable->table.dbName, sizeof(pTable->table.dbName), "%s", pInfo->pDbName); snprintf(pTable->table.tableName, sizeof(pTable->table.tableName), "%s", pInfo->pTableName); TSWAP(pTable->pMeta, pInfo->pRollupTableMeta); - pSelect->pFromTable = (SNode*)pTable; + *pOutput = (SNode*)pTable; + return TSDB_CODE_SUCCESS; +} - TSWAP(pSelect->pProjectionList, pInfo->pFuncs); +static int32_t addWstartToSampleProjects(SNodeList* pProjectionList) { SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); - if (NULL == pSelect->pProjectionList || NULL == pFunc) { - nodesDestroyNode((SNode*)pSelect); + if (NULL == pFunc) { return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pFunc->functionName, "_wstart"); - nodesListPushFront(pSelect->pProjectionList, (SNode*)pFunc); - SNode* pProject = NULL; - FOREACH(pProject, pSelect->pProjectionList) { sprintf(((SExprNode*)pProject)->aliasName, "#%p", pProject); } + return nodesListPushFront(pProjectionList, (SNode*)pFunc); +} - TSWAP(pSelect->pPartitionByList, pInfo->pPartitionByList); +static int32_t addWendToSampleProjects(SNodeList* pProjectionList) { + SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(pFunc->functionName, "_wend"); + return nodesListAppend(pProjectionList, (SNode*)pFunc); +} +static int32_t addWdurationToSampleProjects(SNodeList* pProjectionList) { + SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(pFunc->functionName, "_wduration"); + return nodesListAppend(pProjectionList, (SNode*)pFunc); +} + +static int32_t buildProjectsForSampleAst(SSampleAstInfo* pInfo, SNodeList** pList) { + SNodeList* pProjectionList = pInfo->pFuncs; + pInfo->pFuncs = NULL; + + int32_t code = addWstartToSampleProjects(pProjectionList); + if (TSDB_CODE_SUCCESS == code && pInfo->createSmaIndex) { + code = addWendToSampleProjects(pProjectionList); + if (TSDB_CODE_SUCCESS == code) { + code = addWdurationToSampleProjects(pProjectionList); + } + } + + if (TSDB_CODE_SUCCESS == code) { + SNode* pProject = NULL; + FOREACH(pProject, pProjectionList) { sprintf(((SExprNode*)pProject)->aliasName, "#%p", pProject); } + *pList = pProjectionList; + } else { + nodesDestroyList(pProjectionList); + } + return code; +} + +static int32_t buildIntervalForSampleAst(SSampleAstInfo* pInfo, SNode** pOutput) { SIntervalWindowNode* pInterval = (SIntervalWindowNode*)nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW); if (NULL == pInterval) { - nodesDestroyNode((SNode*)pSelect); return TSDB_CODE_OUT_OF_MEMORY; } - pSelect->pWindow = (SNode*)pInterval; TSWAP(pInterval->pInterval, pInfo->pInterval); TSWAP(pInterval->pOffset, pInfo->pOffset); TSWAP(pInterval->pSliding, pInfo->pSliding); pInterval->pCol = nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pInterval->pCol) { - nodesDestroyNode((SNode*)pSelect); + nodesDestroyNode((SNode*)pInterval); return TSDB_CODE_OUT_OF_MEMORY; } ((SColumnNode*)pInterval->pCol)->colId = PRIMARYKEY_TIMESTAMP_COL_ID; strcpy(((SColumnNode*)pInterval->pCol)->colName, ROWTS_PSEUDO_COLUMN_NAME); + *pOutput = (SNode*)pInterval; + return TSDB_CODE_SUCCESS; +} - pCxt->createStream = true; - int32_t code = translateQuery(pCxt, (SNode*)pSelect); +static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, char** pAst, int32_t* pLen, char** pExpr, + int32_t* pExprLen) { + SSelectStmt* pSelect = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); + if (NULL == pSelect) { + return TSDB_CODE_OUT_OF_MEMORY; + } + sprintf(pSelect->stmtName, "%p", pSelect); + + int32_t code = buildTableForSampleAst(pInfo, &pSelect->pFromTable); + if (TSDB_CODE_SUCCESS == code) { + code = buildProjectsForSampleAst(pInfo, &pSelect->pProjectionList); + } + if (TSDB_CODE_SUCCESS == code) { + TSWAP(pSelect->pPartitionByList, pInfo->pPartitionByList); + code = buildIntervalForSampleAst(pInfo, &pSelect->pWindow); + } + if (TSDB_CODE_SUCCESS == code) { + pCxt->createStream = true; + code = translateQuery(pCxt, (SNode*)pSelect); + } if (TSDB_CODE_SUCCESS == code) { code = nodesNodeToString((SNode*)pSelect, false, pAst, pLen); } @@ -4928,10 +4982,10 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* return TSDB_CODE_SUCCESS; } -static SSchema* getColSchema(STableMeta* pTableMeta, const char* pColName) { +static const SSchema* getColSchema(const STableMeta* pTableMeta, const char* pColName) { int32_t numOfFields = getNumOfTags(pTableMeta) + getNumOfColumns(pTableMeta); for (int32_t i = 0; i < numOfFields; ++i) { - SSchema* pSchema = pTableMeta->schema + i; + const SSchema* pSchema = pTableMeta->schema + i; if (0 == strcmp(pColName, pSchema->name)) { return pSchema; } @@ -4951,7 +5005,8 @@ static SSchema* getTagSchema(STableMeta* pTableMeta, const char* pTagName) { return NULL; } -static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta) { +static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTableStmt* pStmt, + const STableMeta* pTableMeta) { SSchema* pTagsSchema = getTableTagSchema(pTableMeta); if (getNumOfTags(pTableMeta) == 1 && pTagsSchema->type == TSDB_DATA_TYPE_JSON && (pStmt->alterType == TSDB_ALTER_TABLE_ADD_TAG || pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG || @@ -4959,6 +5014,10 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG); } + if (getNumOfTags(pTableMeta) == 1 && pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "the only tag cannot be dropped"); + } + int32_t tagsLen = 0; for (int32_t i = 0; i < pTableMeta->tableInfo.numOfTags; ++i) { tagsLen += pTagsSchema[i].bytes; @@ -4970,7 +5029,7 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "Table is not super table"); } - SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); + const SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); if (NULL == pSchema) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); } else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type || @@ -5162,6 +5221,7 @@ static int32_t getSmaIndexSql(STranslateContext* pCxt, char** pSql, int32_t* pLe } static int32_t buildSampleAstInfoByIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SSampleAstInfo* pInfo) { + pInfo->createSmaIndex = true; pInfo->pDbName = pStmt->dbName; pInfo->pTableName = pStmt->tableName; pInfo->pFuncs = nodesCloneList(pStmt->pOptions->pFuncs); @@ -5534,7 +5594,8 @@ static void getStreamQueryFirstProjectAliasName(SHashObj* pUserAliasSet, char* a return; } -static int32_t addWstartTsToCreateStreamQueryImpl(SSelectStmt* pSelect, SHashObj* pUserAliasSet) { +static int32_t addWstartTsToCreateStreamQueryImpl(STranslateContext* pCxt, SSelectStmt* pSelect, + SHashObj* pUserAliasSet) { SNode* pProj = nodesListGetNode(pSelect->pProjectionList, 0); if (NULL == pSelect->pWindow || (QUERY_NODE_FUNCTION == nodeType(pProj) && 0 == strcmp("_wstart", ((SFunctionNode*)pProj)->functionName))) { @@ -5546,7 +5607,10 @@ static int32_t addWstartTsToCreateStreamQueryImpl(SSelectStmt* pSelect, SHashObj } strcpy(pFunc->functionName, "_wstart"); getStreamQueryFirstProjectAliasName(pUserAliasSet, pFunc->node.aliasName, sizeof(pFunc->node.aliasName)); - int32_t code = nodesListPushFront(pSelect->pProjectionList, (SNode*)pFunc); + int32_t code = getFuncInfo(pCxt, pFunc); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListPushFront(pSelect->pProjectionList, (SNode*)pFunc); + } if (TSDB_CODE_SUCCESS != code) { nodesDestroyNode((SNode*)pFunc); } @@ -5558,7 +5622,7 @@ static int32_t addWstartTsToCreateStreamQuery(STranslateContext* pCxt, SNode* pS SHashObj* pUserAliasSet = NULL; int32_t code = checkProjectAlias(pCxt, pSelect->pProjectionList, &pUserAliasSet); if (TSDB_CODE_SUCCESS == code) { - code = addWstartTsToCreateStreamQueryImpl(pSelect, pUserAliasSet); + code = addWstartTsToCreateStreamQueryImpl(pCxt, pSelect, pUserAliasSet); } taosHashCleanup(pUserAliasSet); return code; @@ -5651,7 +5715,8 @@ static int32_t addSubtableInfoToCreateStreamQuery(STranslateContext* pCxt, SCrea return code; } -static int32_t checkStreamQuery(STranslateContext* pCxt, SSelectStmt* pSelect) { +static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt) { + SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; if (TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type || !pSelect->isTimeLineResult || crossTableWithoutAggOper(pSelect) || NULL != pSelect->pOrderByList || crossTableWithUdaf(pSelect)) { @@ -5661,20 +5726,165 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SSelectStmt* pSelect) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "SUBTABLE expression must be of VARCHAR type"); } + if (NULL == pSelect->pWindow && STREAM_TRIGGER_AT_ONCE != pStmt->pOptions->triggerType) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "The trigger mode of non window query can only be AT_ONCE"); + } return TSDB_CODE_SUCCESS; } +static int32_t adjustDataTypeOfProjections(STranslateContext* pCxt, const STableMeta* pMeta, SNodeList* pProjections) { + if (getNumOfColumns(pMeta) != LIST_LENGTH(pProjections)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM, "Illegal number of columns"); + } + + SSchema* pSchemas = getTableColumnSchema(pMeta); + int32_t index = 0; + SNode* pProj = NULL; + FOREACH(pProj, pProjections) { + SSchema* pSchema = pSchemas + index++; + SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes}; + if (!dataTypeEqual(&dt, &((SExprNode*)pProj)->resType)) { + SNode* pFunc = NULL; + int32_t code = createCastFunc(pCxt, pProj, dt, &pFunc); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + REPLACE_NODE(pFunc); + } + } + + return TSDB_CODE_SUCCESS; +} + +typedef struct SProjColPos { + int32_t colId; + SNode* pProj; +} SProjColPos; + +static int32_t projColPosCompar(const void* l, const void* r) { + return ((SProjColPos*)l)->colId > ((SProjColPos*)r)->colId; +} + +static void projColPosDelete(void* p) { taosMemoryFree(((SProjColPos*)p)->pProj); } + +static int32_t addProjToProjColPos(STranslateContext* pCxt, const SSchema* pSchema, SNode* pProj, SArray* pProjColPos) { + SNode* pNewProj = nodesCloneNode(pProj); + if (NULL == pNewProj) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes}; + if (!dataTypeEqual(&dt, &((SExprNode*)pNewProj)->resType)) { + SNode* pFunc = NULL; + code = createCastFunc(pCxt, pNewProj, dt, &pFunc); + pNewProj = pFunc; + } + if (TSDB_CODE_SUCCESS == code) { + SProjColPos pos = {.colId = pSchema->colId, .pProj = pNewProj}; + code = (NULL == taosArrayPush(pProjColPos, &pos) ? TSDB_CODE_OUT_OF_MEMORY : TSDB_CODE_SUCCESS); + } + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pNewProj); + } + return code; +} + +static int32_t adjustOrderOfProjection(STranslateContext* pCxt, SNodeList* pCols, const STableMeta* pMeta, + SNodeList** pProjections) { + if (LIST_LENGTH(pCols) != LIST_LENGTH(*pProjections)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM, "Illegal number of columns"); + } + + SArray* pProjColPos = taosArrayInit(LIST_LENGTH(pCols), sizeof(SProjColPos)); + if (NULL == pProjColPos) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + SNode* pCol = NULL; + SNode* pProj = NULL; + FORBOTH(pCol, pCols, pProj, *pProjections) { + const SSchema* pSchema = getColSchema(pMeta, ((SColumnNode*)pCol)->colName); + code = addProjToProjColPos(pCxt, pSchema, pProj, pProjColPos); + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + + SNodeList* pNewProjections = NULL; + if (TSDB_CODE_SUCCESS == code) { + taosArraySort(pProjColPos, projColPosCompar); + int32_t num = taosArrayGetSize(pProjColPos); + pNewProjections = nodesMakeList(); + if (NULL == pNewProjections) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + for (int32_t i = 0; TSDB_CODE_SUCCESS == code && i < num; ++i) { + SProjColPos* pPos = taosArrayGet(pProjColPos, i); + code = nodesListStrictAppend(pNewProjections, pPos->pProj); + pPos->pProj = NULL; + } + } + + if (TSDB_CODE_SUCCESS == code) { + taosArrayDestroy(pProjColPos); + nodesDestroyList(*pProjections); + *pProjections = pNewProjections; + } else { + taosArrayDestroyEx(pProjColPos, projColPosDelete); + nodesDestroyList(pNewProjections); + } + + return code; +} + +static int32_t adjustStreamQueryForExistTableImpl(STranslateContext* pCxt, SCreateStreamStmt* pStmt, + const STableMeta* pMeta) { + SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; + if (NULL == pStmt->pCols) { + return adjustDataTypeOfProjections(pCxt, pMeta, pSelect->pProjectionList); + } + return adjustOrderOfProjection(pCxt, pStmt->pCols, pMeta, &pSelect->pProjectionList); +} + +static int32_t adjustStreamQueryForExistTable(STranslateContext* pCxt, SCreateStreamStmt* pStmt, + SCMCreateStreamReq* pReq) { + STableMeta* pMeta = NULL; + int32_t code = getTableMeta(pCxt, pStmt->targetDbName, pStmt->targetTabName, &pMeta); + if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { + if (NULL != pStmt->pCols) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TABLE_NOT_EXIST, pStmt->targetTabName); + } + pReq->createStb = STREAM_CREATE_STABLE_TRUE; + pReq->targetStbUid = 0; + return TSDB_CODE_SUCCESS; + } else { + pReq->createStb = STREAM_CREATE_STABLE_FALSE; + pReq->targetStbUid = pMeta->suid; + } + if (TSDB_CODE_SUCCESS == code) { + code = adjustStreamQueryForExistTableImpl(pCxt, pStmt, pMeta); + } + taosMemoryFree(pMeta); + return code; +} + static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SCMCreateStreamReq* pReq) { pCxt->createStream = true; - int32_t code = addWstartTsToCreateStreamQuery(pCxt, pStmt->pQuery); + int32_t code = addSubtableInfoToCreateStreamQuery(pCxt, pStmt); if (TSDB_CODE_SUCCESS == code) { - code = addSubtableInfoToCreateStreamQuery(pCxt, pStmt); + code = translateQuery(pCxt, pStmt->pQuery); } if (TSDB_CODE_SUCCESS == code) { - code = translateQuery(pCxt, pStmt->pQuery); + code = addWstartTsToCreateStreamQuery(pCxt, pStmt->pQuery); } if (TSDB_CODE_SUCCESS == code) { - code = checkStreamQuery(pCxt, (SSelectStmt*)pStmt->pQuery); + code = checkStreamQuery(pCxt, pStmt); + } + if (TSDB_CODE_SUCCESS == code) { + code = adjustStreamQueryForExistTable(pCxt, pStmt, pReq); } if (TSDB_CODE_SUCCESS == code) { getSourceDatabase(pStmt->pQuery, pCxt->pParseCxt->acctId, pReq->sourceDB); @@ -6170,6 +6380,20 @@ static int32_t extractShowCreateDatabaseResultSchema(int32_t* numOfCols, SSchema return TSDB_CODE_SUCCESS; } +static int32_t extractShowAliveResultSchema(int32_t* numOfCols, SSchema** pSchema) { + *numOfCols = 1; + *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); + if (NULL == (*pSchema)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + (*pSchema)[0].type = TSDB_DATA_TYPE_INT; + (*pSchema)[0].bytes = sizeof(int32_t); + strcpy((*pSchema)[0].name, "status"); + + return TSDB_CODE_SUCCESS; +} + static int32_t extractShowCreateTableResultSchema(int32_t* numOfCols, SSchema** pSchema) { *numOfCols = 2; *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); @@ -6221,6 +6445,9 @@ int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pS return extractDescribeResultSchema(numOfCols, pSchema); case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: return extractShowCreateDatabaseResultSchema(numOfCols, pSchema); + case QUERY_NODE_SHOW_DB_ALIVE_STMT: + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: + return extractShowAliveResultSchema(numOfCols, pSchema); case QUERY_NODE_SHOW_CREATE_TABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT: return extractShowCreateTableResultSchema(numOfCols, pSchema); @@ -6325,7 +6552,7 @@ static int32_t createOperatorNode(EOperatorType opType, const char* pColName, SN nodesDestroyNode((SNode*)pOper); return TSDB_CODE_OUT_OF_MEMORY; } - strcpy(((SColumnNode*)pOper->pLeft)->colName, pColName); + snprintf(((SColumnNode*)pOper->pLeft)->colName, sizeof(((SColumnNode*)pOper->pLeft)->colName), "%s", pColName); *pOp = (SNode*)pOper; return TSDB_CODE_SUCCESS; @@ -7299,12 +7526,12 @@ static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, S return TSDB_CODE_SUCCESS; } -static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, +static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, const STableMeta* pTableMeta, SVAlterTbReq* pReq) { if (2 == getNumOfColumns(pTableMeta)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DROP_COL); } - SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); + const SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); if (NULL == pSchema) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); } else if (PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { @@ -7320,11 +7547,11 @@ static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, return TSDB_CODE_SUCCESS; } -static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, +static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, const STableMeta* pTableMeta, SVAlterTbReq* pReq) { pReq->colModBytes = calcTypeBytes(pStmt->dataType); pReq->colModType = pStmt->dataType.type; - SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); + const SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); if (NULL == pSchema) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); } else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type || @@ -7488,7 +7715,7 @@ static void destoryAlterTbReq(SVAlterTbReq* pReq) { static int32_t rewriteAlterTableImpl(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, SQuery* pQuery) { if (getNumOfTags(pTableMeta) == 1 && pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "the only tag cannot be dropped"); } if (TSDB_SUPER_TABLE == pTableMeta->tableType) { @@ -7740,6 +7967,8 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { break; case QUERY_NODE_DESCRIBE_STMT: case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: + case QUERY_NODE_SHOW_DB_ALIVE_STMT: + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: case QUERY_NODE_SHOW_CREATE_TABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT: case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 1021aab6f099e7f65201684ab07ef5412d47ab60..7460664de2c955ff60f08faf6b739942ac914785 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -104,26 +104,26 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 461 +#define YYNOCODE 465 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SToken typedef union { int yyinit; ParseTOKENTYPE yy0; - EOrder yy32; - SToken yy77; - int32_t yy248; - int8_t yy287; - ENullOrder yy385; - EJoinType yy560; - SNode* yy600; - SNodeList* yy601; - SAlterOption yy661; - EOperatorType yy666; - int64_t yy717; - EFillMode yy798; - bool yy841; - SDataType yy888; + EJoinType yy42; + int8_t yy113; + int64_t yy159; + SToken yy179; + EOperatorType yy290; + EFillMode yy324; + SDataType yy394; + ENullOrder yy487; + SNode* yy602; + bool yy767; + int32_t yy820; + SAlterOption yy845; + SNodeList* yy874; + EOrder yy878; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -139,18 +139,17 @@ typedef union { #define ParseCTX_FETCH #define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 714 -#define YYNRULE 545 -#define YYNRULE_WITH_ACTION 545 -#define YYNTOKEN 325 -#define YY_MAX_SHIFT 713 -#define YY_MIN_SHIFTREDUCE 1061 -#define YY_MAX_SHIFTREDUCE 1605 -#define YY_ERROR_ACTION 1606 -#define YY_ACCEPT_ACTION 1607 -#define YY_NO_ACTION 1608 -#define YY_MIN_REDUCE 1609 -#define YY_MAX_REDUCE 2153 +#define YYNSTATE 730 +#define YYNRULE 553 +#define YYNTOKEN 326 +#define YY_MAX_SHIFT 729 +#define YY_MIN_SHIFTREDUCE 1081 +#define YY_MAX_SHIFTREDUCE 1633 +#define YY_ERROR_ACTION 1634 +#define YY_ACCEPT_ACTION 1635 +#define YY_NO_ACTION 1636 +#define YY_MIN_REDUCE 1637 +#define YY_MAX_REDUCE 2189 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -217,843 +216,740 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (3069) +#define YY_ACTTAB_COUNT (2706) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 35, 276, 460, 1876, 461, 1645, 469, 367, 461, 1645, - /* 10 */ 1808, 1810, 45, 43, 1535, 1954, 1874, 590, 1751, 466, - /* 20 */ 362, 407, 1385, 38, 37, 462, 1950, 44, 42, 41, - /* 30 */ 40, 39, 169, 1465, 459, 1383, 229, 464, 1651, 1792, - /* 40 */ 571, 602, 38, 37, 2124, 602, 44, 42, 41, 40, - /* 50 */ 39, 8, 602, 334, 1862, 1946, 1952, 345, 1460, 570, - /* 60 */ 175, 320, 187, 18, 2125, 572, 613, 38, 37, 1967, - /* 70 */ 1391, 44, 42, 41, 40, 39, 468, 1411, 583, 464, - /* 80 */ 1651, 38, 37, 45, 43, 44, 42, 41, 40, 39, - /* 90 */ 2129, 362, 171, 1385, 1609, 14, 136, 327, 81, 2026, - /* 100 */ 1985, 80, 60, 27, 1465, 1802, 1383, 1412, 586, 134, - /* 110 */ 159, 1572, 1621, 1936, 603, 619, 48, 710, 125, 124, - /* 120 */ 123, 122, 121, 120, 119, 118, 117, 48, 126, 1460, - /* 130 */ 100, 160, 1467, 1468, 18, 499, 1716, 443, 1494, 1966, - /* 140 */ 64, 1391, 478, 2002, 135, 1762, 103, 1968, 623, 1970, - /* 150 */ 1971, 618, 1754, 613, 1410, 2129, 1815, 146, 172, 2124, - /* 160 */ 2055, 1440, 1450, 355, 356, 2051, 14, 1466, 1469, 257, - /* 170 */ 2063, 582, 1813, 127, 581, 2128, 566, 2124, 177, 2125, - /* 180 */ 2127, 2129, 1386, 1595, 1384, 2124, 2081, 528, 710, 1263, - /* 190 */ 1264, 260, 570, 175, 1495, 191, 190, 2125, 572, 49, - /* 200 */ 526, 2128, 524, 1467, 1468, 2125, 2126, 1389, 1390, 53, - /* 210 */ 1439, 1442, 1443, 1444, 1445, 1446, 1447, 1448, 1449, 615, - /* 220 */ 611, 1458, 1459, 1461, 1462, 1463, 1464, 2, 60, 60, - /* 230 */ 89, 60, 1440, 1450, 2070, 1410, 156, 116, 1466, 1469, - /* 240 */ 115, 114, 113, 112, 111, 110, 109, 108, 107, 648, - /* 250 */ 259, 227, 178, 1386, 2070, 1384, 1528, 38, 37, 1632, - /* 260 */ 2067, 44, 42, 41, 40, 39, 34, 360, 1489, 1490, - /* 270 */ 1491, 1492, 1493, 1497, 1498, 1499, 1500, 178, 1389, 1390, - /* 280 */ 2066, 1439, 1442, 1443, 1444, 1445, 1446, 1447, 1448, 1449, - /* 290 */ 615, 611, 1458, 1459, 1461, 1462, 1463, 1464, 2, 396, - /* 300 */ 11, 45, 43, 1936, 1954, 1385, 1539, 1631, 1985, 362, - /* 310 */ 408, 1385, 1410, 86, 322, 1950, 565, 532, 1383, 530, - /* 320 */ 398, 394, 1465, 409, 1383, 1220, 645, 644, 643, 1224, - /* 330 */ 642, 1226, 1227, 641, 1229, 638, 1678, 1235, 635, 1237, - /* 340 */ 1238, 632, 629, 213, 1946, 1952, 357, 1460, 1180, 178, - /* 350 */ 401, 1936, 18, 1391, 1815, 613, 564, 1532, 164, 1391, - /* 360 */ 1113, 366, 1112, 1410, 495, 491, 487, 483, 210, 1630, - /* 370 */ 1813, 1629, 45, 43, 1470, 212, 1316, 1317, 178, 178, - /* 380 */ 362, 178, 1385, 1182, 14, 44, 42, 41, 40, 39, - /* 390 */ 657, 1114, 547, 1465, 1409, 1383, 2124, 1113, 1628, 1112, - /* 400 */ 710, 514, 513, 512, 85, 1094, 710, 208, 1610, 131, - /* 410 */ 508, 2130, 175, 1936, 507, 1936, 2125, 572, 1460, 506, - /* 420 */ 511, 1467, 1468, 1602, 1627, 505, 267, 268, 1114, 116, - /* 430 */ 1391, 266, 115, 114, 113, 112, 111, 110, 109, 108, - /* 440 */ 107, 583, 1936, 1411, 1096, 1626, 1099, 1100, 1391, 1876, - /* 450 */ 1440, 1450, 603, 1441, 1815, 46, 1466, 1469, 11, 353, - /* 460 */ 9, 331, 1873, 590, 603, 1386, 54, 1384, 1936, 575, - /* 470 */ 1813, 1386, 134, 1384, 207, 201, 1625, 710, 180, 206, - /* 480 */ 38, 37, 474, 1762, 44, 42, 41, 40, 39, 1936, - /* 490 */ 1389, 1390, 1467, 1468, 84, 1762, 1389, 1390, 199, 1439, - /* 500 */ 1442, 1443, 1444, 1445, 1446, 1447, 1448, 1449, 615, 611, - /* 510 */ 1458, 1459, 1461, 1462, 1463, 1464, 2, 1758, 1601, 1412, - /* 520 */ 1936, 1440, 1450, 583, 1142, 1624, 571, 1466, 1469, 603, - /* 530 */ 2124, 585, 173, 2063, 2064, 603, 132, 2068, 1413, 514, - /* 540 */ 513, 512, 1386, 405, 1384, 570, 175, 131, 508, 406, - /* 550 */ 2125, 572, 507, 1623, 134, 1330, 1331, 506, 511, 1143, - /* 560 */ 1762, 1967, 1620, 505, 1809, 1810, 1762, 1389, 1390, 1936, - /* 570 */ 1439, 1442, 1443, 1444, 1445, 1446, 1447, 1448, 1449, 615, - /* 580 */ 611, 1458, 1459, 1461, 1462, 1463, 1464, 2, 45, 43, - /* 590 */ 1329, 1332, 1985, 1740, 84, 354, 362, 1936, 1385, 1531, - /* 600 */ 620, 1607, 605, 157, 2027, 1936, 1936, 619, 130, 1465, - /* 610 */ 228, 1383, 1764, 11, 174, 2063, 2064, 1757, 132, 2068, - /* 620 */ 38, 37, 1967, 2070, 44, 42, 41, 40, 39, 519, - /* 630 */ 561, 1966, 1413, 603, 1460, 2002, 669, 1562, 103, 1968, - /* 640 */ 623, 1970, 1971, 618, 529, 613, 1391, 126, 137, 2065, - /* 650 */ 143, 2026, 2055, 1985, 504, 478, 356, 2051, 226, 45, - /* 660 */ 43, 620, 259, 603, 1762, 377, 1936, 362, 619, 1385, - /* 670 */ 607, 46, 2027, 522, 1619, 1362, 1363, 415, 516, 1618, - /* 680 */ 1465, 184, 1383, 225, 1753, 60, 558, 1560, 1561, 1563, - /* 690 */ 1564, 576, 1966, 710, 1762, 1950, 2002, 510, 509, 161, - /* 700 */ 1968, 623, 1970, 1971, 618, 1460, 613, 547, 1467, 1468, - /* 710 */ 1738, 2124, 542, 368, 567, 562, 556, 1391, 1936, 67, - /* 720 */ 1739, 157, 66, 1936, 1946, 1952, 2130, 175, 1475, 1747, - /* 730 */ 1764, 2125, 572, 1617, 1410, 613, 649, 1440, 1450, 1806, - /* 740 */ 548, 2092, 14, 1466, 1469, 603, 38, 37, 1496, 1749, - /* 750 */ 44, 42, 41, 40, 39, 31, 169, 2128, 1386, 429, - /* 760 */ 1384, 38, 37, 503, 710, 44, 42, 41, 40, 39, - /* 770 */ 1745, 400, 657, 399, 1616, 237, 1762, 1936, 1863, 1467, - /* 780 */ 1468, 1737, 1615, 1389, 1390, 502, 1439, 1442, 1443, 1444, - /* 790 */ 1445, 1446, 1447, 1448, 1449, 615, 611, 1458, 1459, 1461, - /* 800 */ 1462, 1463, 1464, 2, 41, 40, 39, 603, 1440, 1450, - /* 810 */ 319, 50, 1408, 3, 1466, 1469, 365, 94, 1936, 437, - /* 820 */ 32, 430, 450, 232, 157, 449, 1936, 655, 614, 1386, - /* 830 */ 1501, 1384, 1614, 1764, 1551, 178, 681, 679, 1762, 1755, - /* 840 */ 421, 1955, 451, 647, 157, 423, 148, 147, 652, 651, - /* 850 */ 650, 145, 1950, 1765, 1389, 1390, 1410, 1439, 1442, 1443, - /* 860 */ 1444, 1445, 1446, 1447, 1448, 1449, 615, 611, 1458, 1459, - /* 870 */ 1461, 1462, 1463, 1464, 2, 1441, 1936, 670, 1413, 1732, - /* 880 */ 158, 1946, 1952, 33, 1845, 296, 335, 1675, 655, 38, - /* 890 */ 37, 1858, 613, 44, 42, 41, 40, 39, 411, 294, - /* 900 */ 70, 1815, 183, 69, 1858, 1858, 1622, 148, 147, 652, - /* 910 */ 651, 650, 145, 13, 12, 185, 189, 1814, 1099, 1100, - /* 920 */ 99, 195, 456, 454, 653, 2095, 654, 1806, 447, 1806, - /* 930 */ 96, 442, 441, 440, 439, 436, 435, 434, 433, 432, - /* 940 */ 428, 427, 426, 425, 336, 418, 417, 416, 589, 413, - /* 950 */ 412, 333, 687, 686, 685, 684, 372, 60, 683, 682, - /* 960 */ 138, 677, 676, 675, 674, 673, 672, 671, 150, 667, - /* 970 */ 666, 665, 371, 370, 662, 661, 660, 659, 658, 1613, - /* 980 */ 254, 603, 655, 1612, 290, 603, 375, 1792, 374, 578, - /* 990 */ 547, 2075, 1528, 603, 2124, 476, 102, 1441, 1717, 477, - /* 1000 */ 1967, 148, 147, 652, 651, 650, 145, 1759, 1508, 2130, - /* 1010 */ 175, 140, 1762, 128, 2125, 572, 1762, 218, 574, 71, - /* 1020 */ 216, 236, 220, 1936, 1762, 219, 52, 1936, 547, 603, - /* 1030 */ 547, 1985, 2124, 546, 2124, 424, 78, 77, 404, 586, - /* 1040 */ 583, 182, 603, 142, 1936, 1394, 619, 2130, 175, 2130, - /* 1050 */ 175, 559, 2125, 572, 2125, 572, 543, 1923, 535, 318, - /* 1060 */ 1762, 603, 392, 235, 390, 386, 382, 379, 376, 79, - /* 1070 */ 1966, 134, 1967, 1762, 2002, 587, 1393, 103, 1968, 623, - /* 1080 */ 1970, 1971, 618, 603, 613, 222, 62, 224, 221, 172, - /* 1090 */ 223, 2055, 1762, 1665, 211, 356, 2051, 271, 603, 603, - /* 1100 */ 547, 1658, 87, 1985, 2124, 384, 1656, 178, 603, 610, - /* 1110 */ 339, 620, 598, 600, 1762, 515, 1936, 2082, 619, 2130, - /* 1120 */ 175, 1957, 601, 517, 2125, 572, 1967, 603, 520, 1762, - /* 1130 */ 1762, 176, 2063, 2064, 241, 132, 2068, 248, 1559, 1762, - /* 1140 */ 47, 277, 1966, 1604, 1605, 1986, 2002, 264, 373, 103, - /* 1150 */ 1968, 623, 1970, 1971, 618, 663, 613, 1985, 1762, 68, - /* 1160 */ 144, 2144, 1867, 2055, 146, 620, 1646, 356, 2051, 1959, - /* 1170 */ 1936, 340, 619, 338, 337, 62, 501, 1161, 2089, 664, - /* 1180 */ 503, 1652, 1803, 603, 47, 47, 243, 1967, 13, 12, - /* 1190 */ 2085, 627, 1327, 584, 1397, 144, 1966, 369, 256, 269, - /* 1200 */ 2002, 1159, 502, 103, 1968, 623, 1970, 1971, 618, 253, - /* 1210 */ 613, 595, 273, 579, 1762, 2144, 1213, 2055, 1985, 1, - /* 1220 */ 146, 356, 2051, 129, 4, 1396, 620, 1502, 359, 358, - /* 1230 */ 705, 1936, 2102, 619, 378, 1349, 1451, 289, 1399, 144, - /* 1240 */ 383, 332, 284, 1241, 188, 410, 1413, 1245, 414, 1465, - /* 1250 */ 1967, 1392, 1868, 1486, 445, 419, 1408, 1966, 438, 444, - /* 1260 */ 431, 2002, 1860, 446, 103, 1968, 623, 1970, 1971, 618, - /* 1270 */ 1967, 613, 1252, 452, 1460, 1250, 2144, 453, 2055, 192, - /* 1280 */ 455, 1985, 356, 2051, 457, 1414, 1391, 458, 467, 620, - /* 1290 */ 1416, 149, 470, 554, 1936, 198, 619, 200, 1411, 471, - /* 1300 */ 1415, 1985, 472, 1417, 473, 203, 205, 475, 82, 620, - /* 1310 */ 83, 479, 1116, 209, 1936, 496, 619, 497, 500, 498, - /* 1320 */ 1966, 1752, 321, 106, 2002, 534, 215, 103, 1968, 623, - /* 1330 */ 1970, 1971, 618, 609, 613, 536, 285, 230, 1748, 2144, - /* 1340 */ 1966, 2055, 217, 151, 2002, 356, 2051, 103, 1968, 623, - /* 1350 */ 1970, 1971, 618, 152, 613, 1750, 2118, 1746, 153, 2144, - /* 1360 */ 154, 2055, 1913, 1912, 537, 356, 2051, 538, 233, 541, - /* 1370 */ 713, 544, 560, 1967, 551, 2086, 2074, 2101, 593, 557, - /* 1380 */ 2100, 346, 2096, 7, 283, 563, 569, 249, 239, 242, - /* 1390 */ 2077, 165, 247, 552, 550, 250, 549, 347, 1400, 168, - /* 1400 */ 1395, 580, 577, 1528, 1985, 703, 699, 695, 691, 281, - /* 1410 */ 251, 2147, 620, 255, 252, 2123, 133, 1936, 1412, 619, - /* 1420 */ 2071, 588, 261, 1403, 1405, 350, 591, 596, 286, 287, - /* 1430 */ 592, 1884, 1967, 1883, 1882, 352, 611, 1458, 1459, 1461, - /* 1440 */ 1462, 1463, 1464, 1966, 288, 101, 1763, 2002, 274, 597, - /* 1450 */ 103, 1968, 623, 1970, 1971, 618, 59, 613, 91, 93, - /* 1460 */ 2036, 95, 2030, 1985, 2055, 625, 280, 1807, 356, 2051, - /* 1470 */ 1733, 620, 291, 706, 707, 709, 1936, 315, 619, 51, - /* 1480 */ 295, 599, 1930, 323, 324, 293, 1967, 1929, 300, 75, - /* 1490 */ 1928, 314, 304, 1927, 76, 1924, 380, 381, 1377, 1378, - /* 1500 */ 181, 1922, 1966, 385, 387, 388, 2002, 389, 1921, 103, - /* 1510 */ 1968, 623, 1970, 1971, 618, 391, 613, 1985, 1920, 393, - /* 1520 */ 262, 2028, 1919, 2055, 395, 620, 397, 356, 2051, 1918, - /* 1530 */ 1936, 1352, 619, 1895, 1351, 1894, 402, 1356, 1893, 231, - /* 1540 */ 403, 1892, 1853, 1307, 1852, 1850, 139, 1967, 1849, 1848, - /* 1550 */ 1851, 1847, 1846, 1844, 1843, 1842, 1966, 186, 420, 1841, - /* 1560 */ 2002, 422, 1840, 103, 1968, 623, 1970, 1971, 618, 1839, - /* 1570 */ 613, 1838, 1837, 1836, 1835, 606, 1834, 2055, 1985, 1833, - /* 1580 */ 1832, 356, 2051, 1831, 1830, 1829, 620, 1828, 1827, 1826, - /* 1590 */ 1825, 1936, 1824, 619, 448, 1817, 1816, 1309, 141, 1823, - /* 1600 */ 1822, 1821, 1820, 1819, 1818, 1967, 1680, 1679, 1677, 1641, - /* 1610 */ 196, 170, 1188, 1956, 1640, 193, 194, 1966, 1908, 1102, - /* 1620 */ 73, 2002, 197, 1902, 104, 1968, 623, 1970, 1971, 618, - /* 1630 */ 1101, 613, 1891, 463, 1890, 204, 1985, 74, 2055, 1870, - /* 1640 */ 465, 1741, 2054, 2051, 620, 1676, 1674, 480, 1672, 1936, - /* 1650 */ 202, 619, 482, 1670, 484, 1135, 486, 488, 481, 1967, - /* 1660 */ 490, 1668, 492, 485, 494, 1655, 1654, 1637, 489, 1743, - /* 1670 */ 1257, 1256, 493, 1742, 61, 1966, 1171, 1179, 1178, 2002, - /* 1680 */ 678, 680, 104, 1968, 623, 1970, 1971, 618, 1967, 613, - /* 1690 */ 1985, 1177, 1176, 1173, 1172, 1170, 2055, 1666, 620, 214, - /* 1700 */ 608, 2051, 341, 1936, 1659, 619, 342, 1657, 343, 521, - /* 1710 */ 1636, 523, 518, 1635, 525, 1634, 527, 1367, 105, 1985, - /* 1720 */ 1366, 531, 1369, 26, 1907, 1358, 55, 617, 1901, 621, - /* 1730 */ 539, 155, 1936, 2002, 619, 1889, 104, 1968, 623, 1970, - /* 1740 */ 1971, 618, 1887, 613, 2129, 28, 1967, 19, 16, 553, - /* 1750 */ 2055, 1574, 246, 555, 326, 2051, 58, 63, 1966, 238, - /* 1760 */ 245, 1957, 2002, 30, 240, 312, 1968, 623, 1970, 1971, - /* 1770 */ 618, 616, 613, 604, 2020, 21, 163, 1985, 1589, 20, - /* 1780 */ 17, 1588, 348, 1558, 244, 620, 29, 1593, 1550, 1594, - /* 1790 */ 1936, 88, 619, 540, 234, 1595, 344, 1592, 349, 1888, - /* 1800 */ 1967, 57, 545, 258, 56, 1525, 166, 1524, 5, 1886, - /* 1810 */ 6, 1885, 22, 594, 263, 265, 1966, 1556, 270, 1869, - /* 1820 */ 2002, 65, 90, 162, 1968, 623, 1970, 1971, 618, 92, - /* 1830 */ 613, 1985, 96, 275, 272, 23, 12, 1401, 1432, 620, - /* 1840 */ 2005, 167, 179, 612, 1936, 1455, 619, 1453, 36, 626, - /* 1850 */ 624, 1234, 364, 15, 1967, 630, 1452, 1424, 1487, 24, - /* 1860 */ 25, 622, 1242, 628, 633, 1239, 631, 636, 639, 1236, - /* 1870 */ 1966, 634, 1477, 1230, 2002, 573, 2145, 104, 1968, 623, - /* 1880 */ 1970, 1971, 618, 637, 613, 1985, 1228, 640, 1219, 1233, - /* 1890 */ 278, 2055, 646, 620, 1232, 97, 2052, 1231, 1936, 98, - /* 1900 */ 619, 10, 1251, 1476, 1247, 72, 1967, 1133, 656, 1167, - /* 1910 */ 1166, 1165, 1164, 1163, 1162, 1160, 1158, 1186, 1157, 1156, - /* 1920 */ 668, 1154, 1967, 1153, 1966, 1152, 1151, 1183, 2002, 279, - /* 1930 */ 1150, 161, 1968, 623, 1970, 1971, 618, 1985, 613, 1139, - /* 1940 */ 1149, 1148, 1181, 1145, 1144, 620, 1141, 1140, 1138, 1673, - /* 1950 */ 1936, 688, 619, 1985, 690, 1671, 692, 694, 689, 1669, - /* 1960 */ 696, 620, 693, 698, 1667, 700, 1936, 697, 619, 1653, - /* 1970 */ 701, 702, 704, 2093, 1091, 1633, 1966, 712, 282, 708, - /* 1980 */ 2002, 1608, 1387, 306, 1968, 623, 1970, 1971, 618, 711, - /* 1990 */ 613, 1608, 1966, 292, 1608, 1608, 2002, 1608, 1608, 162, - /* 2000 */ 1968, 623, 1970, 1971, 618, 1608, 613, 1608, 1608, 1608, - /* 2010 */ 1608, 1967, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 2020 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 568, 1608, 1967, - /* 2030 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 2040 */ 1608, 1608, 1985, 1608, 1608, 1608, 1608, 351, 1608, 1608, - /* 2050 */ 620, 1608, 2146, 1608, 1608, 1936, 1608, 619, 1608, 1608, - /* 2060 */ 1985, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 617, 1608, - /* 2070 */ 1608, 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, 1608, - /* 2080 */ 1608, 1966, 1608, 1967, 1608, 2002, 1608, 1608, 313, 1968, - /* 2090 */ 623, 1970, 1971, 618, 1608, 613, 1608, 1608, 1608, 1966, - /* 2100 */ 1967, 1608, 1608, 2002, 1608, 1608, 312, 1968, 623, 1970, - /* 2110 */ 1971, 618, 1608, 613, 1985, 2021, 1608, 1608, 1608, 361, - /* 2120 */ 1608, 1608, 620, 1608, 1608, 1608, 1608, 1936, 1608, 619, - /* 2130 */ 1608, 1985, 1608, 1608, 1608, 1608, 363, 1608, 1608, 620, - /* 2140 */ 1608, 1608, 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, - /* 2150 */ 1608, 1608, 1967, 1966, 1608, 1608, 1608, 2002, 1608, 1608, - /* 2160 */ 313, 1968, 623, 1970, 1971, 618, 1608, 613, 1967, 1608, - /* 2170 */ 1966, 1608, 1608, 1608, 2002, 1608, 1608, 313, 1968, 623, - /* 2180 */ 1970, 1971, 618, 1985, 613, 1608, 1608, 1608, 1608, 1608, - /* 2190 */ 1608, 620, 1608, 1608, 1608, 1608, 1936, 1608, 619, 1985, - /* 2200 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 620, 1608, 1608, - /* 2210 */ 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, 1608, 1608, - /* 2220 */ 1967, 1608, 533, 1608, 1608, 1608, 2002, 1608, 1608, 308, - /* 2230 */ 1968, 623, 1970, 1971, 618, 1608, 613, 1608, 1966, 1608, - /* 2240 */ 1608, 1608, 2002, 1608, 1608, 297, 1968, 623, 1970, 1971, - /* 2250 */ 618, 1985, 613, 1608, 1608, 1608, 1608, 1608, 1608, 620, - /* 2260 */ 1608, 1608, 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, - /* 2270 */ 1608, 1608, 1608, 1608, 1967, 1608, 1608, 1608, 1608, 1608, - /* 2280 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 2290 */ 1966, 1608, 1608, 1608, 2002, 1967, 1608, 298, 1968, 623, - /* 2300 */ 1970, 1971, 618, 1608, 613, 1985, 1608, 1608, 1608, 1608, - /* 2310 */ 1608, 1608, 1608, 620, 1608, 1608, 1608, 1608, 1936, 1608, - /* 2320 */ 619, 1608, 1608, 1608, 1608, 1608, 1985, 1608, 1608, 1608, - /* 2330 */ 1608, 1608, 1608, 1608, 620, 1608, 1608, 1608, 1608, 1936, - /* 2340 */ 1608, 619, 1608, 1608, 1966, 1608, 1608, 1608, 2002, 1608, - /* 2350 */ 1608, 299, 1968, 623, 1970, 1971, 618, 1608, 613, 1608, - /* 2360 */ 1608, 1608, 1608, 1608, 1608, 1966, 1608, 1608, 1608, 2002, - /* 2370 */ 1608, 1967, 305, 1968, 623, 1970, 1971, 618, 1608, 613, - /* 2380 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1967, - /* 2390 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 2400 */ 1608, 1608, 1985, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 2410 */ 620, 1608, 1608, 1608, 1608, 1936, 1608, 619, 1608, 1608, - /* 2420 */ 1985, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 620, 1608, - /* 2430 */ 1608, 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, 1608, - /* 2440 */ 1608, 1966, 1608, 1967, 1608, 2002, 1608, 1608, 309, 1968, - /* 2450 */ 623, 1970, 1971, 618, 1608, 613, 1608, 1608, 1608, 1966, - /* 2460 */ 1967, 1608, 1608, 2002, 1608, 1608, 301, 1968, 623, 1970, - /* 2470 */ 1971, 618, 1608, 613, 1985, 1608, 1608, 1608, 1608, 1608, - /* 2480 */ 1608, 1608, 620, 1608, 1608, 1608, 1608, 1936, 1608, 619, - /* 2490 */ 1608, 1985, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 620, - /* 2500 */ 1608, 1608, 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, - /* 2510 */ 1608, 1608, 1967, 1966, 1608, 1608, 1608, 2002, 1608, 1608, - /* 2520 */ 310, 1968, 623, 1970, 1971, 618, 1608, 613, 1967, 1608, - /* 2530 */ 1966, 1608, 1608, 1608, 2002, 1608, 1608, 302, 1968, 623, - /* 2540 */ 1970, 1971, 618, 1985, 613, 1608, 1608, 1608, 1608, 1608, - /* 2550 */ 1608, 620, 1608, 1608, 1608, 1608, 1936, 1608, 619, 1985, - /* 2560 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 620, 1608, 1608, - /* 2570 */ 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, 1608, 1608, - /* 2580 */ 1967, 1608, 1966, 1608, 1608, 1608, 2002, 1608, 1608, 311, - /* 2590 */ 1968, 623, 1970, 1971, 618, 1608, 613, 1608, 1966, 1608, - /* 2600 */ 1608, 1608, 2002, 1967, 1608, 303, 1968, 623, 1970, 1971, - /* 2610 */ 618, 1985, 613, 1608, 1608, 1608, 1608, 1608, 1608, 620, - /* 2620 */ 1608, 1608, 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, - /* 2630 */ 1608, 1608, 1608, 1608, 1985, 1608, 1608, 1608, 1608, 1608, - /* 2640 */ 1608, 1608, 620, 1608, 1608, 1608, 1608, 1936, 1608, 619, - /* 2650 */ 1966, 1608, 1608, 1608, 2002, 1967, 1608, 316, 1968, 623, - /* 2660 */ 1970, 1971, 618, 1608, 613, 1608, 1608, 1608, 1608, 1608, - /* 2670 */ 1608, 1608, 1608, 1966, 1608, 1608, 1608, 2002, 1608, 1608, - /* 2680 */ 317, 1968, 623, 1970, 1971, 618, 1985, 613, 1608, 1608, - /* 2690 */ 1608, 1608, 1608, 1608, 620, 1608, 1608, 1608, 1608, 1936, - /* 2700 */ 1608, 619, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 2710 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1967, 1608, - /* 2720 */ 1608, 1608, 1608, 1608, 1608, 1966, 1608, 1608, 1608, 2002, - /* 2730 */ 1608, 1608, 1979, 1968, 623, 1970, 1971, 618, 1608, 613, - /* 2740 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1985, - /* 2750 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 620, 1608, 1608, - /* 2760 */ 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, 1608, 1608, - /* 2770 */ 1967, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 2780 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1966, 1967, - /* 2790 */ 1608, 1608, 2002, 1608, 1608, 1978, 1968, 623, 1970, 1971, - /* 2800 */ 618, 1985, 613, 1608, 1608, 1608, 1608, 1608, 1608, 620, - /* 2810 */ 1608, 1608, 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, - /* 2820 */ 1985, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 620, 1608, - /* 2830 */ 1608, 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, 1608, - /* 2840 */ 1966, 1967, 1608, 1608, 2002, 1608, 1608, 1977, 1968, 623, - /* 2850 */ 1970, 1971, 618, 1608, 613, 1608, 1608, 1967, 1608, 1966, - /* 2860 */ 1608, 1608, 1608, 2002, 1608, 1608, 328, 1968, 623, 1970, - /* 2870 */ 1971, 618, 1985, 613, 1608, 1608, 1608, 1608, 1608, 1608, - /* 2880 */ 620, 1608, 1608, 1608, 1608, 1936, 1608, 619, 1985, 1608, - /* 2890 */ 1608, 1608, 1608, 1608, 1608, 1608, 620, 1608, 1608, 1608, - /* 2900 */ 1608, 1936, 1608, 619, 1608, 1608, 1608, 1608, 1608, 1967, - /* 2910 */ 1608, 1966, 1608, 1608, 1608, 2002, 1608, 1608, 329, 1968, - /* 2920 */ 623, 1970, 1971, 618, 1608, 613, 1608, 1966, 1608, 1608, - /* 2930 */ 1608, 2002, 1967, 1608, 325, 1968, 623, 1970, 1971, 618, - /* 2940 */ 1985, 613, 1608, 1608, 1608, 1608, 1608, 1608, 620, 1608, - /* 2950 */ 1608, 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, 1608, - /* 2960 */ 1608, 1608, 1608, 1985, 1608, 1608, 1608, 1608, 1608, 1608, - /* 2970 */ 1608, 620, 1608, 1608, 1608, 1608, 1936, 1608, 619, 1966, - /* 2980 */ 1608, 1608, 1608, 2002, 1967, 1608, 330, 1968, 623, 1970, - /* 2990 */ 1971, 618, 1608, 613, 1608, 1608, 1608, 1608, 1608, 1608, - /* 3000 */ 1608, 1608, 621, 1608, 1608, 1608, 2002, 1608, 1608, 308, - /* 3010 */ 1968, 623, 1970, 1971, 618, 1985, 613, 1608, 1608, 1608, - /* 3020 */ 1608, 1608, 1608, 620, 1608, 1608, 1608, 1608, 1936, 1608, - /* 3030 */ 619, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 3040 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 3050 */ 1608, 1608, 1608, 1608, 1966, 1608, 1608, 1608, 2002, 1608, - /* 3060 */ 1608, 307, 1968, 623, 1970, 1971, 618, 1608, 613, + /* 0 */ 1909, 471, 376, 472, 1673, 1836, 1838, 480, 1989, 472, + /* 10 */ 1673, 1909, 45, 43, 1563, 1907, 606, 173, 1650, 1985, + /* 20 */ 371, 1989, 1412, 362, 1843, 1781, 1906, 606, 2003, 35, + /* 30 */ 283, 339, 1985, 1493, 1660, 1410, 1985, 618, 342, 1892, + /* 40 */ 1841, 470, 38, 37, 475, 1679, 44, 42, 41, 40, + /* 50 */ 39, 1981, 1987, 353, 416, 594, 604, 582, 1488, 2021, + /* 60 */ 618, 2160, 629, 18, 1981, 1987, 366, 597, 1981, 1987, + /* 70 */ 1418, 619, 1971, 1635, 635, 629, 581, 179, 1971, 629, + /* 80 */ 1843, 2161, 583, 45, 43, 130, 138, 351, 1437, 1114, + /* 90 */ 618, 371, 510, 1412, 325, 14, 1841, 335, 1437, 582, + /* 100 */ 1843, 2002, 1790, 2160, 1493, 2038, 1410, 359, 107, 2004, + /* 110 */ 639, 2006, 2007, 634, 64, 629, 1841, 726, 581, 179, + /* 120 */ 176, 265, 2091, 2161, 583, 191, 365, 2087, 1116, 1488, + /* 130 */ 1119, 1120, 1495, 1496, 18, 48, 1990, 577, 1522, 386, + /* 140 */ 181, 1418, 44, 42, 41, 40, 39, 1985, 2117, 262, + /* 150 */ 2099, 593, 479, 131, 592, 475, 1679, 2160, 60, 2021, + /* 160 */ 90, 81, 1468, 1478, 80, 48, 14, 576, 1494, 1497, + /* 170 */ 274, 275, 581, 179, 163, 273, 1649, 2161, 583, 1981, + /* 180 */ 1987, 477, 558, 1413, 8, 1411, 2160, 473, 726, 346, + /* 190 */ 629, 233, 38, 37, 1820, 1523, 44, 42, 41, 40, + /* 200 */ 39, 2166, 179, 1495, 1496, 60, 2161, 583, 1416, 1417, + /* 210 */ 575, 1467, 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477, + /* 220 */ 631, 627, 1486, 1487, 1489, 1490, 1491, 1492, 2, 572, + /* 230 */ 1338, 1339, 1439, 1468, 1478, 60, 489, 84, 120, 1494, + /* 240 */ 1497, 119, 118, 117, 116, 115, 114, 113, 112, 111, + /* 250 */ 347, 134, 345, 344, 1413, 512, 1411, 38, 37, 514, + /* 260 */ 1785, 44, 42, 41, 40, 39, 673, 34, 369, 1517, + /* 270 */ 1518, 1519, 1520, 1521, 1525, 1526, 1527, 1528, 173, 1416, + /* 280 */ 1417, 513, 1467, 1470, 1471, 1472, 1473, 1474, 1475, 1476, + /* 290 */ 1477, 631, 627, 1486, 1487, 1489, 1490, 1491, 1492, 2, + /* 300 */ 1893, 11, 45, 43, 1706, 605, 1133, 1659, 1132, 182, + /* 310 */ 371, 1658, 1412, 578, 573, 567, 182, 11, 2003, 9, + /* 320 */ 60, 216, 409, 1493, 408, 1410, 1240, 661, 660, 659, + /* 330 */ 1244, 658, 1246, 1247, 657, 1249, 654, 1134, 1255, 651, + /* 340 */ 1257, 1258, 648, 645, 1567, 1600, 405, 2164, 1488, 2021, + /* 350 */ 1437, 1971, 487, 18, 1902, 1971, 182, 636, 619, 1630, + /* 360 */ 1418, 1133, 1971, 1132, 635, 1436, 1437, 407, 403, 525, + /* 370 */ 524, 523, 130, 45, 43, 1498, 264, 135, 519, 515, + /* 380 */ 2106, 371, 518, 1412, 1992, 14, 182, 517, 522, 1790, + /* 390 */ 1556, 2002, 1134, 516, 1493, 2038, 1410, 605, 107, 2004, + /* 400 */ 639, 2006, 2007, 634, 2003, 629, 2103, 726, 141, 2165, + /* 410 */ 147, 2062, 2091, 2160, 31, 217, 365, 2087, 1524, 1488, + /* 420 */ 38, 37, 1495, 1496, 44, 42, 41, 40, 39, 2164, + /* 430 */ 168, 1418, 1994, 2161, 2163, 2021, 506, 502, 498, 494, + /* 440 */ 214, 60, 1418, 636, 603, 1503, 1902, 2165, 1971, 619, + /* 450 */ 635, 1437, 1468, 1478, 1629, 1438, 46, 539, 1494, 1497, + /* 460 */ 104, 38, 37, 54, 1439, 44, 42, 41, 40, 39, + /* 470 */ 537, 182, 535, 1413, 139, 1411, 85, 2002, 726, 212, + /* 480 */ 1790, 2038, 1782, 454, 165, 2004, 639, 2006, 2007, 634, + /* 490 */ 32, 629, 1469, 1495, 1496, 1657, 664, 27, 1416, 1417, + /* 500 */ 1529, 1467, 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477, + /* 510 */ 631, 627, 1486, 1487, 1489, 1490, 1491, 1492, 2, 525, + /* 520 */ 524, 523, 84, 1468, 1478, 559, 2128, 135, 519, 1494, + /* 530 */ 1497, 164, 518, 11, 49, 619, 1744, 517, 522, 1971, + /* 540 */ 1623, 195, 194, 516, 1413, 1786, 1411, 211, 205, 184, + /* 550 */ 1656, 685, 210, 38, 37, 485, 1779, 44, 42, 41, + /* 560 */ 40, 39, 1438, 605, 453, 1440, 1790, 1355, 1356, 1416, + /* 570 */ 1417, 203, 1467, 1470, 1471, 1472, 1473, 1474, 1475, 1476, + /* 580 */ 1477, 631, 627, 1486, 1487, 1489, 1490, 1491, 1492, 2, + /* 590 */ 45, 43, 182, 1469, 1971, 671, 1837, 1838, 371, 1638, + /* 600 */ 1412, 521, 520, 1354, 1357, 1590, 2003, 1873, 264, 585, + /* 610 */ 614, 1493, 1902, 1410, 152, 151, 668, 667, 666, 149, + /* 620 */ 120, 1775, 594, 119, 118, 117, 116, 115, 114, 113, + /* 630 */ 112, 111, 13, 12, 38, 37, 1488, 2021, 44, 42, + /* 640 */ 41, 40, 39, 2165, 1768, 597, 619, 2160, 1418, 1777, + /* 650 */ 1971, 175, 635, 138, 569, 1588, 1589, 1591, 1592, 1888, + /* 660 */ 414, 45, 43, 2164, 1830, 1843, 231, 2161, 2162, 371, + /* 670 */ 187, 1412, 364, 46, 1283, 1284, 232, 1790, 621, 2002, + /* 680 */ 2063, 1841, 1493, 2038, 1410, 182, 107, 2004, 639, 2006, + /* 690 */ 2007, 634, 2003, 629, 619, 726, 1843, 1766, 176, 623, + /* 700 */ 2091, 2063, 1200, 375, 365, 2087, 489, 1488, 415, 242, + /* 710 */ 1495, 1496, 1841, 1655, 1773, 596, 177, 2099, 2100, 1418, + /* 720 */ 136, 2104, 1579, 2021, 1637, 1790, 2118, 1536, 237, 87, + /* 730 */ 330, 636, 1412, 543, 1654, 541, 1971, 1202, 635, 1653, + /* 740 */ 1468, 1478, 1389, 1390, 14, 1410, 1494, 1497, 129, 128, + /* 750 */ 127, 126, 125, 124, 123, 122, 121, 1971, 434, 673, + /* 760 */ 630, 1413, 686, 1411, 1760, 2002, 726, 433, 663, 2038, + /* 770 */ 697, 695, 165, 2004, 639, 2006, 2007, 634, 1971, 629, + /* 780 */ 1418, 1495, 1496, 1971, 2106, 1652, 1416, 1417, 410, 1467, + /* 790 */ 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477, 631, 627, + /* 800 */ 1486, 1487, 1489, 1490, 1491, 1492, 2, 619, 1888, 1888, + /* 810 */ 2102, 1468, 1478, 328, 2129, 1435, 619, 1494, 1497, 189, + /* 820 */ 193, 424, 447, 619, 98, 461, 2106, 726, 460, 1971, + /* 830 */ 439, 558, 1413, 1843, 1411, 2160, 363, 440, 1790, 553, + /* 840 */ 41, 40, 39, 430, 161, 462, 1783, 1790, 432, 1842, + /* 850 */ 2166, 179, 2101, 1792, 1790, 2161, 583, 1416, 1417, 1958, + /* 860 */ 1467, 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477, 631, + /* 870 */ 627, 1486, 1487, 1489, 1490, 1491, 1492, 2, 1767, 374, + /* 880 */ 1119, 1120, 161, 162, 1651, 619, 619, 161, 303, 343, + /* 890 */ 1703, 1793, 514, 1413, 1765, 1411, 1792, 1437, 33, 488, + /* 900 */ 1787, 420, 301, 70, 38, 37, 69, 393, 44, 42, + /* 910 */ 41, 40, 39, 377, 513, 1745, 1790, 1790, 1416, 1417, + /* 920 */ 140, 161, 259, 2062, 199, 467, 465, 2131, 1971, 665, + /* 930 */ 1792, 458, 1834, 1560, 452, 451, 450, 449, 446, 445, + /* 940 */ 444, 443, 442, 438, 437, 436, 435, 327, 427, 426, + /* 950 */ 425, 570, 422, 421, 341, 703, 702, 701, 700, 381, + /* 960 */ 60, 699, 698, 142, 693, 692, 691, 690, 689, 688, + /* 970 */ 687, 154, 683, 682, 681, 380, 379, 678, 677, 676, + /* 980 */ 675, 674, 1648, 530, 1440, 671, 38, 37, 594, 417, + /* 990 */ 44, 42, 41, 40, 39, 1632, 1633, 1440, 540, 106, + /* 1000 */ 619, 671, 418, 619, 152, 151, 668, 667, 666, 149, + /* 1010 */ 2111, 1556, 230, 1647, 234, 619, 2003, 284, 586, 138, + /* 1020 */ 152, 151, 668, 667, 666, 149, 1971, 533, 594, 554, + /* 1030 */ 619, 1790, 527, 1646, 1790, 1645, 619, 229, 150, 1469, + /* 1040 */ 78, 77, 413, 253, 598, 186, 1790, 2021, 1644, 669, + /* 1050 */ 602, 670, 1834, 297, 1834, 636, 1820, 1971, 1693, 138, + /* 1060 */ 1971, 1790, 635, 326, 619, 1643, 401, 1790, 399, 395, + /* 1070 */ 391, 388, 385, 67, 2003, 589, 66, 1971, 278, 1971, + /* 1080 */ 526, 1421, 178, 2099, 2100, 1642, 136, 2104, 619, 2002, + /* 1090 */ 53, 215, 1971, 2038, 71, 1790, 107, 2004, 639, 2006, + /* 1100 */ 2007, 634, 616, 629, 2003, 2021, 13, 12, 2180, 1971, + /* 1110 */ 2091, 182, 150, 636, 365, 2087, 382, 2022, 1971, 1790, + /* 1120 */ 635, 2003, 180, 2099, 2100, 2125, 136, 2104, 188, 1971, + /* 1130 */ 1641, 619, 38, 37, 1674, 2021, 44, 42, 41, 40, + /* 1140 */ 39, 241, 160, 636, 79, 617, 1640, 2002, 1971, 619, + /* 1150 */ 635, 2038, 2021, 1686, 107, 2004, 639, 2006, 2007, 634, + /* 1160 */ 636, 629, 1790, 378, 1387, 1971, 2180, 635, 2091, 50, + /* 1170 */ 1684, 3, 365, 2087, 1971, 528, 1559, 2002, 2003, 240, + /* 1180 */ 1790, 2038, 600, 2138, 313, 2004, 639, 2006, 2007, 634, + /* 1190 */ 1971, 629, 531, 144, 2002, 132, 222, 224, 2038, 220, + /* 1200 */ 223, 107, 2004, 639, 2006, 2007, 634, 1420, 629, 2021, + /* 1210 */ 1897, 226, 1831, 2180, 225, 2091, 52, 636, 88, 365, + /* 1220 */ 2087, 384, 1971, 557, 635, 558, 62, 626, 579, 2160, + /* 1230 */ 565, 1424, 228, 368, 367, 227, 2003, 2121, 246, 595, + /* 1240 */ 150, 587, 47, 1426, 2166, 179, 1, 271, 258, 2161, + /* 1250 */ 583, 2002, 47, 1680, 1493, 2038, 1419, 4, 107, 2004, + /* 1260 */ 639, 2006, 2007, 634, 558, 629, 2003, 2021, 2160, 261, + /* 1270 */ 2180, 1162, 2091, 68, 148, 636, 365, 2087, 1587, 1488, + /* 1280 */ 1971, 150, 635, 2166, 179, 103, 679, 2154, 2161, 583, + /* 1290 */ 248, 1418, 601, 387, 1352, 100, 392, 2021, 2003, 276, + /* 1300 */ 590, 340, 721, 62, 1479, 636, 1163, 47, 1181, 2002, + /* 1310 */ 1971, 643, 635, 2038, 291, 148, 107, 2004, 639, 2006, + /* 1320 */ 2007, 634, 150, 629, 133, 611, 280, 1374, 2180, 2021, + /* 1330 */ 2091, 192, 680, 1233, 365, 2087, 148, 636, 625, 2002, + /* 1340 */ 1440, 419, 1971, 2038, 635, 2110, 107, 2004, 639, 2006, + /* 1350 */ 2007, 634, 383, 629, 1179, 1530, 1898, 1423, 2066, 296, + /* 1360 */ 2091, 423, 456, 1261, 365, 2087, 1435, 1265, 428, 441, + /* 1370 */ 455, 2002, 1514, 1890, 1272, 2038, 1270, 2003, 107, 2004, + /* 1380 */ 639, 2006, 2007, 634, 448, 629, 457, 463, 153, 464, + /* 1390 */ 2064, 196, 2091, 466, 468, 558, 365, 2087, 1441, 2160, + /* 1400 */ 469, 478, 1443, 481, 1427, 202, 1422, 1438, 2021, 482, + /* 1410 */ 204, 1442, 483, 1444, 2166, 179, 636, 484, 207, 2161, + /* 1420 */ 583, 1971, 486, 635, 209, 546, 82, 490, 83, 1430, + /* 1430 */ 1432, 213, 1136, 507, 508, 2003, 511, 1780, 509, 219, + /* 1440 */ 1776, 110, 627, 1486, 1487, 1489, 1490, 1491, 1492, 221, + /* 1450 */ 2002, 155, 156, 329, 2038, 2003, 1778, 107, 2004, 639, + /* 1460 */ 2006, 2007, 634, 1774, 629, 545, 2021, 547, 558, 622, + /* 1470 */ 157, 2091, 2160, 158, 636, 365, 2087, 235, 86, 1971, + /* 1480 */ 292, 635, 1948, 548, 1947, 238, 2021, 2166, 179, 552, + /* 1490 */ 571, 555, 2161, 583, 636, 146, 549, 609, 2122, 1971, + /* 1500 */ 2132, 635, 562, 568, 2137, 2136, 244, 247, 2002, 7, + /* 1510 */ 354, 574, 2038, 2003, 580, 108, 2004, 639, 2006, 2007, + /* 1520 */ 634, 2113, 629, 560, 563, 252, 355, 561, 2002, 2091, + /* 1530 */ 255, 169, 2038, 2090, 2087, 108, 2004, 639, 2006, 2007, + /* 1540 */ 634, 588, 629, 2003, 2021, 254, 257, 591, 1556, 2091, + /* 1550 */ 256, 137, 636, 624, 2087, 1439, 2107, 1971, 358, 635, + /* 1560 */ 2003, 599, 1445, 266, 2183, 1903, 93, 612, 293, 294, + /* 1570 */ 607, 613, 608, 260, 2021, 1917, 95, 1916, 2159, 1915, + /* 1580 */ 361, 97, 633, 295, 1791, 59, 637, 1971, 2072, 635, + /* 1590 */ 2038, 2021, 99, 108, 2004, 639, 2006, 2007, 634, 636, + /* 1600 */ 629, 1835, 641, 298, 1971, 722, 635, 2091, 1761, 723, + /* 1610 */ 287, 334, 2087, 725, 51, 331, 2002, 2003, 300, 307, + /* 1620 */ 2038, 332, 322, 319, 2004, 639, 2006, 2007, 634, 632, + /* 1630 */ 629, 620, 2056, 2002, 302, 321, 1965, 2038, 311, 1964, + /* 1640 */ 166, 2004, 639, 2006, 2007, 634, 75, 629, 2021, 1963, + /* 1650 */ 1962, 76, 1959, 389, 390, 1404, 636, 1405, 185, 394, + /* 1660 */ 1957, 1971, 396, 635, 2003, 397, 398, 1956, 400, 1955, + /* 1670 */ 402, 1954, 1953, 404, 1377, 406, 1376, 1928, 1927, 411, + /* 1680 */ 412, 2003, 1926, 1925, 1329, 1881, 1880, 1878, 1877, 143, + /* 1690 */ 2002, 1876, 584, 2181, 2038, 2021, 1879, 108, 2004, 639, + /* 1700 */ 2006, 2007, 634, 636, 629, 1875, 1874, 1872, 1971, 190, + /* 1710 */ 635, 2091, 2021, 1871, 1870, 429, 2088, 360, 1869, 431, + /* 1720 */ 636, 1883, 1868, 1867, 1866, 1971, 1865, 635, 1864, 1863, + /* 1730 */ 1862, 1861, 1860, 1859, 1858, 1857, 2003, 2002, 1856, 1855, + /* 1740 */ 1854, 2038, 145, 1853, 166, 2004, 639, 2006, 2007, 634, + /* 1750 */ 1852, 629, 1851, 1882, 2002, 1850, 1849, 1848, 2038, 2003, + /* 1760 */ 1847, 320, 2004, 639, 2006, 2007, 634, 2021, 629, 1331, + /* 1770 */ 1846, 1845, 1844, 459, 1708, 633, 1208, 197, 1707, 1705, + /* 1780 */ 1971, 1669, 635, 198, 200, 73, 1668, 1122, 174, 1121, + /* 1790 */ 2021, 2003, 1991, 1941, 1935, 370, 474, 2182, 636, 201, + /* 1800 */ 74, 476, 1924, 1971, 206, 635, 208, 1923, 1901, 2002, + /* 1810 */ 1769, 1704, 1155, 2038, 1702, 491, 319, 2004, 639, 2006, + /* 1820 */ 2007, 634, 2021, 629, 492, 2057, 493, 372, 1700, 497, + /* 1830 */ 636, 495, 2002, 496, 1698, 1971, 2038, 635, 2003, 320, + /* 1840 */ 2004, 639, 2006, 2007, 634, 500, 629, 499, 501, 1696, + /* 1850 */ 503, 504, 729, 1683, 1682, 505, 1665, 1771, 1277, 1276, + /* 1860 */ 218, 1770, 2003, 694, 2002, 1199, 290, 61, 2038, 2021, + /* 1870 */ 1198, 320, 2004, 639, 2006, 2007, 634, 636, 629, 1197, + /* 1880 */ 1196, 172, 1971, 696, 635, 1193, 1192, 719, 715, 711, + /* 1890 */ 707, 288, 1191, 2021, 1190, 1694, 348, 1687, 349, 1685, + /* 1900 */ 350, 636, 529, 532, 1664, 1663, 1971, 534, 635, 1662, + /* 1910 */ 536, 544, 538, 109, 1394, 2038, 1393, 1396, 315, 2004, + /* 1920 */ 639, 2006, 2007, 634, 26, 629, 1940, 105, 2003, 1383, + /* 1930 */ 281, 542, 55, 1934, 550, 2002, 1922, 159, 1920, 2038, + /* 1940 */ 2165, 551, 304, 2004, 639, 2006, 2007, 634, 239, 629, + /* 1950 */ 2003, 19, 352, 16, 1602, 556, 564, 28, 566, 2021, + /* 1960 */ 243, 58, 5, 615, 63, 245, 6, 636, 250, 1586, + /* 1970 */ 251, 30, 1971, 1578, 635, 2003, 1992, 167, 249, 29, + /* 1980 */ 89, 2021, 1622, 1623, 21, 1617, 1616, 356, 1621, 636, + /* 1990 */ 1620, 357, 1553, 263, 1971, 170, 635, 1552, 268, 92, + /* 2000 */ 1899, 2002, 57, 267, 1921, 2038, 2021, 1919, 305, 2004, + /* 2010 */ 639, 2006, 2007, 634, 636, 629, 20, 1918, 1900, 1971, + /* 2020 */ 1381, 635, 236, 2002, 269, 91, 22, 2038, 56, 272, + /* 2030 */ 306, 2004, 639, 2006, 2007, 634, 277, 629, 2003, 17, + /* 2040 */ 270, 1584, 65, 282, 94, 23, 1505, 610, 2002, 96, + /* 2050 */ 12, 10, 2038, 1504, 279, 312, 2004, 639, 2006, 2007, + /* 2060 */ 634, 1428, 629, 1515, 100, 2041, 2003, 1483, 628, 2021, + /* 2070 */ 1481, 36, 171, 183, 1480, 15, 24, 636, 640, 1452, + /* 2080 */ 1460, 638, 1971, 25, 635, 1262, 642, 373, 644, 646, + /* 2090 */ 1259, 1256, 647, 650, 649, 652, 2003, 2021, 1250, 1248, + /* 2100 */ 655, 653, 1239, 656, 1254, 636, 1253, 101, 285, 1252, + /* 2110 */ 1971, 2002, 635, 1267, 102, 2038, 662, 1271, 316, 2004, + /* 2120 */ 639, 2006, 2007, 634, 72, 629, 1153, 2021, 2003, 1251, + /* 2130 */ 672, 1187, 1186, 1185, 1184, 636, 1183, 1182, 1180, 2002, + /* 2140 */ 1971, 1178, 635, 2038, 1177, 1176, 308, 2004, 639, 2006, + /* 2150 */ 2007, 634, 286, 629, 1206, 684, 1174, 1173, 1172, 2021, + /* 2160 */ 2003, 1171, 1170, 1169, 1168, 1203, 1201, 636, 1165, 2002, + /* 2170 */ 1164, 1161, 1971, 2038, 635, 1160, 317, 2004, 639, 2006, + /* 2180 */ 2007, 634, 1159, 629, 1158, 1701, 704, 706, 1699, 705, + /* 2190 */ 708, 2021, 2003, 709, 1697, 710, 712, 714, 713, 636, + /* 2200 */ 1695, 2002, 716, 718, 1971, 2038, 635, 717, 309, 2004, + /* 2210 */ 639, 2006, 2007, 634, 1681, 629, 720, 2003, 1111, 1661, + /* 2220 */ 289, 1414, 724, 2021, 299, 727, 728, 1636, 1636, 1636, + /* 2230 */ 1636, 636, 1636, 2002, 1636, 1636, 1971, 2038, 635, 1636, + /* 2240 */ 318, 2004, 639, 2006, 2007, 634, 1636, 629, 2021, 2003, + /* 2250 */ 1636, 1636, 1636, 1636, 1636, 1636, 636, 1636, 1636, 1636, + /* 2260 */ 1636, 1971, 1636, 635, 1636, 2002, 1636, 1636, 1636, 2038, + /* 2270 */ 1636, 1636, 310, 2004, 639, 2006, 2007, 634, 1636, 629, + /* 2280 */ 2021, 1636, 1636, 1636, 1636, 1636, 1636, 1636, 636, 1636, + /* 2290 */ 2002, 1636, 1636, 1971, 2038, 635, 1636, 323, 2004, 639, + /* 2300 */ 2006, 2007, 634, 1636, 629, 1636, 1636, 1636, 1636, 1636, + /* 2310 */ 1636, 2003, 1636, 1636, 1636, 1636, 1636, 1636, 1636, 1636, + /* 2320 */ 1636, 1636, 2002, 1636, 1636, 1636, 2038, 1636, 2003, 324, + /* 2330 */ 2004, 639, 2006, 2007, 634, 1636, 629, 1636, 1636, 1636, + /* 2340 */ 1636, 1636, 2021, 1636, 1636, 1636, 1636, 1636, 1636, 1636, + /* 2350 */ 636, 1636, 1636, 1636, 1636, 1971, 1636, 635, 1636, 2021, + /* 2360 */ 1636, 1636, 1636, 1636, 1636, 1636, 1636, 636, 1636, 1636, + /* 2370 */ 1636, 1636, 1971, 1636, 635, 1636, 1636, 1636, 1636, 1636, + /* 2380 */ 1636, 1636, 1636, 1636, 2002, 2003, 1636, 1636, 2038, 1636, + /* 2390 */ 1636, 2015, 2004, 639, 2006, 2007, 634, 1636, 629, 1636, + /* 2400 */ 1636, 2002, 1636, 1636, 1636, 2038, 1636, 2003, 2014, 2004, + /* 2410 */ 639, 2006, 2007, 634, 1636, 629, 2021, 1636, 1636, 1636, + /* 2420 */ 1636, 1636, 1636, 1636, 636, 1636, 1636, 1636, 1636, 1971, + /* 2430 */ 1636, 635, 1636, 1636, 1636, 1636, 1636, 1636, 2021, 1636, + /* 2440 */ 1636, 1636, 1636, 1636, 1636, 1636, 636, 1636, 1636, 1636, + /* 2450 */ 1636, 1971, 1636, 635, 1636, 1636, 1636, 1636, 2002, 1636, + /* 2460 */ 1636, 1636, 2038, 1636, 1636, 2013, 2004, 639, 2006, 2007, + /* 2470 */ 634, 1636, 629, 2003, 1636, 1636, 1636, 1636, 1636, 1636, + /* 2480 */ 2002, 1636, 1636, 1636, 2038, 1636, 1636, 336, 2004, 639, + /* 2490 */ 2006, 2007, 634, 1636, 629, 1636, 1636, 2003, 1636, 1636, + /* 2500 */ 1636, 1636, 1636, 1636, 2021, 1636, 1636, 1636, 1636, 1636, + /* 2510 */ 1636, 1636, 636, 1636, 1636, 1636, 1636, 1971, 1636, 635, + /* 2520 */ 1636, 1636, 1636, 1636, 1636, 1636, 1636, 1636, 2021, 1636, + /* 2530 */ 1636, 1636, 1636, 1636, 1636, 1636, 636, 1636, 1636, 1636, + /* 2540 */ 1636, 1971, 1636, 635, 1636, 1636, 2002, 1636, 1636, 1636, + /* 2550 */ 2038, 1636, 1636, 337, 2004, 639, 2006, 2007, 634, 1636, + /* 2560 */ 629, 2003, 1636, 1636, 1636, 1636, 1636, 1636, 1636, 1636, + /* 2570 */ 2002, 1636, 1636, 1636, 2038, 1636, 1636, 333, 2004, 639, + /* 2580 */ 2006, 2007, 634, 1636, 629, 1636, 2003, 1636, 1636, 1636, + /* 2590 */ 1636, 1636, 2021, 1636, 1636, 1636, 1636, 1636, 1636, 1636, + /* 2600 */ 636, 1636, 1636, 1636, 1636, 1971, 1636, 635, 1636, 1636, + /* 2610 */ 1636, 1636, 1636, 1636, 1636, 1636, 1636, 2021, 2003, 1636, + /* 2620 */ 1636, 1636, 1636, 1636, 1636, 636, 1636, 1636, 1636, 1636, + /* 2630 */ 1971, 1636, 635, 1636, 2002, 1636, 1636, 1636, 2038, 1636, + /* 2640 */ 1636, 338, 2004, 639, 2006, 2007, 634, 1636, 629, 2021, + /* 2650 */ 1636, 1636, 1636, 1636, 1636, 1636, 1636, 636, 1636, 637, + /* 2660 */ 1636, 1636, 1971, 2038, 635, 1636, 315, 2004, 639, 2006, + /* 2670 */ 2007, 634, 1636, 629, 1636, 1636, 1636, 1636, 1636, 1636, + /* 2680 */ 1636, 1636, 1636, 1636, 1636, 1636, 1636, 1636, 1636, 1636, + /* 2690 */ 1636, 2002, 1636, 1636, 1636, 2038, 1636, 1636, 314, 2004, + /* 2700 */ 639, 2006, 2007, 634, 1636, 629, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 420, 421, 332, 374, 334, 335, 332, 370, 334, 335, - /* 10 */ 373, 374, 12, 13, 14, 361, 387, 388, 360, 14, - /* 20 */ 20, 336, 22, 8, 9, 20, 372, 12, 13, 14, - /* 30 */ 15, 16, 359, 33, 333, 35, 352, 336, 337, 355, - /* 40 */ 431, 20, 8, 9, 435, 20, 12, 13, 14, 15, - /* 50 */ 16, 39, 20, 380, 381, 401, 402, 403, 58, 450, - /* 60 */ 451, 376, 58, 63, 455, 456, 412, 8, 9, 328, - /* 70 */ 70, 12, 13, 14, 15, 16, 333, 20, 336, 336, - /* 80 */ 337, 8, 9, 12, 13, 12, 13, 14, 15, 16, - /* 90 */ 3, 20, 358, 22, 0, 95, 415, 63, 94, 418, - /* 100 */ 359, 97, 95, 44, 33, 371, 35, 20, 367, 367, - /* 110 */ 327, 96, 329, 372, 336, 374, 95, 117, 24, 25, - /* 120 */ 26, 27, 28, 29, 30, 31, 32, 95, 350, 58, - /* 130 */ 340, 343, 132, 133, 63, 357, 348, 79, 104, 398, - /* 140 */ 4, 70, 62, 402, 354, 367, 405, 406, 407, 408, - /* 150 */ 409, 410, 362, 412, 20, 431, 359, 44, 417, 435, - /* 160 */ 419, 161, 162, 366, 423, 424, 95, 167, 168, 427, - /* 170 */ 428, 429, 375, 431, 432, 451, 20, 435, 437, 455, - /* 180 */ 456, 431, 182, 96, 184, 435, 445, 21, 117, 132, - /* 190 */ 133, 58, 450, 451, 160, 137, 138, 455, 456, 95, - /* 200 */ 34, 451, 36, 132, 133, 455, 456, 207, 208, 96, - /* 210 */ 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, - /* 220 */ 220, 221, 222, 223, 224, 225, 226, 227, 95, 95, - /* 230 */ 97, 95, 161, 162, 404, 20, 163, 21, 167, 168, - /* 240 */ 24, 25, 26, 27, 28, 29, 30, 31, 32, 106, - /* 250 */ 163, 127, 245, 182, 404, 184, 244, 8, 9, 328, - /* 260 */ 430, 12, 13, 14, 15, 16, 232, 233, 234, 235, - /* 270 */ 236, 237, 238, 239, 240, 241, 242, 245, 207, 208, - /* 280 */ 430, 210, 211, 212, 213, 214, 215, 216, 217, 218, - /* 290 */ 219, 220, 221, 222, 223, 224, 225, 226, 227, 177, - /* 300 */ 229, 12, 13, 372, 361, 22, 14, 328, 359, 20, - /* 310 */ 22, 22, 20, 189, 190, 372, 367, 193, 35, 195, - /* 320 */ 198, 199, 33, 35, 35, 108, 109, 110, 111, 112, - /* 330 */ 113, 114, 115, 116, 117, 118, 0, 120, 121, 122, - /* 340 */ 123, 124, 125, 33, 401, 402, 403, 58, 35, 245, - /* 350 */ 389, 372, 63, 70, 359, 412, 407, 4, 48, 70, - /* 360 */ 20, 366, 22, 20, 54, 55, 56, 57, 58, 328, - /* 370 */ 375, 328, 12, 13, 14, 35, 161, 162, 245, 245, - /* 380 */ 20, 245, 22, 70, 95, 12, 13, 14, 15, 16, - /* 390 */ 62, 51, 431, 33, 20, 35, 435, 20, 328, 22, - /* 400 */ 117, 65, 66, 67, 94, 4, 117, 97, 0, 73, - /* 410 */ 74, 450, 451, 372, 78, 372, 455, 456, 58, 83, - /* 420 */ 84, 132, 133, 174, 328, 89, 126, 127, 51, 21, - /* 430 */ 70, 131, 24, 25, 26, 27, 28, 29, 30, 31, - /* 440 */ 32, 336, 372, 20, 43, 328, 45, 46, 70, 374, - /* 450 */ 161, 162, 336, 161, 359, 95, 167, 168, 229, 384, - /* 460 */ 231, 366, 387, 388, 336, 182, 350, 184, 372, 44, - /* 470 */ 375, 182, 367, 184, 164, 165, 328, 117, 350, 169, - /* 480 */ 8, 9, 172, 367, 12, 13, 14, 15, 16, 372, - /* 490 */ 207, 208, 132, 133, 342, 367, 207, 208, 188, 210, - /* 500 */ 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, - /* 510 */ 221, 222, 223, 224, 225, 226, 227, 365, 269, 20, - /* 520 */ 372, 161, 162, 336, 35, 328, 431, 167, 168, 336, - /* 530 */ 435, 426, 427, 428, 429, 336, 431, 432, 20, 65, - /* 540 */ 66, 67, 182, 350, 184, 450, 451, 73, 74, 350, - /* 550 */ 455, 456, 78, 328, 367, 132, 133, 83, 84, 70, - /* 560 */ 367, 328, 328, 89, 373, 374, 367, 207, 208, 372, - /* 570 */ 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, - /* 580 */ 220, 221, 222, 223, 224, 225, 226, 227, 12, 13, - /* 590 */ 167, 168, 359, 0, 342, 351, 20, 372, 22, 246, - /* 600 */ 367, 325, 416, 359, 418, 372, 372, 374, 356, 33, - /* 610 */ 126, 35, 368, 229, 427, 428, 429, 365, 431, 432, - /* 620 */ 8, 9, 328, 404, 12, 13, 14, 15, 16, 4, - /* 630 */ 166, 398, 20, 336, 58, 402, 70, 207, 405, 406, - /* 640 */ 407, 408, 409, 410, 19, 412, 70, 350, 415, 430, - /* 650 */ 417, 418, 419, 359, 357, 62, 423, 424, 33, 12, - /* 660 */ 13, 367, 163, 336, 367, 389, 372, 20, 374, 22, - /* 670 */ 416, 95, 418, 48, 328, 191, 192, 350, 53, 328, - /* 680 */ 33, 163, 35, 58, 361, 95, 256, 257, 258, 259, - /* 690 */ 260, 266, 398, 117, 367, 372, 402, 345, 346, 405, - /* 700 */ 406, 407, 408, 409, 410, 58, 412, 431, 132, 133, - /* 710 */ 0, 435, 393, 351, 250, 251, 252, 70, 372, 94, - /* 720 */ 0, 359, 97, 372, 401, 402, 450, 451, 14, 360, - /* 730 */ 368, 455, 456, 328, 20, 412, 369, 161, 162, 372, - /* 740 */ 446, 447, 95, 167, 168, 336, 8, 9, 160, 360, - /* 750 */ 12, 13, 14, 15, 16, 2, 359, 3, 182, 350, - /* 760 */ 184, 8, 9, 107, 117, 12, 13, 14, 15, 16, - /* 770 */ 360, 181, 62, 183, 328, 163, 367, 372, 381, 132, - /* 780 */ 133, 0, 328, 207, 208, 129, 210, 211, 212, 213, - /* 790 */ 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, - /* 800 */ 224, 225, 226, 227, 14, 15, 16, 336, 161, 162, - /* 810 */ 18, 42, 20, 44, 167, 168, 351, 340, 372, 27, - /* 820 */ 232, 350, 30, 360, 359, 33, 372, 107, 360, 182, - /* 830 */ 242, 184, 328, 368, 96, 245, 345, 346, 367, 362, - /* 840 */ 48, 361, 50, 360, 359, 53, 126, 127, 128, 129, - /* 850 */ 130, 131, 372, 368, 207, 208, 20, 210, 211, 212, - /* 860 */ 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, - /* 870 */ 223, 224, 225, 226, 227, 161, 372, 347, 20, 349, - /* 880 */ 18, 401, 402, 2, 0, 23, 94, 0, 107, 8, - /* 890 */ 9, 367, 412, 12, 13, 14, 15, 16, 106, 37, - /* 900 */ 38, 359, 378, 41, 367, 367, 329, 126, 127, 128, - /* 910 */ 129, 130, 131, 1, 2, 378, 378, 375, 45, 46, - /* 920 */ 95, 59, 60, 61, 369, 382, 369, 372, 136, 372, - /* 930 */ 105, 139, 140, 141, 142, 143, 144, 145, 146, 147, - /* 940 */ 148, 149, 150, 151, 152, 153, 154, 155, 389, 157, - /* 950 */ 158, 159, 65, 66, 67, 68, 69, 95, 71, 72, - /* 960 */ 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, - /* 970 */ 83, 84, 85, 86, 87, 88, 89, 90, 91, 328, - /* 980 */ 459, 336, 107, 328, 352, 336, 389, 355, 389, 44, - /* 990 */ 431, 243, 244, 336, 435, 350, 134, 161, 348, 350, - /* 1000 */ 328, 126, 127, 128, 129, 130, 131, 350, 96, 450, - /* 1010 */ 451, 42, 367, 44, 455, 456, 367, 99, 264, 106, - /* 1020 */ 102, 163, 99, 372, 367, 102, 163, 372, 431, 336, - /* 1030 */ 431, 359, 435, 170, 435, 151, 174, 175, 176, 367, - /* 1040 */ 336, 179, 336, 350, 372, 35, 374, 450, 451, 450, - /* 1050 */ 451, 448, 455, 456, 455, 456, 350, 0, 389, 197, - /* 1060 */ 367, 336, 200, 58, 202, 203, 204, 205, 206, 156, - /* 1070 */ 398, 367, 328, 367, 402, 350, 35, 405, 406, 407, - /* 1080 */ 408, 409, 410, 336, 412, 99, 44, 99, 102, 417, - /* 1090 */ 102, 419, 367, 0, 338, 423, 424, 350, 336, 336, - /* 1100 */ 431, 0, 97, 359, 435, 48, 0, 245, 336, 63, - /* 1110 */ 37, 367, 350, 350, 367, 22, 372, 445, 374, 450, - /* 1120 */ 451, 47, 350, 22, 455, 456, 328, 336, 22, 367, - /* 1130 */ 367, 427, 428, 429, 44, 431, 432, 442, 96, 367, - /* 1140 */ 44, 350, 398, 132, 133, 359, 402, 44, 338, 405, - /* 1150 */ 406, 407, 408, 409, 410, 13, 412, 359, 367, 44, - /* 1160 */ 44, 417, 382, 419, 44, 367, 335, 423, 424, 95, - /* 1170 */ 372, 98, 374, 100, 101, 44, 103, 35, 434, 13, - /* 1180 */ 107, 0, 371, 336, 44, 44, 96, 328, 1, 2, - /* 1190 */ 382, 44, 96, 433, 184, 44, 398, 350, 452, 96, - /* 1200 */ 402, 35, 129, 405, 406, 407, 408, 409, 410, 425, - /* 1210 */ 412, 96, 96, 268, 367, 417, 96, 419, 359, 436, - /* 1220 */ 44, 423, 424, 44, 247, 184, 367, 96, 12, 13, - /* 1230 */ 49, 372, 434, 374, 400, 180, 96, 96, 22, 44, - /* 1240 */ 48, 399, 391, 96, 42, 379, 20, 96, 379, 33, - /* 1250 */ 328, 35, 382, 207, 160, 377, 20, 398, 379, 377, - /* 1260 */ 336, 402, 336, 377, 405, 406, 407, 408, 409, 410, - /* 1270 */ 328, 412, 96, 93, 58, 96, 417, 344, 419, 336, - /* 1280 */ 336, 359, 423, 424, 336, 20, 70, 330, 330, 367, - /* 1290 */ 20, 96, 395, 434, 372, 342, 374, 342, 20, 374, - /* 1300 */ 20, 359, 337, 20, 390, 342, 342, 337, 342, 367, - /* 1310 */ 342, 336, 52, 342, 372, 339, 374, 339, 359, 330, - /* 1320 */ 398, 359, 330, 336, 402, 196, 359, 405, 406, 407, - /* 1330 */ 408, 409, 410, 117, 412, 397, 395, 340, 359, 417, - /* 1340 */ 398, 419, 359, 359, 402, 423, 424, 405, 406, 407, - /* 1350 */ 408, 409, 410, 359, 412, 359, 434, 359, 359, 417, - /* 1360 */ 359, 419, 372, 372, 187, 423, 424, 394, 340, 374, - /* 1370 */ 19, 336, 255, 328, 372, 382, 434, 441, 254, 372, - /* 1380 */ 441, 372, 382, 261, 33, 372, 173, 440, 385, 385, - /* 1390 */ 444, 441, 443, 263, 262, 439, 248, 270, 182, 48, - /* 1400 */ 184, 267, 265, 244, 359, 54, 55, 56, 57, 58, - /* 1410 */ 438, 460, 367, 453, 400, 454, 367, 372, 20, 374, - /* 1420 */ 404, 336, 340, 207, 208, 337, 372, 165, 385, 385, - /* 1430 */ 372, 372, 328, 372, 372, 372, 220, 221, 222, 223, - /* 1440 */ 224, 225, 226, 398, 355, 94, 367, 402, 97, 383, - /* 1450 */ 405, 406, 407, 408, 409, 410, 95, 412, 340, 340, - /* 1460 */ 422, 95, 417, 359, 419, 363, 340, 372, 423, 424, - /* 1470 */ 349, 367, 336, 36, 331, 330, 372, 396, 374, 392, - /* 1480 */ 326, 130, 0, 386, 386, 341, 328, 0, 353, 189, - /* 1490 */ 0, 353, 353, 0, 42, 0, 35, 201, 35, 35, - /* 1500 */ 35, 0, 398, 201, 35, 35, 402, 201, 0, 405, - /* 1510 */ 406, 407, 408, 409, 410, 201, 412, 359, 0, 35, - /* 1520 */ 169, 417, 0, 419, 22, 367, 35, 423, 424, 0, - /* 1530 */ 372, 184, 374, 0, 182, 0, 178, 186, 0, 188, - /* 1540 */ 177, 0, 0, 47, 0, 0, 42, 328, 0, 0, - /* 1550 */ 0, 0, 0, 0, 0, 0, 398, 151, 35, 0, - /* 1560 */ 402, 151, 0, 405, 406, 407, 408, 409, 410, 0, - /* 1570 */ 412, 0, 0, 0, 0, 417, 0, 419, 359, 0, - /* 1580 */ 0, 423, 424, 0, 0, 0, 367, 0, 0, 0, - /* 1590 */ 0, 372, 0, 374, 135, 0, 0, 22, 42, 0, - /* 1600 */ 0, 0, 0, 0, 0, 328, 0, 0, 0, 0, - /* 1610 */ 42, 44, 35, 47, 0, 58, 58, 398, 0, 14, - /* 1620 */ 39, 402, 40, 0, 405, 406, 407, 408, 409, 410, - /* 1630 */ 14, 412, 0, 47, 0, 173, 359, 39, 419, 0, - /* 1640 */ 47, 0, 423, 424, 367, 0, 0, 35, 0, 372, - /* 1650 */ 39, 374, 39, 0, 35, 64, 39, 35, 48, 328, - /* 1660 */ 39, 0, 35, 48, 39, 0, 0, 0, 48, 0, - /* 1670 */ 35, 22, 48, 0, 104, 398, 22, 35, 35, 402, - /* 1680 */ 44, 44, 405, 406, 407, 408, 409, 410, 328, 412, - /* 1690 */ 359, 35, 35, 35, 35, 35, 419, 0, 367, 102, - /* 1700 */ 423, 424, 22, 372, 0, 374, 22, 0, 22, 35, - /* 1710 */ 0, 35, 50, 0, 35, 0, 22, 35, 20, 359, - /* 1720 */ 35, 194, 96, 95, 0, 35, 163, 367, 0, 398, - /* 1730 */ 22, 185, 372, 402, 374, 0, 405, 406, 407, 408, - /* 1740 */ 409, 410, 0, 412, 3, 95, 328, 44, 249, 228, - /* 1750 */ 419, 96, 47, 253, 423, 424, 44, 3, 398, 95, - /* 1760 */ 44, 47, 402, 44, 96, 405, 406, 407, 408, 409, - /* 1770 */ 410, 411, 412, 413, 414, 44, 95, 359, 35, 249, - /* 1780 */ 249, 35, 35, 96, 95, 367, 95, 35, 96, 96, - /* 1790 */ 372, 95, 374, 163, 165, 96, 163, 35, 35, 0, - /* 1800 */ 328, 44, 171, 47, 243, 96, 47, 96, 170, 0, - /* 1810 */ 170, 0, 95, 166, 96, 95, 398, 96, 95, 0, - /* 1820 */ 402, 95, 39, 405, 406, 407, 408, 409, 410, 95, - /* 1830 */ 412, 359, 105, 47, 164, 44, 2, 22, 22, 367, - /* 1840 */ 95, 47, 47, 95, 372, 96, 374, 96, 95, 35, - /* 1850 */ 106, 119, 35, 95, 328, 35, 96, 96, 207, 95, - /* 1860 */ 95, 209, 96, 95, 35, 96, 95, 35, 35, 96, - /* 1870 */ 398, 95, 228, 96, 402, 457, 458, 405, 406, 407, - /* 1880 */ 408, 409, 410, 95, 412, 359, 96, 95, 22, 119, - /* 1890 */ 44, 419, 107, 367, 119, 95, 424, 119, 372, 95, - /* 1900 */ 374, 230, 35, 228, 22, 95, 328, 64, 63, 35, - /* 1910 */ 35, 35, 35, 35, 35, 35, 35, 70, 35, 35, - /* 1920 */ 92, 35, 328, 35, 398, 35, 22, 70, 402, 44, - /* 1930 */ 35, 405, 406, 407, 408, 409, 410, 359, 412, 22, - /* 1940 */ 35, 35, 35, 35, 35, 367, 35, 35, 35, 0, - /* 1950 */ 372, 35, 374, 359, 39, 0, 35, 39, 48, 0, - /* 1960 */ 35, 367, 48, 39, 0, 35, 372, 48, 374, 0, - /* 1970 */ 48, 39, 35, 447, 35, 0, 398, 20, 22, 21, - /* 1980 */ 402, 461, 22, 405, 406, 407, 408, 409, 410, 21, - /* 1990 */ 412, 461, 398, 22, 461, 461, 402, 461, 461, 405, - /* 2000 */ 406, 407, 408, 409, 410, 461, 412, 461, 461, 461, - /* 2010 */ 461, 328, 461, 461, 461, 461, 461, 461, 461, 461, - /* 2020 */ 461, 461, 461, 461, 461, 461, 461, 449, 461, 328, - /* 2030 */ 461, 461, 461, 461, 461, 461, 461, 461, 461, 461, - /* 2040 */ 461, 461, 359, 461, 461, 461, 461, 364, 461, 461, - /* 2050 */ 367, 461, 458, 461, 461, 372, 461, 374, 461, 461, - /* 2060 */ 359, 461, 461, 461, 461, 461, 461, 461, 367, 461, - /* 2070 */ 461, 461, 461, 372, 461, 374, 461, 461, 461, 461, - /* 2080 */ 461, 398, 461, 328, 461, 402, 461, 461, 405, 406, - /* 2090 */ 407, 408, 409, 410, 461, 412, 461, 461, 461, 398, - /* 2100 */ 328, 461, 461, 402, 461, 461, 405, 406, 407, 408, - /* 2110 */ 409, 410, 461, 412, 359, 414, 461, 461, 461, 364, - /* 2120 */ 461, 461, 367, 461, 461, 461, 461, 372, 461, 374, - /* 2130 */ 461, 359, 461, 461, 461, 461, 364, 461, 461, 367, - /* 2140 */ 461, 461, 461, 461, 372, 461, 374, 461, 461, 461, - /* 2150 */ 461, 461, 328, 398, 461, 461, 461, 402, 461, 461, - /* 2160 */ 405, 406, 407, 408, 409, 410, 461, 412, 328, 461, - /* 2170 */ 398, 461, 461, 461, 402, 461, 461, 405, 406, 407, - /* 2180 */ 408, 409, 410, 359, 412, 461, 461, 461, 461, 461, - /* 2190 */ 461, 367, 461, 461, 461, 461, 372, 461, 374, 359, - /* 2200 */ 461, 461, 461, 461, 461, 461, 461, 367, 461, 461, - /* 2210 */ 461, 461, 372, 461, 374, 461, 461, 461, 461, 461, - /* 2220 */ 328, 461, 398, 461, 461, 461, 402, 461, 461, 405, - /* 2230 */ 406, 407, 408, 409, 410, 461, 412, 461, 398, 461, - /* 2240 */ 461, 461, 402, 461, 461, 405, 406, 407, 408, 409, - /* 2250 */ 410, 359, 412, 461, 461, 461, 461, 461, 461, 367, - /* 2260 */ 461, 461, 461, 461, 372, 461, 374, 461, 461, 461, - /* 2270 */ 461, 461, 461, 461, 328, 461, 461, 461, 461, 461, - /* 2280 */ 461, 461, 461, 461, 461, 461, 461, 461, 461, 461, - /* 2290 */ 398, 461, 461, 461, 402, 328, 461, 405, 406, 407, - /* 2300 */ 408, 409, 410, 461, 412, 359, 461, 461, 461, 461, - /* 2310 */ 461, 461, 461, 367, 461, 461, 461, 461, 372, 461, - /* 2320 */ 374, 461, 461, 461, 461, 461, 359, 461, 461, 461, - /* 2330 */ 461, 461, 461, 461, 367, 461, 461, 461, 461, 372, - /* 2340 */ 461, 374, 461, 461, 398, 461, 461, 461, 402, 461, - /* 2350 */ 461, 405, 406, 407, 408, 409, 410, 461, 412, 461, - /* 2360 */ 461, 461, 461, 461, 461, 398, 461, 461, 461, 402, - /* 2370 */ 461, 328, 405, 406, 407, 408, 409, 410, 461, 412, - /* 2380 */ 461, 461, 461, 461, 461, 461, 461, 461, 461, 328, - /* 2390 */ 461, 461, 461, 461, 461, 461, 461, 461, 461, 461, - /* 2400 */ 461, 461, 359, 461, 461, 461, 461, 461, 461, 461, - /* 2410 */ 367, 461, 461, 461, 461, 372, 461, 374, 461, 461, - /* 2420 */ 359, 461, 461, 461, 461, 461, 461, 461, 367, 461, - /* 2430 */ 461, 461, 461, 372, 461, 374, 461, 461, 461, 461, - /* 2440 */ 461, 398, 461, 328, 461, 402, 461, 461, 405, 406, - /* 2450 */ 407, 408, 409, 410, 461, 412, 461, 461, 461, 398, - /* 2460 */ 328, 461, 461, 402, 461, 461, 405, 406, 407, 408, - /* 2470 */ 409, 410, 461, 412, 359, 461, 461, 461, 461, 461, - /* 2480 */ 461, 461, 367, 461, 461, 461, 461, 372, 461, 374, - /* 2490 */ 461, 359, 461, 461, 461, 461, 461, 461, 461, 367, - /* 2500 */ 461, 461, 461, 461, 372, 461, 374, 461, 461, 461, - /* 2510 */ 461, 461, 328, 398, 461, 461, 461, 402, 461, 461, - /* 2520 */ 405, 406, 407, 408, 409, 410, 461, 412, 328, 461, - /* 2530 */ 398, 461, 461, 461, 402, 461, 461, 405, 406, 407, - /* 2540 */ 408, 409, 410, 359, 412, 461, 461, 461, 461, 461, - /* 2550 */ 461, 367, 461, 461, 461, 461, 372, 461, 374, 359, - /* 2560 */ 461, 461, 461, 461, 461, 461, 461, 367, 461, 461, - /* 2570 */ 461, 461, 372, 461, 374, 461, 461, 461, 461, 461, - /* 2580 */ 328, 461, 398, 461, 461, 461, 402, 461, 461, 405, - /* 2590 */ 406, 407, 408, 409, 410, 461, 412, 461, 398, 461, - /* 2600 */ 461, 461, 402, 328, 461, 405, 406, 407, 408, 409, - /* 2610 */ 410, 359, 412, 461, 461, 461, 461, 461, 461, 367, - /* 2620 */ 461, 461, 461, 461, 372, 461, 374, 461, 461, 461, - /* 2630 */ 461, 461, 461, 461, 359, 461, 461, 461, 461, 461, - /* 2640 */ 461, 461, 367, 461, 461, 461, 461, 372, 461, 374, - /* 2650 */ 398, 461, 461, 461, 402, 328, 461, 405, 406, 407, - /* 2660 */ 408, 409, 410, 461, 412, 461, 461, 461, 461, 461, - /* 2670 */ 461, 461, 461, 398, 461, 461, 461, 402, 461, 461, - /* 2680 */ 405, 406, 407, 408, 409, 410, 359, 412, 461, 461, - /* 2690 */ 461, 461, 461, 461, 367, 461, 461, 461, 461, 372, - /* 2700 */ 461, 374, 461, 461, 461, 461, 461, 461, 461, 461, - /* 2710 */ 461, 461, 461, 461, 461, 461, 461, 461, 328, 461, - /* 2720 */ 461, 461, 461, 461, 461, 398, 461, 461, 461, 402, - /* 2730 */ 461, 461, 405, 406, 407, 408, 409, 410, 461, 412, - /* 2740 */ 461, 461, 461, 461, 461, 461, 461, 461, 461, 359, - /* 2750 */ 461, 461, 461, 461, 461, 461, 461, 367, 461, 461, - /* 2760 */ 461, 461, 372, 461, 374, 461, 461, 461, 461, 461, - /* 2770 */ 328, 461, 461, 461, 461, 461, 461, 461, 461, 461, - /* 2780 */ 461, 461, 461, 461, 461, 461, 461, 461, 398, 328, - /* 2790 */ 461, 461, 402, 461, 461, 405, 406, 407, 408, 409, - /* 2800 */ 410, 359, 412, 461, 461, 461, 461, 461, 461, 367, - /* 2810 */ 461, 461, 461, 461, 372, 461, 374, 461, 461, 461, - /* 2820 */ 359, 461, 461, 461, 461, 461, 461, 461, 367, 461, - /* 2830 */ 461, 461, 461, 372, 461, 374, 461, 461, 461, 461, - /* 2840 */ 398, 328, 461, 461, 402, 461, 461, 405, 406, 407, - /* 2850 */ 408, 409, 410, 461, 412, 461, 461, 328, 461, 398, - /* 2860 */ 461, 461, 461, 402, 461, 461, 405, 406, 407, 408, - /* 2870 */ 409, 410, 359, 412, 461, 461, 461, 461, 461, 461, - /* 2880 */ 367, 461, 461, 461, 461, 372, 461, 374, 359, 461, - /* 2890 */ 461, 461, 461, 461, 461, 461, 367, 461, 461, 461, - /* 2900 */ 461, 372, 461, 374, 461, 461, 461, 461, 461, 328, - /* 2910 */ 461, 398, 461, 461, 461, 402, 461, 461, 405, 406, - /* 2920 */ 407, 408, 409, 410, 461, 412, 461, 398, 461, 461, - /* 2930 */ 461, 402, 328, 461, 405, 406, 407, 408, 409, 410, - /* 2940 */ 359, 412, 461, 461, 461, 461, 461, 461, 367, 461, - /* 2950 */ 461, 461, 461, 372, 461, 374, 461, 461, 461, 461, - /* 2960 */ 461, 461, 461, 359, 461, 461, 461, 461, 461, 461, - /* 2970 */ 461, 367, 461, 461, 461, 461, 372, 461, 374, 398, - /* 2980 */ 461, 461, 461, 402, 328, 461, 405, 406, 407, 408, - /* 2990 */ 409, 410, 461, 412, 461, 461, 461, 461, 461, 461, - /* 3000 */ 461, 461, 398, 461, 461, 461, 402, 461, 461, 405, - /* 3010 */ 406, 407, 408, 409, 410, 359, 412, 461, 461, 461, - /* 3020 */ 461, 461, 461, 367, 461, 461, 461, 461, 372, 461, - /* 3030 */ 374, 461, 461, 461, 461, 461, 461, 461, 461, 461, - /* 3040 */ 461, 461, 461, 461, 461, 461, 461, 461, 461, 461, - /* 3050 */ 461, 461, 461, 461, 398, 461, 461, 461, 402, 461, - /* 3060 */ 461, 405, 406, 407, 408, 409, 410, 461, 412, 325, - /* 3070 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3080 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3090 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3100 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3110 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3120 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3130 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3140 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3150 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3160 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3170 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3180 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3190 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3200 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3210 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3220 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3230 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3240 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3250 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3260 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3270 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3280 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3290 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3300 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3310 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3320 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3330 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3340 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3350 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3360 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3370 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3380 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3390 */ 325, 325, 325, 325, + /* 0 */ 375, 333, 371, 335, 336, 374, 375, 333, 362, 335, + /* 10 */ 336, 375, 12, 13, 14, 390, 391, 360, 330, 373, + /* 20 */ 20, 362, 22, 387, 360, 362, 390, 391, 329, 424, + /* 30 */ 425, 367, 373, 33, 329, 35, 373, 20, 381, 382, + /* 40 */ 376, 334, 8, 9, 337, 338, 12, 13, 14, 15, + /* 50 */ 16, 405, 406, 407, 337, 337, 20, 435, 58, 360, + /* 60 */ 20, 439, 416, 63, 405, 406, 407, 368, 405, 406, + /* 70 */ 70, 337, 373, 326, 375, 416, 454, 455, 373, 416, + /* 80 */ 360, 459, 460, 12, 13, 351, 368, 367, 20, 4, + /* 90 */ 20, 20, 358, 22, 377, 95, 376, 63, 20, 435, + /* 100 */ 360, 402, 368, 439, 33, 406, 35, 367, 409, 410, + /* 110 */ 411, 412, 413, 414, 4, 416, 376, 117, 454, 455, + /* 120 */ 421, 58, 423, 459, 460, 58, 427, 428, 43, 58, + /* 130 */ 45, 46, 132, 133, 63, 95, 362, 20, 104, 392, + /* 140 */ 441, 70, 12, 13, 14, 15, 16, 373, 449, 431, + /* 150 */ 432, 433, 334, 435, 436, 337, 338, 439, 95, 360, + /* 160 */ 97, 94, 162, 163, 97, 95, 95, 368, 168, 169, + /* 170 */ 126, 127, 454, 455, 328, 131, 330, 459, 460, 405, + /* 180 */ 406, 14, 435, 183, 39, 185, 439, 20, 117, 37, + /* 190 */ 416, 353, 8, 9, 356, 161, 12, 13, 14, 15, + /* 200 */ 16, 454, 455, 132, 133, 95, 459, 460, 208, 209, + /* 210 */ 411, 211, 212, 213, 214, 215, 216, 217, 218, 219, + /* 220 */ 220, 221, 222, 223, 224, 225, 226, 227, 228, 167, + /* 230 */ 162, 163, 20, 162, 163, 95, 62, 343, 21, 168, + /* 240 */ 169, 24, 25, 26, 27, 28, 29, 30, 31, 32, + /* 250 */ 98, 357, 100, 101, 183, 103, 185, 8, 9, 107, + /* 260 */ 366, 12, 13, 14, 15, 16, 62, 233, 234, 235, + /* 270 */ 236, 237, 238, 239, 240, 241, 242, 243, 360, 208, + /* 280 */ 209, 129, 211, 212, 213, 214, 215, 216, 217, 218, + /* 290 */ 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, + /* 300 */ 382, 230, 12, 13, 0, 337, 20, 329, 22, 246, + /* 310 */ 20, 329, 22, 251, 252, 253, 246, 230, 329, 232, + /* 320 */ 95, 35, 182, 33, 184, 35, 108, 109, 110, 111, + /* 330 */ 112, 113, 114, 115, 116, 117, 118, 51, 120, 121, + /* 340 */ 122, 123, 124, 125, 14, 96, 178, 3, 58, 360, + /* 350 */ 20, 373, 384, 63, 386, 373, 246, 368, 337, 175, + /* 360 */ 70, 20, 373, 22, 375, 20, 20, 199, 200, 65, + /* 370 */ 66, 67, 351, 12, 13, 14, 164, 73, 74, 358, + /* 380 */ 408, 20, 78, 22, 47, 95, 246, 83, 84, 368, + /* 390 */ 245, 402, 51, 89, 33, 406, 35, 337, 409, 410, + /* 400 */ 411, 412, 413, 414, 329, 416, 434, 117, 419, 435, + /* 410 */ 421, 422, 423, 439, 2, 33, 427, 428, 161, 58, + /* 420 */ 8, 9, 132, 133, 12, 13, 14, 15, 16, 455, + /* 430 */ 48, 70, 95, 459, 460, 360, 54, 55, 56, 57, + /* 440 */ 58, 95, 70, 368, 384, 14, 386, 3, 373, 337, + /* 450 */ 375, 20, 162, 163, 270, 20, 95, 21, 168, 169, + /* 460 */ 341, 8, 9, 351, 20, 12, 13, 14, 15, 16, + /* 470 */ 34, 246, 36, 183, 355, 185, 94, 402, 117, 97, + /* 480 */ 368, 406, 363, 79, 409, 410, 411, 412, 413, 414, + /* 490 */ 233, 416, 162, 132, 133, 329, 106, 44, 208, 209, + /* 500 */ 243, 211, 212, 213, 214, 215, 216, 217, 218, 219, + /* 510 */ 220, 221, 222, 223, 224, 225, 226, 227, 228, 65, + /* 520 */ 66, 67, 343, 162, 163, 450, 451, 73, 74, 168, + /* 530 */ 169, 344, 78, 230, 95, 337, 349, 83, 84, 373, + /* 540 */ 96, 137, 138, 89, 183, 366, 185, 165, 166, 351, + /* 550 */ 329, 70, 170, 8, 9, 173, 361, 12, 13, 14, + /* 560 */ 15, 16, 20, 337, 160, 20, 368, 132, 133, 208, + /* 570 */ 209, 189, 211, 212, 213, 214, 215, 216, 217, 218, + /* 580 */ 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, + /* 590 */ 12, 13, 246, 162, 373, 107, 374, 375, 20, 0, + /* 600 */ 22, 346, 347, 168, 169, 208, 329, 0, 164, 265, + /* 610 */ 384, 33, 386, 35, 126, 127, 128, 129, 130, 131, + /* 620 */ 21, 361, 337, 24, 25, 26, 27, 28, 29, 30, + /* 630 */ 31, 32, 1, 2, 8, 9, 58, 360, 12, 13, + /* 640 */ 14, 15, 16, 435, 0, 368, 337, 439, 70, 361, + /* 650 */ 373, 359, 375, 368, 257, 258, 259, 260, 261, 368, + /* 660 */ 351, 12, 13, 455, 372, 360, 127, 459, 460, 20, + /* 670 */ 379, 22, 367, 95, 132, 133, 126, 368, 420, 402, + /* 680 */ 422, 376, 33, 406, 35, 246, 409, 410, 411, 412, + /* 690 */ 413, 414, 329, 416, 337, 117, 360, 0, 421, 420, + /* 700 */ 423, 422, 35, 367, 427, 428, 62, 58, 351, 164, + /* 710 */ 132, 133, 376, 329, 361, 430, 431, 432, 433, 70, + /* 720 */ 435, 436, 96, 360, 0, 368, 449, 96, 361, 190, + /* 730 */ 191, 368, 22, 194, 329, 196, 373, 70, 375, 329, + /* 740 */ 162, 163, 192, 193, 95, 35, 168, 169, 24, 25, + /* 750 */ 26, 27, 28, 29, 30, 31, 32, 373, 151, 62, + /* 760 */ 361, 183, 348, 185, 350, 402, 117, 160, 361, 406, + /* 770 */ 346, 347, 409, 410, 411, 412, 413, 414, 373, 416, + /* 780 */ 70, 132, 133, 373, 408, 329, 208, 209, 392, 211, + /* 790 */ 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, + /* 800 */ 222, 223, 224, 225, 226, 227, 228, 337, 368, 368, + /* 810 */ 434, 162, 163, 18, 451, 20, 337, 168, 169, 379, + /* 820 */ 379, 351, 27, 337, 341, 30, 408, 117, 33, 373, + /* 830 */ 351, 435, 183, 360, 185, 439, 352, 351, 368, 396, + /* 840 */ 14, 15, 16, 48, 360, 50, 363, 368, 53, 376, + /* 850 */ 454, 455, 434, 369, 368, 459, 460, 208, 209, 0, + /* 860 */ 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, + /* 870 */ 221, 222, 223, 224, 225, 226, 227, 228, 0, 352, + /* 880 */ 45, 46, 360, 18, 329, 337, 337, 360, 23, 94, + /* 890 */ 0, 369, 107, 183, 0, 185, 369, 20, 2, 351, + /* 900 */ 351, 106, 37, 38, 8, 9, 41, 48, 12, 13, + /* 910 */ 14, 15, 16, 352, 129, 349, 368, 368, 208, 209, + /* 920 */ 419, 360, 463, 422, 59, 60, 61, 383, 373, 370, + /* 930 */ 369, 136, 373, 4, 139, 140, 141, 142, 143, 144, + /* 940 */ 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, + /* 950 */ 155, 452, 157, 158, 159, 65, 66, 67, 68, 69, + /* 960 */ 95, 71, 72, 73, 74, 75, 76, 77, 78, 79, + /* 970 */ 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + /* 980 */ 90, 91, 329, 4, 20, 107, 8, 9, 337, 22, + /* 990 */ 12, 13, 14, 15, 16, 132, 133, 20, 19, 134, + /* 1000 */ 337, 107, 35, 337, 126, 127, 128, 129, 130, 131, + /* 1010 */ 244, 245, 33, 329, 351, 337, 329, 351, 44, 368, + /* 1020 */ 126, 127, 128, 129, 130, 131, 373, 48, 337, 351, + /* 1030 */ 337, 368, 53, 329, 368, 329, 337, 58, 44, 162, + /* 1040 */ 175, 176, 177, 446, 351, 180, 368, 360, 329, 370, + /* 1050 */ 351, 370, 373, 353, 373, 368, 356, 373, 0, 368, + /* 1060 */ 373, 368, 375, 198, 337, 329, 201, 368, 203, 204, + /* 1070 */ 205, 206, 207, 94, 329, 44, 97, 373, 351, 373, + /* 1080 */ 22, 35, 431, 432, 433, 329, 435, 436, 337, 402, + /* 1090 */ 96, 339, 373, 406, 106, 368, 409, 410, 411, 412, + /* 1100 */ 413, 414, 351, 416, 329, 360, 1, 2, 421, 373, + /* 1110 */ 423, 246, 44, 368, 427, 428, 339, 360, 373, 368, + /* 1120 */ 375, 329, 431, 432, 433, 438, 435, 436, 164, 373, + /* 1130 */ 329, 337, 8, 9, 336, 360, 12, 13, 14, 15, + /* 1140 */ 16, 164, 164, 368, 156, 351, 329, 402, 373, 337, + /* 1150 */ 375, 406, 360, 0, 409, 410, 411, 412, 413, 414, + /* 1160 */ 368, 416, 368, 351, 96, 373, 421, 375, 423, 42, + /* 1170 */ 0, 44, 427, 428, 373, 22, 247, 402, 329, 58, + /* 1180 */ 368, 406, 392, 438, 409, 410, 411, 412, 413, 414, + /* 1190 */ 373, 416, 22, 42, 402, 44, 99, 99, 406, 102, + /* 1200 */ 102, 409, 410, 411, 412, 413, 414, 35, 416, 360, + /* 1210 */ 383, 99, 372, 421, 102, 423, 164, 368, 97, 427, + /* 1220 */ 428, 392, 373, 171, 375, 435, 44, 63, 453, 439, + /* 1230 */ 438, 185, 99, 12, 13, 102, 329, 383, 44, 437, + /* 1240 */ 44, 267, 44, 22, 454, 455, 440, 44, 429, 459, + /* 1250 */ 460, 402, 44, 0, 33, 406, 35, 248, 409, 410, + /* 1260 */ 411, 412, 413, 414, 435, 416, 329, 360, 439, 456, + /* 1270 */ 421, 35, 423, 44, 44, 368, 427, 428, 96, 58, + /* 1280 */ 373, 44, 375, 454, 455, 95, 13, 438, 459, 460, + /* 1290 */ 96, 70, 96, 404, 96, 105, 48, 360, 329, 96, + /* 1300 */ 269, 403, 49, 44, 96, 368, 70, 44, 35, 402, + /* 1310 */ 373, 44, 375, 406, 394, 44, 409, 410, 411, 412, + /* 1320 */ 413, 414, 44, 416, 44, 96, 96, 181, 421, 360, + /* 1330 */ 423, 42, 13, 96, 427, 428, 44, 368, 117, 402, + /* 1340 */ 20, 380, 373, 406, 375, 438, 409, 410, 411, 412, + /* 1350 */ 413, 414, 392, 416, 35, 96, 383, 185, 421, 96, + /* 1360 */ 423, 380, 161, 96, 427, 428, 20, 96, 378, 337, + /* 1370 */ 378, 402, 208, 337, 96, 406, 96, 329, 409, 410, + /* 1380 */ 411, 412, 413, 414, 380, 416, 378, 93, 96, 345, + /* 1390 */ 421, 337, 423, 337, 337, 435, 427, 428, 20, 439, + /* 1400 */ 331, 331, 20, 398, 183, 343, 185, 20, 360, 375, + /* 1410 */ 343, 20, 338, 20, 454, 455, 368, 393, 343, 459, + /* 1420 */ 460, 373, 338, 375, 343, 392, 343, 337, 343, 208, + /* 1430 */ 209, 343, 52, 340, 340, 329, 360, 360, 331, 360, + /* 1440 */ 360, 337, 221, 222, 223, 224, 225, 226, 227, 360, + /* 1450 */ 402, 360, 360, 331, 406, 329, 360, 409, 410, 411, + /* 1460 */ 412, 413, 414, 360, 416, 197, 360, 401, 435, 421, + /* 1470 */ 360, 423, 439, 360, 368, 427, 428, 341, 95, 373, + /* 1480 */ 398, 375, 373, 188, 373, 341, 360, 454, 455, 375, + /* 1490 */ 256, 337, 459, 460, 368, 400, 397, 255, 383, 373, + /* 1500 */ 383, 375, 373, 373, 445, 445, 388, 388, 402, 262, + /* 1510 */ 373, 373, 406, 329, 174, 409, 410, 411, 412, 413, + /* 1520 */ 414, 448, 416, 249, 264, 447, 271, 263, 402, 423, + /* 1530 */ 443, 445, 406, 427, 428, 409, 410, 411, 412, 413, + /* 1540 */ 414, 266, 416, 329, 360, 444, 404, 268, 245, 423, + /* 1550 */ 442, 368, 368, 427, 428, 20, 408, 373, 338, 375, + /* 1560 */ 329, 337, 20, 341, 464, 386, 341, 166, 388, 388, + /* 1570 */ 373, 385, 373, 457, 360, 373, 341, 373, 458, 373, + /* 1580 */ 373, 341, 368, 356, 368, 95, 402, 373, 426, 375, + /* 1590 */ 406, 360, 95, 409, 410, 411, 412, 413, 414, 368, + /* 1600 */ 416, 373, 364, 337, 373, 36, 375, 423, 350, 332, + /* 1610 */ 341, 427, 428, 331, 395, 389, 402, 329, 342, 354, + /* 1620 */ 406, 389, 399, 409, 410, 411, 412, 413, 414, 415, + /* 1630 */ 416, 417, 418, 402, 327, 354, 0, 406, 354, 0, + /* 1640 */ 409, 410, 411, 412, 413, 414, 190, 416, 360, 0, + /* 1650 */ 0, 42, 0, 35, 202, 35, 368, 35, 35, 202, + /* 1660 */ 0, 373, 35, 375, 329, 35, 202, 0, 202, 0, + /* 1670 */ 35, 0, 0, 22, 185, 35, 183, 0, 0, 179, + /* 1680 */ 178, 329, 0, 0, 47, 0, 0, 0, 0, 42, + /* 1690 */ 402, 0, 461, 462, 406, 360, 0, 409, 410, 411, + /* 1700 */ 412, 413, 414, 368, 416, 0, 0, 0, 373, 151, + /* 1710 */ 375, 423, 360, 0, 0, 35, 428, 365, 0, 151, + /* 1720 */ 368, 0, 0, 0, 0, 373, 0, 375, 0, 0, + /* 1730 */ 0, 0, 0, 0, 0, 0, 329, 402, 0, 0, + /* 1740 */ 0, 406, 42, 0, 409, 410, 411, 412, 413, 414, + /* 1750 */ 0, 416, 0, 0, 402, 0, 0, 0, 406, 329, + /* 1760 */ 0, 409, 410, 411, 412, 413, 414, 360, 416, 22, + /* 1770 */ 0, 0, 0, 135, 0, 368, 35, 58, 0, 0, + /* 1780 */ 373, 0, 375, 58, 42, 39, 0, 14, 44, 14, + /* 1790 */ 360, 329, 47, 0, 0, 365, 47, 462, 368, 40, + /* 1800 */ 39, 47, 0, 373, 39, 375, 174, 0, 0, 402, + /* 1810 */ 0, 0, 64, 406, 0, 35, 409, 410, 411, 412, + /* 1820 */ 413, 414, 360, 416, 48, 418, 39, 365, 0, 39, + /* 1830 */ 368, 35, 402, 48, 0, 373, 406, 375, 329, 409, + /* 1840 */ 410, 411, 412, 413, 414, 48, 416, 35, 39, 0, + /* 1850 */ 35, 48, 19, 0, 0, 39, 0, 0, 35, 22, + /* 1860 */ 102, 0, 329, 44, 402, 35, 33, 104, 406, 360, + /* 1870 */ 35, 409, 410, 411, 412, 413, 414, 368, 416, 35, + /* 1880 */ 35, 48, 373, 44, 375, 35, 35, 54, 55, 56, + /* 1890 */ 57, 58, 22, 360, 35, 0, 22, 0, 22, 0, + /* 1900 */ 22, 368, 50, 35, 0, 0, 373, 35, 375, 0, + /* 1910 */ 35, 402, 22, 20, 35, 406, 35, 96, 409, 410, + /* 1920 */ 411, 412, 413, 414, 95, 416, 0, 94, 329, 35, + /* 1930 */ 97, 195, 164, 0, 22, 402, 0, 186, 0, 406, + /* 1940 */ 3, 164, 409, 410, 411, 412, 413, 414, 166, 416, + /* 1950 */ 329, 44, 164, 250, 96, 172, 229, 95, 254, 360, + /* 1960 */ 95, 44, 171, 130, 3, 96, 171, 368, 44, 96, + /* 1970 */ 47, 44, 373, 96, 375, 329, 47, 95, 95, 95, + /* 1980 */ 95, 360, 96, 96, 44, 35, 35, 35, 35, 368, + /* 1990 */ 35, 35, 96, 47, 373, 47, 375, 96, 165, 39, + /* 2000 */ 0, 402, 44, 170, 0, 406, 360, 0, 409, 410, + /* 2010 */ 411, 412, 413, 414, 368, 416, 250, 0, 0, 373, + /* 2020 */ 187, 375, 189, 402, 47, 95, 95, 406, 244, 95, + /* 2030 */ 409, 410, 411, 412, 413, 414, 95, 416, 329, 250, + /* 2040 */ 96, 96, 95, 47, 39, 44, 229, 167, 402, 95, + /* 2050 */ 2, 231, 406, 229, 165, 409, 410, 411, 412, 413, + /* 2060 */ 414, 22, 416, 208, 105, 95, 329, 96, 95, 360, + /* 2070 */ 96, 95, 47, 47, 96, 95, 95, 368, 106, 96, + /* 2080 */ 22, 210, 373, 95, 375, 96, 35, 35, 95, 35, + /* 2090 */ 96, 96, 95, 95, 35, 35, 329, 360, 96, 96, + /* 2100 */ 35, 95, 22, 95, 119, 368, 119, 95, 44, 119, + /* 2110 */ 373, 402, 375, 22, 95, 406, 107, 35, 409, 410, + /* 2120 */ 411, 412, 413, 414, 95, 416, 64, 360, 329, 119, + /* 2130 */ 63, 35, 35, 35, 35, 368, 35, 35, 35, 402, + /* 2140 */ 373, 35, 375, 406, 35, 35, 409, 410, 411, 412, + /* 2150 */ 413, 414, 44, 416, 70, 92, 35, 35, 35, 360, + /* 2160 */ 329, 22, 35, 35, 35, 70, 35, 368, 35, 402, + /* 2170 */ 35, 35, 373, 406, 375, 35, 409, 410, 411, 412, + /* 2180 */ 413, 414, 22, 416, 35, 0, 35, 39, 0, 48, + /* 2190 */ 35, 360, 329, 48, 0, 39, 35, 39, 48, 368, + /* 2200 */ 0, 402, 35, 39, 373, 406, 375, 48, 409, 410, + /* 2210 */ 411, 412, 413, 414, 0, 416, 35, 329, 35, 0, + /* 2220 */ 22, 22, 21, 360, 22, 21, 20, 465, 465, 465, + /* 2230 */ 465, 368, 465, 402, 465, 465, 373, 406, 375, 465, + /* 2240 */ 409, 410, 411, 412, 413, 414, 465, 416, 360, 329, + /* 2250 */ 465, 465, 465, 465, 465, 465, 368, 465, 465, 465, + /* 2260 */ 465, 373, 465, 375, 465, 402, 465, 465, 465, 406, + /* 2270 */ 465, 465, 409, 410, 411, 412, 413, 414, 465, 416, + /* 2280 */ 360, 465, 465, 465, 465, 465, 465, 465, 368, 465, + /* 2290 */ 402, 465, 465, 373, 406, 375, 465, 409, 410, 411, + /* 2300 */ 412, 413, 414, 465, 416, 465, 465, 465, 465, 465, + /* 2310 */ 465, 329, 465, 465, 465, 465, 465, 465, 465, 465, + /* 2320 */ 465, 465, 402, 465, 465, 465, 406, 465, 329, 409, + /* 2330 */ 410, 411, 412, 413, 414, 465, 416, 465, 465, 465, + /* 2340 */ 465, 465, 360, 465, 465, 465, 465, 465, 465, 465, + /* 2350 */ 368, 465, 465, 465, 465, 373, 465, 375, 465, 360, + /* 2360 */ 465, 465, 465, 465, 465, 465, 465, 368, 465, 465, + /* 2370 */ 465, 465, 373, 465, 375, 465, 465, 465, 465, 465, + /* 2380 */ 465, 465, 465, 465, 402, 329, 465, 465, 406, 465, + /* 2390 */ 465, 409, 410, 411, 412, 413, 414, 465, 416, 465, + /* 2400 */ 465, 402, 465, 465, 465, 406, 465, 329, 409, 410, + /* 2410 */ 411, 412, 413, 414, 465, 416, 360, 465, 465, 465, + /* 2420 */ 465, 465, 465, 465, 368, 465, 465, 465, 465, 373, + /* 2430 */ 465, 375, 465, 465, 465, 465, 465, 465, 360, 465, + /* 2440 */ 465, 465, 465, 465, 465, 465, 368, 465, 465, 465, + /* 2450 */ 465, 373, 465, 375, 465, 465, 465, 465, 402, 465, + /* 2460 */ 465, 465, 406, 465, 465, 409, 410, 411, 412, 413, + /* 2470 */ 414, 465, 416, 329, 465, 465, 465, 465, 465, 465, + /* 2480 */ 402, 465, 465, 465, 406, 465, 465, 409, 410, 411, + /* 2490 */ 412, 413, 414, 465, 416, 465, 465, 329, 465, 465, + /* 2500 */ 465, 465, 465, 465, 360, 465, 465, 465, 465, 465, + /* 2510 */ 465, 465, 368, 465, 465, 465, 465, 373, 465, 375, + /* 2520 */ 465, 465, 465, 465, 465, 465, 465, 465, 360, 465, + /* 2530 */ 465, 465, 465, 465, 465, 465, 368, 465, 465, 465, + /* 2540 */ 465, 373, 465, 375, 465, 465, 402, 465, 465, 465, + /* 2550 */ 406, 465, 465, 409, 410, 411, 412, 413, 414, 465, + /* 2560 */ 416, 329, 465, 465, 465, 465, 465, 465, 465, 465, + /* 2570 */ 402, 465, 465, 465, 406, 465, 465, 409, 410, 411, + /* 2580 */ 412, 413, 414, 465, 416, 465, 329, 465, 465, 465, + /* 2590 */ 465, 465, 360, 465, 465, 465, 465, 465, 465, 465, + /* 2600 */ 368, 465, 465, 465, 465, 373, 465, 375, 465, 465, + /* 2610 */ 465, 465, 465, 465, 465, 465, 465, 360, 329, 465, + /* 2620 */ 465, 465, 465, 465, 465, 368, 465, 465, 465, 465, + /* 2630 */ 373, 465, 375, 465, 402, 465, 465, 465, 406, 465, + /* 2640 */ 465, 409, 410, 411, 412, 413, 414, 465, 416, 360, + /* 2650 */ 465, 465, 465, 465, 465, 465, 465, 368, 465, 402, + /* 2660 */ 465, 465, 373, 406, 375, 465, 409, 410, 411, 412, + /* 2670 */ 413, 414, 465, 416, 465, 465, 465, 465, 465, 465, + /* 2680 */ 465, 465, 465, 465, 465, 465, 465, 465, 465, 465, + /* 2690 */ 465, 402, 465, 465, 465, 406, 465, 465, 409, 410, + /* 2700 */ 411, 412, 413, 414, 465, 416, }; -#define YY_SHIFT_COUNT (713) +#define YY_SHIFT_COUNT (729) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (1975) +#define YY_SHIFT_MAX (2219) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 862, 0, 71, 0, 289, 289, 289, 289, 289, 289, - /* 10 */ 289, 289, 289, 289, 289, 360, 576, 576, 647, 576, - /* 20 */ 576, 576, 576, 576, 576, 576, 576, 576, 576, 576, - /* 30 */ 576, 576, 576, 576, 576, 576, 576, 576, 576, 576, - /* 40 */ 576, 576, 576, 576, 576, 576, 576, 576, 32, 134, - /* 50 */ 21, 590, 133, 7, 104, 7, 21, 21, 1216, 1216, - /* 60 */ 7, 1216, 1216, 136, 7, 423, 25, 25, 423, 401, - /* 70 */ 401, 215, 57, 5, 5, 25, 25, 25, 25, 25, - /* 80 */ 25, 25, 25, 25, 25, 80, 25, 25, 156, 25, - /* 90 */ 25, 25, 343, 25, 25, 343, 25, 343, 343, 343, - /* 100 */ 25, 328, 792, 34, 34, 216, 474, 283, 283, 283, - /* 110 */ 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, - /* 120 */ 283, 283, 283, 283, 283, 283, 1073, 87, 215, 57, - /* 130 */ 593, 313, 499, 499, 499, 710, 229, 229, 313, 374, - /* 140 */ 374, 374, 143, 384, 343, 378, 343, 378, 378, 143, - /* 150 */ 566, 217, 217, 217, 217, 217, 217, 217, 1351, 408, - /* 160 */ 336, 612, 249, 430, 340, 464, 292, 714, 377, 518, - /* 170 */ 873, 656, 858, 748, 12, 754, 748, 769, 353, 836, - /* 180 */ 977, 1192, 1055, 1202, 1226, 1202, 1094, 1236, 1236, 1202, - /* 190 */ 1094, 1094, 1180, 1236, 1236, 1236, 1265, 1265, 1270, 80, - /* 200 */ 1278, 80, 1280, 1283, 80, 1280, 80, 80, 80, 1236, - /* 210 */ 80, 1260, 1260, 1265, 343, 343, 343, 343, 343, 343, - /* 220 */ 343, 343, 343, 343, 343, 1236, 1265, 378, 378, 1129, - /* 230 */ 1270, 328, 1177, 1278, 328, 1236, 1226, 1226, 378, 1117, - /* 240 */ 1124, 378, 1117, 1124, 378, 378, 343, 1122, 1213, 1117, - /* 250 */ 1130, 1132, 1148, 977, 1127, 1134, 1137, 1159, 374, 1398, - /* 260 */ 1236, 1280, 328, 1124, 378, 378, 378, 378, 378, 1124, - /* 270 */ 378, 1262, 328, 143, 328, 374, 1361, 1366, 378, 566, - /* 280 */ 1236, 328, 1437, 1265, 3069, 3069, 3069, 3069, 3069, 3069, - /* 290 */ 3069, 3069, 3069, 887, 310, 94, 625, 15, 59, 738, - /* 300 */ 720, 753, 881, 73, 781, 472, 472, 472, 472, 472, - /* 310 */ 472, 472, 472, 472, 875, 124, 373, 373, 122, 4, - /* 320 */ 58, 166, 484, 300, 300, 790, 912, 588, 790, 790, - /* 330 */ 790, 113, 1057, 288, 969, 913, 884, 918, 923, 986, - /* 340 */ 988, 1093, 1101, 1106, 1005, 1042, 1090, 1011, 425, 945, - /* 350 */ 863, 1096, 1103, 1115, 1116, 1120, 1187, 1131, 1010, 1041, - /* 360 */ 1046, 1140, 1074, 1141, 1147, 1151, 1176, 1179, 1195, 825, - /* 370 */ 1142, 1166, 489, 1181, 1482, 1487, 1300, 1490, 1493, 1452, - /* 380 */ 1495, 1461, 1296, 1463, 1464, 1465, 1302, 1501, 1469, 1470, - /* 390 */ 1306, 1508, 1314, 1518, 1484, 1522, 1502, 1529, 1491, 1347, - /* 400 */ 1352, 1533, 1535, 1358, 1363, 1538, 1541, 1496, 1542, 1544, - /* 410 */ 1545, 1504, 1548, 1549, 1550, 1551, 1552, 1553, 1554, 1555, - /* 420 */ 1406, 1523, 1559, 1410, 1562, 1569, 1571, 1572, 1573, 1574, - /* 430 */ 1576, 1579, 1580, 1583, 1584, 1585, 1587, 1588, 1589, 1556, - /* 440 */ 1590, 1592, 1599, 1600, 1601, 1575, 1602, 1603, 1604, 1459, - /* 450 */ 1595, 1596, 1577, 1606, 1557, 1607, 1558, 1608, 1609, 1568, - /* 460 */ 1581, 1567, 1566, 1605, 1586, 1616, 1593, 1614, 1582, 1598, - /* 470 */ 1618, 1623, 1632, 1611, 1462, 1634, 1639, 1641, 1591, 1645, - /* 480 */ 1646, 1612, 1610, 1613, 1648, 1619, 1615, 1617, 1653, 1622, - /* 490 */ 1620, 1621, 1661, 1627, 1624, 1625, 1665, 1666, 1667, 1669, - /* 500 */ 1570, 1597, 1635, 1649, 1673, 1642, 1643, 1656, 1657, 1636, - /* 510 */ 1637, 1658, 1659, 1654, 1660, 1697, 1680, 1704, 1684, 1662, - /* 520 */ 1707, 1686, 1674, 1710, 1676, 1713, 1679, 1715, 1694, 1698, - /* 530 */ 1682, 1685, 1527, 1626, 1628, 1724, 1563, 1690, 1728, 1546, - /* 540 */ 1708, 1630, 1629, 1735, 1742, 1633, 1631, 1741, 1703, 1499, - /* 550 */ 1650, 1655, 1664, 1638, 1521, 1640, 1500, 1668, 1712, 1687, - /* 560 */ 1681, 1689, 1691, 1692, 1716, 1705, 1714, 1696, 1719, 1530, - /* 570 */ 1693, 1699, 1754, 1731, 1531, 1743, 1746, 1747, 1752, 1762, - /* 580 */ 1763, 1709, 1711, 1756, 1561, 1757, 1759, 1799, 1809, 1811, - /* 590 */ 1717, 1718, 1721, 1720, 1723, 1647, 1726, 1819, 1783, 1670, - /* 600 */ 1734, 1727, 1566, 1786, 1791, 1644, 1671, 1675, 1834, 1815, - /* 610 */ 1651, 1745, 1749, 1748, 1751, 1753, 1760, 1794, 1758, 1764, - /* 620 */ 1795, 1761, 1816, 1652, 1765, 1744, 1766, 1814, 1817, 1768, - /* 630 */ 1769, 1820, 1771, 1773, 1829, 1776, 1777, 1832, 1788, 1790, - /* 640 */ 1833, 1792, 1732, 1770, 1775, 1778, 1866, 1785, 1800, 1846, - /* 650 */ 1804, 1867, 1810, 1846, 1846, 1882, 1843, 1845, 1874, 1875, - /* 660 */ 1876, 1877, 1878, 1879, 1880, 1881, 1883, 1884, 1847, 1828, - /* 670 */ 1885, 1886, 1888, 1890, 1904, 1895, 1905, 1906, 1857, 1636, - /* 680 */ 1907, 1637, 1908, 1909, 1911, 1912, 1917, 1913, 1949, 1916, - /* 690 */ 1910, 1915, 1955, 1921, 1914, 1918, 1959, 1925, 1919, 1924, - /* 700 */ 1964, 1930, 1922, 1932, 1969, 1937, 1939, 1975, 1956, 1958, - /* 710 */ 1960, 1971, 1968, 1957, + /* 0 */ 865, 0, 71, 0, 290, 290, 290, 290, 290, 290, + /* 10 */ 290, 290, 290, 290, 290, 361, 578, 578, 649, 578, + /* 20 */ 578, 578, 578, 578, 578, 578, 578, 578, 578, 578, + /* 30 */ 578, 578, 578, 578, 578, 578, 578, 578, 578, 578, + /* 40 */ 578, 578, 578, 578, 578, 578, 578, 578, 70, 346, + /* 50 */ 40, 140, 63, 225, 439, 225, 40, 40, 1221, 1221, + /* 60 */ 225, 1221, 1221, 110, 225, 435, 17, 17, 435, 85, + /* 70 */ 85, 68, 542, 167, 167, 17, 17, 17, 17, 17, + /* 80 */ 17, 17, 36, 17, 17, 174, 78, 17, 17, 117, + /* 90 */ 17, 78, 17, 36, 17, 36, 78, 17, 17, 78, + /* 100 */ 17, 78, 78, 78, 17, 204, 795, 34, 34, 217, + /* 110 */ 454, 710, 710, 710, 710, 710, 710, 710, 710, 710, + /* 120 */ 710, 710, 710, 710, 710, 710, 710, 710, 710, 710, + /* 130 */ 152, 444, 68, 542, 644, 667, 212, 212, 212, 697, + /* 140 */ 87, 87, 667, 345, 345, 345, 390, 303, 78, 372, + /* 150 */ 78, 372, 372, 390, 481, 218, 218, 218, 218, 218, + /* 160 */ 218, 218, 1833, 599, 304, 545, 184, 397, 286, 62, + /* 170 */ 330, 431, 341, 964, 835, 785, 977, 766, 145, 344, + /* 180 */ 766, 1127, 929, 877, 1009, 1248, 1146, 1289, 1320, 1289, + /* 190 */ 1201, 1346, 1346, 1289, 1201, 1201, 1294, 1346, 1346, 1346, + /* 200 */ 1378, 1378, 1382, 174, 1387, 174, 1391, 1393, 174, 1391, + /* 210 */ 174, 174, 174, 1346, 174, 1380, 1380, 1378, 78, 78, + /* 220 */ 78, 78, 78, 78, 78, 78, 78, 78, 78, 1346, + /* 230 */ 1378, 372, 372, 1268, 1383, 1382, 204, 1295, 1387, 204, + /* 240 */ 1346, 1320, 1320, 372, 1234, 1242, 372, 1234, 1242, 372, + /* 250 */ 372, 78, 1247, 1340, 1234, 1260, 1264, 1274, 1009, 1255, + /* 260 */ 1279, 1275, 1303, 345, 1535, 1346, 1391, 204, 204, 1542, + /* 270 */ 1242, 372, 372, 372, 372, 372, 1242, 372, 1401, 204, + /* 280 */ 390, 204, 345, 1490, 1497, 372, 481, 1346, 204, 1569, + /* 290 */ 1378, 2706, 2706, 2706, 2706, 2706, 2706, 2706, 2706, 2706, + /* 300 */ 890, 382, 724, 979, 249, 453, 626, 878, 412, 896, + /* 310 */ 978, 894, 1124, 1124, 1124, 1124, 1124, 1124, 1124, 1124, + /* 320 */ 1124, 488, 539, 130, 130, 404, 168, 607, 67, 436, + /* 330 */ 550, 44, 44, 826, 631, 257, 826, 826, 826, 994, + /* 340 */ 859, 967, 1151, 988, 1097, 1098, 1112, 1133, 1058, 1153, + /* 350 */ 1170, 1068, 1121, 1182, 1194, 863, 974, 1031, 1052, 1196, + /* 360 */ 1198, 1203, 1229, 1230, 1237, 1105, 1259, 1046, 1172, 1164, + /* 370 */ 1208, 337, 1263, 1267, 1271, 1278, 1280, 1292, 1190, 1273, + /* 380 */ 1319, 1236, 1253, 1636, 1639, 1456, 1649, 1650, 1609, 1652, + /* 390 */ 1618, 1452, 1620, 1622, 1623, 1457, 1660, 1627, 1630, 1464, + /* 400 */ 1667, 1466, 1669, 1635, 1671, 1651, 1672, 1640, 1489, 1493, + /* 410 */ 1677, 1678, 1500, 1502, 1682, 1683, 1637, 1685, 1686, 1687, + /* 420 */ 1647, 1688, 1691, 1696, 1705, 1706, 1707, 1713, 1714, 1558, + /* 430 */ 1680, 1718, 1568, 1721, 1722, 1723, 1724, 1726, 1728, 1729, + /* 440 */ 1730, 1731, 1732, 1733, 1734, 1735, 1738, 1739, 1740, 1700, + /* 450 */ 1743, 1750, 1752, 1753, 1755, 1756, 1747, 1757, 1760, 1770, + /* 460 */ 1638, 1771, 1772, 1741, 1774, 1719, 1778, 1725, 1779, 1781, + /* 470 */ 1742, 1746, 1744, 1745, 1773, 1749, 1775, 1754, 1786, 1759, + /* 480 */ 1761, 1793, 1794, 1802, 1765, 1632, 1807, 1808, 1810, 1748, + /* 490 */ 1811, 1814, 1780, 1776, 1787, 1828, 1796, 1785, 1790, 1834, + /* 500 */ 1812, 1797, 1809, 1849, 1815, 1803, 1816, 1853, 1854, 1856, + /* 510 */ 1857, 1763, 1758, 1823, 1837, 1861, 1830, 1835, 1844, 1845, + /* 520 */ 1819, 1839, 1850, 1851, 1870, 1859, 1895, 1874, 1897, 1876, + /* 530 */ 1852, 1899, 1878, 1868, 1904, 1872, 1905, 1875, 1909, 1890, + /* 540 */ 1893, 1879, 1881, 1736, 1821, 1829, 1926, 1768, 1894, 1933, + /* 550 */ 1751, 1912, 1777, 1782, 1936, 1938, 1788, 1783, 1937, 1907, + /* 560 */ 1703, 1862, 1858, 1865, 1791, 1727, 1795, 1704, 1869, 1917, + /* 570 */ 1873, 1882, 1883, 1884, 1877, 1924, 1923, 1929, 1885, 1927, + /* 580 */ 1766, 1886, 1887, 1961, 1940, 1789, 1950, 1951, 1952, 1953, + /* 590 */ 1955, 1956, 1896, 1901, 1946, 1784, 1958, 1948, 2004, 2007, + /* 600 */ 2017, 2018, 1930, 1960, 1745, 1977, 1931, 1944, 1945, 1934, + /* 610 */ 1941, 1880, 1947, 2000, 2005, 1889, 1954, 1959, 1745, 1996, + /* 620 */ 2001, 1817, 1820, 1824, 2048, 2039, 1855, 1970, 1971, 1973, + /* 630 */ 1974, 1976, 1978, 2025, 1980, 1981, 2026, 1983, 2058, 1871, + /* 640 */ 1988, 1972, 1989, 2051, 2052, 1993, 1994, 2054, 1997, 1995, + /* 650 */ 2059, 1998, 2002, 2060, 2006, 2003, 2065, 2008, 1985, 1987, + /* 660 */ 1990, 2010, 2080, 2009, 2012, 2064, 2019, 2082, 2029, 2064, + /* 670 */ 2064, 2091, 2062, 2067, 2096, 2097, 2098, 2099, 2101, 2102, + /* 680 */ 2103, 2106, 2109, 2110, 2084, 2063, 2108, 2121, 2122, 2123, + /* 690 */ 2139, 2127, 2128, 2129, 2095, 1819, 2131, 1839, 2133, 2135, + /* 700 */ 2136, 2140, 2160, 2149, 2185, 2151, 2141, 2148, 2188, 2155, + /* 710 */ 2145, 2156, 2194, 2161, 2150, 2158, 2200, 2167, 2159, 2164, + /* 720 */ 2214, 2181, 2183, 2219, 2198, 2201, 2199, 2202, 2204, 2206, }; -#define YY_REDUCE_COUNT (292) -#define YY_REDUCE_MIN (-420) -#define YY_REDUCE_MAX (2656) +#define YY_REDUCE_COUNT (299) +#define YY_REDUCE_MIN (-395) +#define YY_REDUCE_MAX (2289) static const short yy_reduce_ofst[] = { - /* 0 */ 276, -259, 233, 672, 744, 798, 859, 922, 942, 1045, - /* 10 */ 1104, 1158, 1219, 1277, 1331, 1360, 294, 1418, 1472, 1526, - /* 20 */ 1578, 1594, 1683, 1701, 1755, 1772, 1824, 1840, 1892, 1946, - /* 30 */ 1967, 2043, 2061, 2115, 2132, 2184, 2200, 2252, 2275, 2327, - /* 40 */ 2390, 2442, 2461, 2513, 2529, 2581, 2604, 2656, -258, 95, - /* 50 */ 105, -39, 559, 597, 599, 669, 187, 704, -346, -57, - /* 60 */ -391, 323, 480, -276, -250, 75, -222, 297, -371, -330, - /* 70 */ -326, -327, -363, -299, -257, 116, 128, 193, 199, 327, - /* 80 */ 409, 471, 645, 649, 657, 252, 693, 706, -51, 725, - /* 90 */ 747, 762, 244, 763, 772, -203, 791, 465, -5, 362, - /* 100 */ 847, -210, -315, -420, -420, -217, -212, -69, -21, 41, - /* 110 */ 43, 70, 96, 117, 148, 197, 225, 234, 346, 351, - /* 120 */ 405, 446, 454, 504, 651, 655, -266, -170, 397, 191, - /* 130 */ 152, 352, -170, -150, 219, 477, 186, 254, 491, 524, - /* 140 */ 537, 538, -316, -319, 485, 367, 542, 555, 557, 632, - /* 150 */ 530, -342, 369, 389, 410, 463, 468, 483, 319, 577, - /* 160 */ 650, 543, 521, 603, 756, 695, 786, 786, 810, 780, - /* 170 */ 831, 811, 808, 760, 760, 746, 760, 784, 783, 786, - /* 180 */ 834, 842, 851, 866, 870, 869, 878, 924, 926, 879, - /* 190 */ 882, 886, 933, 943, 944, 948, 957, 958, 897, 953, - /* 200 */ 925, 955, 965, 914, 963, 970, 964, 966, 968, 975, - /* 210 */ 971, 976, 978, 989, 959, 962, 967, 979, 983, 984, - /* 220 */ 994, 996, 998, 999, 1001, 987, 992, 990, 991, 938, - /* 230 */ 941, 997, 973, 995, 1028, 1035, 993, 1000, 1002, 936, - /* 240 */ 1003, 1007, 939, 1004, 1009, 1013, 786, 946, 949, 950, - /* 250 */ 947, 956, 972, 1014, 951, 961, 960, 760, 1049, 1016, - /* 260 */ 1085, 1088, 1082, 1043, 1054, 1058, 1059, 1061, 1062, 1044, - /* 270 */ 1063, 1066, 1118, 1089, 1119, 1079, 1038, 1102, 1095, 1121, - /* 280 */ 1136, 1126, 1143, 1145, 1087, 1081, 1097, 1098, 1135, 1138, - /* 290 */ 1139, 1144, 1154, + /* 0 */ -253, -301, -11, 277, 687, 745, 792, 849, 907, 937, + /* 10 */ 969, 1048, 1106, 1126, 1184, 1214, 75, 1231, 1288, 363, + /* 20 */ 775, 1335, 1352, 1407, 1430, 1462, 1509, 1533, 1599, 1621, + /* 30 */ 1646, 1709, 1737, 1767, 1799, 1831, 1863, 1888, 1920, 1982, + /* 40 */ 1999, 2056, 2078, 2144, 2168, 2232, 2257, 2289, -282, -336, + /* 50 */ 285, 396, 790, 829, 960, 1033, 651, 691, -354, -341, + /* 60 */ -378, -337, -226, -26, 208, -364, -266, 21, -375, -332, + /* 70 */ -326, -343, -369, -293, -182, 112, 198, 309, 357, 470, + /* 80 */ 479, 486, -32, 548, 549, -106, -280, 663, 678, -201, + /* 90 */ 693, -260, 699, 60, 727, 226, 484, 751, 794, 305, + /* 100 */ 666, 527, 336, 561, 812, 119, -283, -395, -395, -154, + /* 110 */ 187, -295, -22, -18, 166, 221, 384, 405, 410, 456, + /* 120 */ 555, 653, 684, 704, 706, 719, 736, 756, 801, 817, + /* 130 */ 292, -28, -82, 222, 179, 255, -28, 376, 418, 483, + /* 140 */ 258, 279, 424, 291, 440, 441, -162, 501, 522, 559, + /* 150 */ 473, 679, 681, 700, 414, 195, 260, 288, 353, 367, + /* 160 */ 399, 407, 443, -312, 566, 544, 459, 499, 752, 597, + /* 170 */ 757, 757, 777, 827, 798, 840, 854, 802, 802, 813, + /* 180 */ 802, 819, 806, 757, 889, 898, 920, 961, 973, 981, + /* 190 */ 990, 1032, 1036, 1004, 992, 1008, 1044, 1054, 1056, 1057, + /* 200 */ 1069, 1070, 1005, 1062, 1034, 1067, 1074, 1024, 1075, 1084, + /* 210 */ 1081, 1083, 1085, 1090, 1088, 1093, 1094, 1107, 1076, 1077, + /* 220 */ 1079, 1080, 1089, 1091, 1092, 1096, 1103, 1110, 1113, 1104, + /* 230 */ 1122, 1109, 1111, 1066, 1095, 1082, 1136, 1099, 1114, 1144, + /* 240 */ 1154, 1115, 1117, 1129, 1059, 1118, 1130, 1060, 1119, 1137, + /* 250 */ 1138, 757, 1073, 1078, 1086, 1101, 1087, 1108, 1142, 1100, + /* 260 */ 1120, 1116, 802, 1183, 1148, 1224, 1220, 1222, 1225, 1179, + /* 270 */ 1180, 1197, 1199, 1202, 1204, 1206, 1181, 1207, 1186, 1235, + /* 280 */ 1227, 1240, 1216, 1162, 1238, 1228, 1258, 1266, 1269, 1277, + /* 290 */ 1282, 1219, 1223, 1226, 1232, 1265, 1281, 1284, 1276, 1307, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 10 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 20 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 30 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 40 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 50 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 60 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 70 */ 1606, 1861, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 80 */ 1606, 1606, 1606, 1606, 1606, 1684, 1606, 1606, 1606, 1606, - /* 90 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 100 */ 1606, 1682, 1854, 2057, 1606, 1606, 1606, 1606, 1606, 1606, - /* 110 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 120 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 2069, 1606, 1606, - /* 130 */ 1684, 1606, 2069, 2069, 2069, 1682, 2029, 2029, 1606, 1606, - /* 140 */ 1606, 1606, 1791, 1606, 1606, 1606, 1606, 1606, 1606, 1791, - /* 150 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1903, 1606, - /* 160 */ 1606, 2094, 2148, 1606, 1606, 2097, 1606, 1606, 1606, 1866, - /* 170 */ 1606, 1744, 2084, 2061, 2075, 2132, 2062, 2059, 2078, 1606, - /* 180 */ 2088, 1606, 1896, 1859, 1606, 1859, 1856, 1606, 1606, 1859, - /* 190 */ 1856, 1856, 1735, 1606, 1606, 1606, 1606, 1606, 1606, 1684, - /* 200 */ 1606, 1684, 1606, 1606, 1684, 1606, 1684, 1684, 1684, 1606, - /* 210 */ 1684, 1663, 1663, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 220 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1916, - /* 230 */ 1606, 1682, 1905, 1606, 1682, 1606, 1606, 1606, 1606, 2105, - /* 240 */ 2103, 1606, 2105, 2103, 1606, 1606, 1606, 2117, 2113, 2105, - /* 250 */ 2121, 2119, 2090, 2088, 2151, 2138, 2134, 2075, 1606, 1606, - /* 260 */ 1606, 1606, 1682, 2103, 1606, 1606, 1606, 1606, 1606, 2103, - /* 270 */ 1606, 1606, 1682, 1606, 1682, 1606, 1606, 1760, 1606, 1606, - /* 280 */ 1606, 1682, 1638, 1606, 1898, 1909, 1881, 1881, 1794, 1794, - /* 290 */ 1794, 1685, 1611, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 300 */ 1606, 1606, 1606, 1606, 1606, 2116, 2115, 1984, 1606, 2033, - /* 310 */ 2032, 2031, 2022, 1983, 1756, 1606, 1982, 1981, 1606, 1606, - /* 320 */ 1606, 1606, 1606, 1872, 1871, 1975, 1606, 1606, 1976, 1974, - /* 330 */ 1973, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 340 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 2135, 2139, - /* 350 */ 1606, 1606, 1606, 1606, 1606, 1606, 2058, 1606, 1606, 1606, - /* 360 */ 1606, 1606, 1958, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 370 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 380 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 390 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 400 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 410 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 420 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 430 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 440 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 450 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 460 */ 1606, 1643, 1963, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 470 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 480 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 490 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 500 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1723, - /* 510 */ 1722, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 520 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 530 */ 1606, 1606, 1606, 1966, 1606, 1606, 1606, 1606, 1606, 1606, - /* 540 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 2131, 2091, 1606, - /* 550 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 560 */ 1606, 1606, 1606, 1606, 1606, 1606, 1958, 1606, 2114, 1606, - /* 570 */ 1606, 2129, 1606, 2133, 1606, 1606, 1606, 1606, 1606, 1606, - /* 580 */ 1606, 2068, 2064, 1606, 1606, 2060, 1606, 1606, 1606, 1606, - /* 590 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 600 */ 1606, 1606, 1957, 1606, 2019, 1606, 1606, 1606, 2053, 1606, - /* 610 */ 1606, 2004, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 620 */ 1606, 1966, 1606, 1969, 1606, 1606, 1606, 1606, 1606, 1788, - /* 630 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 640 */ 1606, 1606, 1773, 1771, 1770, 1769, 1606, 1766, 1606, 1801, - /* 650 */ 1606, 1606, 1606, 1797, 1796, 1606, 1606, 1606, 1606, 1606, - /* 660 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 670 */ 1703, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1695, - /* 680 */ 1606, 1694, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 690 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 700 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 710 */ 1606, 1606, 1606, 1606, + /* 0 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 10 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 20 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 30 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 40 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 50 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 60 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 70 */ 1634, 1891, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 80 */ 1634, 1634, 1634, 1634, 1634, 1712, 1634, 1634, 1634, 1634, + /* 90 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 100 */ 1634, 1634, 1634, 1634, 1634, 1710, 1884, 2093, 1634, 1634, + /* 110 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 120 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 130 */ 1634, 2105, 1634, 1634, 1712, 1634, 2105, 2105, 2105, 1710, + /* 140 */ 2065, 2065, 1634, 1634, 1634, 1634, 1819, 1634, 1634, 1634, + /* 150 */ 1634, 1634, 1634, 1819, 1634, 1634, 1634, 1634, 1634, 1634, + /* 160 */ 1634, 1634, 1936, 1634, 1634, 2130, 2184, 1634, 1634, 2133, + /* 170 */ 1634, 1634, 1634, 1896, 1634, 1772, 2120, 2097, 2111, 2168, + /* 180 */ 2098, 2095, 2114, 1634, 2124, 1634, 1929, 1889, 1634, 1889, + /* 190 */ 1886, 1634, 1634, 1889, 1886, 1886, 1763, 1634, 1634, 1634, + /* 200 */ 1634, 1634, 1634, 1712, 1634, 1712, 1634, 1634, 1712, 1634, + /* 210 */ 1712, 1712, 1712, 1634, 1712, 1691, 1691, 1634, 1634, 1634, + /* 220 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 230 */ 1634, 1634, 1634, 1951, 1942, 1634, 1710, 1938, 1634, 1710, + /* 240 */ 1634, 1634, 1634, 1634, 2141, 2139, 1634, 2141, 2139, 1634, + /* 250 */ 1634, 1634, 2153, 2149, 2141, 2157, 2155, 2126, 2124, 2187, + /* 260 */ 2174, 2170, 2111, 1634, 1634, 1634, 1634, 1710, 1710, 1634, + /* 270 */ 2139, 1634, 1634, 1634, 1634, 1634, 2139, 1634, 1634, 1710, + /* 280 */ 1634, 1710, 1634, 1634, 1788, 1634, 1634, 1634, 1710, 1666, + /* 290 */ 1634, 1931, 1944, 1914, 1914, 1822, 1822, 1822, 1713, 1639, + /* 300 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 310 */ 1634, 1634, 2152, 2151, 2020, 1634, 2069, 2068, 2067, 2058, + /* 320 */ 2019, 1784, 1634, 2018, 2017, 1634, 1634, 1634, 1634, 1634, + /* 330 */ 1634, 1905, 1904, 2011, 1634, 1634, 2012, 2010, 2009, 1634, + /* 340 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 350 */ 1634, 1634, 1634, 1634, 1634, 1634, 2171, 2175, 1634, 1634, + /* 360 */ 1634, 1634, 1634, 1634, 1634, 2094, 1634, 1634, 1634, 1634, + /* 370 */ 1634, 1993, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 380 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 390 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 400 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 410 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 420 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 430 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 440 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 450 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 460 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 470 */ 1634, 1634, 1671, 1998, 1634, 1634, 1634, 1634, 1634, 1634, + /* 480 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 490 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 500 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 510 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 520 */ 1751, 1750, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 530 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 540 */ 1634, 1634, 1634, 1634, 2002, 1634, 1634, 1634, 1634, 1634, + /* 550 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 2167, 2127, + /* 560 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 570 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1993, 1634, 2150, + /* 580 */ 1634, 1634, 2165, 1634, 2169, 1634, 1634, 1634, 1634, 1634, + /* 590 */ 1634, 1634, 2104, 2100, 1634, 1634, 2096, 1634, 1634, 1634, + /* 600 */ 1634, 1634, 1634, 1634, 2001, 1634, 1634, 1634, 1634, 1634, + /* 610 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1992, 1634, + /* 620 */ 2055, 1634, 1634, 1634, 2089, 1634, 1634, 2040, 1634, 1634, + /* 630 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 2002, 1634, 2005, + /* 640 */ 1634, 1634, 1634, 1634, 1634, 1816, 1634, 1634, 1634, 1634, + /* 650 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1801, 1799, + /* 660 */ 1798, 1797, 1634, 1794, 1634, 1829, 1634, 1634, 1634, 1825, + /* 670 */ 1824, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 680 */ 1634, 1634, 1634, 1634, 1634, 1634, 1731, 1634, 1634, 1634, + /* 690 */ 1634, 1634, 1634, 1634, 1634, 1723, 1634, 1722, 1634, 1634, + /* 700 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 710 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, + /* 720 */ 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, 1634, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1233,6 +1129,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* CONSUMERS => nothing */ 0, /* SUBSCRIPTIONS => nothing */ 0, /* VNODES => nothing */ + 0, /* ALIVE => nothing */ 0, /* LIKE => nothing */ 0, /* TBNAME => nothing */ 0, /* QTAGS => nothing */ @@ -1301,7 +1198,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* SERVER_STATUS => nothing */ 0, /* CURRENT_USER => nothing */ 0, /* CASE => nothing */ - 271, /* END => ABORT */ + 272, /* END => ABORT */ 0, /* WHEN => nothing */ 0, /* THEN => nothing */ 0, /* ELSE => nothing */ @@ -1345,59 +1242,59 @@ static const YYCODETYPE yyFallback[] = { 0, /* ASC => nothing */ 0, /* NULLS => nothing */ 0, /* ABORT => nothing */ - 271, /* AFTER => ABORT */ - 271, /* ATTACH => ABORT */ - 271, /* BEFORE => ABORT */ - 271, /* BEGIN => ABORT */ - 271, /* BITAND => ABORT */ - 271, /* BITNOT => ABORT */ - 271, /* BITOR => ABORT */ - 271, /* BLOCKS => ABORT */ - 271, /* CHANGE => ABORT */ - 271, /* COMMA => ABORT */ - 271, /* COMPACT => ABORT */ - 271, /* CONCAT => ABORT */ - 271, /* CONFLICT => ABORT */ - 271, /* COPY => ABORT */ - 271, /* DEFERRED => ABORT */ - 271, /* DELIMITERS => ABORT */ - 271, /* DETACH => ABORT */ - 271, /* DIVIDE => ABORT */ - 271, /* DOT => ABORT */ - 271, /* EACH => ABORT */ - 271, /* FAIL => ABORT */ - 271, /* FILE => ABORT */ - 271, /* FOR => ABORT */ - 271, /* GLOB => ABORT */ - 271, /* ID => ABORT */ - 271, /* IMMEDIATE => ABORT */ - 271, /* IMPORT => ABORT */ - 271, /* INITIALLY => ABORT */ - 271, /* INSTEAD => ABORT */ - 271, /* ISNULL => ABORT */ - 271, /* KEY => ABORT */ - 271, /* MODULES => ABORT */ - 271, /* NK_BITNOT => ABORT */ - 271, /* NK_SEMI => ABORT */ - 271, /* NOTNULL => ABORT */ - 271, /* OF => ABORT */ - 271, /* PLUS => ABORT */ - 271, /* PRIVILEGE => ABORT */ - 271, /* RAISE => ABORT */ - 271, /* REPLACE => ABORT */ - 271, /* RESTRICT => ABORT */ - 271, /* ROW => ABORT */ - 271, /* SEMI => ABORT */ - 271, /* STAR => ABORT */ - 271, /* STATEMENT => ABORT */ - 271, /* STRICT => ABORT */ - 271, /* STRING => ABORT */ - 271, /* TIMES => ABORT */ - 271, /* UPDATE => ABORT */ - 271, /* VALUES => ABORT */ - 271, /* VARIABLE => ABORT */ - 271, /* VIEW => ABORT */ - 271, /* WAL => ABORT */ + 272, /* AFTER => ABORT */ + 272, /* ATTACH => ABORT */ + 272, /* BEFORE => ABORT */ + 272, /* BEGIN => ABORT */ + 272, /* BITAND => ABORT */ + 272, /* BITNOT => ABORT */ + 272, /* BITOR => ABORT */ + 272, /* BLOCKS => ABORT */ + 272, /* CHANGE => ABORT */ + 272, /* COMMA => ABORT */ + 272, /* COMPACT => ABORT */ + 272, /* CONCAT => ABORT */ + 272, /* CONFLICT => ABORT */ + 272, /* COPY => ABORT */ + 272, /* DEFERRED => ABORT */ + 272, /* DELIMITERS => ABORT */ + 272, /* DETACH => ABORT */ + 272, /* DIVIDE => ABORT */ + 272, /* DOT => ABORT */ + 272, /* EACH => ABORT */ + 272, /* FAIL => ABORT */ + 272, /* FILE => ABORT */ + 272, /* FOR => ABORT */ + 272, /* GLOB => ABORT */ + 272, /* ID => ABORT */ + 272, /* IMMEDIATE => ABORT */ + 272, /* IMPORT => ABORT */ + 272, /* INITIALLY => ABORT */ + 272, /* INSTEAD => ABORT */ + 272, /* ISNULL => ABORT */ + 272, /* KEY => ABORT */ + 272, /* MODULES => ABORT */ + 272, /* NK_BITNOT => ABORT */ + 272, /* NK_SEMI => ABORT */ + 272, /* NOTNULL => ABORT */ + 272, /* OF => ABORT */ + 272, /* PLUS => ABORT */ + 272, /* PRIVILEGE => ABORT */ + 272, /* RAISE => ABORT */ + 272, /* REPLACE => ABORT */ + 272, /* RESTRICT => ABORT */ + 272, /* ROW => ABORT */ + 272, /* SEMI => ABORT */ + 272, /* STAR => ABORT */ + 272, /* STATEMENT => ABORT */ + 272, /* STRICT => ABORT */ + 272, /* STRING => ABORT */ + 272, /* TIMES => ABORT */ + 272, /* UPDATE => ABORT */ + 272, /* VALUES => ABORT */ + 272, /* VARIABLE => ABORT */ + 272, /* VIEW => ABORT */ + 272, /* WAL => ABORT */ }; #endif /* YYFALLBACK */ @@ -1645,307 +1542,311 @@ static const char *const yyTokenName[] = { /* 157 */ "CONSUMERS", /* 158 */ "SUBSCRIPTIONS", /* 159 */ "VNODES", - /* 160 */ "LIKE", - /* 161 */ "TBNAME", - /* 162 */ "QTAGS", - /* 163 */ "AS", - /* 164 */ "INDEX", - /* 165 */ "FUNCTION", - /* 166 */ "INTERVAL", - /* 167 */ "COUNT", - /* 168 */ "LAST_ROW", - /* 169 */ "TOPIC", - /* 170 */ "WITH", - /* 171 */ "META", - /* 172 */ "CONSUMER", - /* 173 */ "GROUP", - /* 174 */ "DESC", - /* 175 */ "DESCRIBE", - /* 176 */ "RESET", - /* 177 */ "QUERY", - /* 178 */ "CACHE", - /* 179 */ "EXPLAIN", - /* 180 */ "ANALYZE", - /* 181 */ "VERBOSE", - /* 182 */ "NK_BOOL", - /* 183 */ "RATIO", - /* 184 */ "NK_FLOAT", - /* 185 */ "OUTPUTTYPE", - /* 186 */ "AGGREGATE", - /* 187 */ "BUFSIZE", - /* 188 */ "STREAM", - /* 189 */ "INTO", - /* 190 */ "TRIGGER", - /* 191 */ "AT_ONCE", - /* 192 */ "WINDOW_CLOSE", - /* 193 */ "IGNORE", - /* 194 */ "EXPIRED", - /* 195 */ "FILL_HISTORY", - /* 196 */ "SUBTABLE", - /* 197 */ "KILL", - /* 198 */ "CONNECTION", - /* 199 */ "TRANSACTION", - /* 200 */ "BALANCE", - /* 201 */ "VGROUP", - /* 202 */ "MERGE", - /* 203 */ "REDISTRIBUTE", - /* 204 */ "SPLIT", - /* 205 */ "DELETE", - /* 206 */ "INSERT", - /* 207 */ "NULL", - /* 208 */ "NK_QUESTION", - /* 209 */ "NK_ARROW", - /* 210 */ "ROWTS", - /* 211 */ "QSTART", - /* 212 */ "QEND", - /* 213 */ "QDURATION", - /* 214 */ "WSTART", - /* 215 */ "WEND", - /* 216 */ "WDURATION", - /* 217 */ "IROWTS", - /* 218 */ "ISFILLED", - /* 219 */ "CAST", - /* 220 */ "NOW", - /* 221 */ "TODAY", - /* 222 */ "TIMEZONE", - /* 223 */ "CLIENT_VERSION", - /* 224 */ "SERVER_VERSION", - /* 225 */ "SERVER_STATUS", - /* 226 */ "CURRENT_USER", - /* 227 */ "CASE", - /* 228 */ "END", - /* 229 */ "WHEN", - /* 230 */ "THEN", - /* 231 */ "ELSE", - /* 232 */ "BETWEEN", - /* 233 */ "IS", - /* 234 */ "NK_LT", - /* 235 */ "NK_GT", - /* 236 */ "NK_LE", - /* 237 */ "NK_GE", - /* 238 */ "NK_NE", - /* 239 */ "MATCH", - /* 240 */ "NMATCH", - /* 241 */ "CONTAINS", - /* 242 */ "IN", - /* 243 */ "JOIN", - /* 244 */ "INNER", - /* 245 */ "SELECT", - /* 246 */ "DISTINCT", - /* 247 */ "WHERE", - /* 248 */ "PARTITION", - /* 249 */ "BY", - /* 250 */ "SESSION", - /* 251 */ "STATE_WINDOW", - /* 252 */ "EVENT_WINDOW", - /* 253 */ "START", - /* 254 */ "SLIDING", - /* 255 */ "FILL", - /* 256 */ "VALUE", - /* 257 */ "NONE", - /* 258 */ "PREV", - /* 259 */ "LINEAR", - /* 260 */ "NEXT", - /* 261 */ "HAVING", - /* 262 */ "RANGE", - /* 263 */ "EVERY", - /* 264 */ "ORDER", - /* 265 */ "SLIMIT", - /* 266 */ "SOFFSET", - /* 267 */ "LIMIT", - /* 268 */ "OFFSET", - /* 269 */ "ASC", - /* 270 */ "NULLS", - /* 271 */ "ABORT", - /* 272 */ "AFTER", - /* 273 */ "ATTACH", - /* 274 */ "BEFORE", - /* 275 */ "BEGIN", - /* 276 */ "BITAND", - /* 277 */ "BITNOT", - /* 278 */ "BITOR", - /* 279 */ "BLOCKS", - /* 280 */ "CHANGE", - /* 281 */ "COMMA", - /* 282 */ "COMPACT", - /* 283 */ "CONCAT", - /* 284 */ "CONFLICT", - /* 285 */ "COPY", - /* 286 */ "DEFERRED", - /* 287 */ "DELIMITERS", - /* 288 */ "DETACH", - /* 289 */ "DIVIDE", - /* 290 */ "DOT", - /* 291 */ "EACH", - /* 292 */ "FAIL", - /* 293 */ "FILE", - /* 294 */ "FOR", - /* 295 */ "GLOB", - /* 296 */ "ID", - /* 297 */ "IMMEDIATE", - /* 298 */ "IMPORT", - /* 299 */ "INITIALLY", - /* 300 */ "INSTEAD", - /* 301 */ "ISNULL", - /* 302 */ "KEY", - /* 303 */ "MODULES", - /* 304 */ "NK_BITNOT", - /* 305 */ "NK_SEMI", - /* 306 */ "NOTNULL", - /* 307 */ "OF", - /* 308 */ "PLUS", - /* 309 */ "PRIVILEGE", - /* 310 */ "RAISE", - /* 311 */ "REPLACE", - /* 312 */ "RESTRICT", - /* 313 */ "ROW", - /* 314 */ "SEMI", - /* 315 */ "STAR", - /* 316 */ "STATEMENT", - /* 317 */ "STRICT", - /* 318 */ "STRING", - /* 319 */ "TIMES", - /* 320 */ "UPDATE", - /* 321 */ "VALUES", - /* 322 */ "VARIABLE", - /* 323 */ "VIEW", - /* 324 */ "WAL", - /* 325 */ "cmd", - /* 326 */ "account_options", - /* 327 */ "alter_account_options", - /* 328 */ "literal", - /* 329 */ "alter_account_option", - /* 330 */ "user_name", - /* 331 */ "sysinfo_opt", - /* 332 */ "privileges", - /* 333 */ "priv_level", - /* 334 */ "priv_type_list", - /* 335 */ "priv_type", - /* 336 */ "db_name", - /* 337 */ "topic_name", - /* 338 */ "dnode_endpoint", - /* 339 */ "force_opt", - /* 340 */ "not_exists_opt", - /* 341 */ "db_options", - /* 342 */ "exists_opt", - /* 343 */ "alter_db_options", - /* 344 */ "speed_opt", - /* 345 */ "integer_list", - /* 346 */ "variable_list", - /* 347 */ "retention_list", - /* 348 */ "alter_db_option", - /* 349 */ "retention", - /* 350 */ "full_table_name", - /* 351 */ "column_def_list", - /* 352 */ "tags_def_opt", - /* 353 */ "table_options", - /* 354 */ "multi_create_clause", - /* 355 */ "tags_def", - /* 356 */ "multi_drop_clause", - /* 357 */ "alter_table_clause", - /* 358 */ "alter_table_options", - /* 359 */ "column_name", - /* 360 */ "type_name", - /* 361 */ "signed_literal", - /* 362 */ "create_subtable_clause", - /* 363 */ "specific_cols_opt", - /* 364 */ "expression_list", - /* 365 */ "drop_table_clause", - /* 366 */ "col_name_list", - /* 367 */ "table_name", - /* 368 */ "column_def", - /* 369 */ "duration_list", - /* 370 */ "rollup_func_list", - /* 371 */ "alter_table_option", - /* 372 */ "duration_literal", - /* 373 */ "rollup_func_name", - /* 374 */ "function_name", - /* 375 */ "col_name", - /* 376 */ "db_name_cond_opt", - /* 377 */ "like_pattern_opt", - /* 378 */ "table_name_cond", - /* 379 */ "from_db_opt", - /* 380 */ "tag_list_opt", - /* 381 */ "tag_item", - /* 382 */ "column_alias", - /* 383 */ "index_options", - /* 384 */ "func_list", - /* 385 */ "sliding_opt", - /* 386 */ "sma_stream_opt", - /* 387 */ "func", - /* 388 */ "sma_func_name", - /* 389 */ "query_or_subquery", - /* 390 */ "cgroup_name", - /* 391 */ "analyze_opt", - /* 392 */ "explain_options", - /* 393 */ "agg_func_opt", - /* 394 */ "bufsize_opt", - /* 395 */ "stream_name", - /* 396 */ "stream_options", - /* 397 */ "subtable_opt", - /* 398 */ "expression", - /* 399 */ "dnode_list", - /* 400 */ "where_clause_opt", - /* 401 */ "signed", - /* 402 */ "literal_func", - /* 403 */ "literal_list", - /* 404 */ "table_alias", - /* 405 */ "expr_or_subquery", - /* 406 */ "pseudo_column", - /* 407 */ "column_reference", - /* 408 */ "function_expression", - /* 409 */ "case_when_expression", - /* 410 */ "star_func", - /* 411 */ "star_func_para_list", - /* 412 */ "noarg_func", - /* 413 */ "other_para_list", - /* 414 */ "star_func_para", - /* 415 */ "when_then_list", - /* 416 */ "case_when_else_opt", - /* 417 */ "common_expression", - /* 418 */ "when_then_expr", - /* 419 */ "predicate", - /* 420 */ "compare_op", - /* 421 */ "in_op", - /* 422 */ "in_predicate_value", - /* 423 */ "boolean_value_expression", - /* 424 */ "boolean_primary", - /* 425 */ "from_clause_opt", - /* 426 */ "table_reference_list", - /* 427 */ "table_reference", - /* 428 */ "table_primary", - /* 429 */ "joined_table", - /* 430 */ "alias_opt", - /* 431 */ "subquery", - /* 432 */ "parenthesized_joined_table", - /* 433 */ "join_type", - /* 434 */ "search_condition", - /* 435 */ "query_specification", - /* 436 */ "set_quantifier_opt", - /* 437 */ "select_list", - /* 438 */ "partition_by_clause_opt", - /* 439 */ "range_opt", - /* 440 */ "every_opt", - /* 441 */ "fill_opt", - /* 442 */ "twindow_clause_opt", - /* 443 */ "group_by_clause_opt", - /* 444 */ "having_clause_opt", - /* 445 */ "select_item", - /* 446 */ "partition_list", - /* 447 */ "partition_item", - /* 448 */ "fill_mode", - /* 449 */ "group_by_list", - /* 450 */ "query_expression", - /* 451 */ "query_simple", - /* 452 */ "order_by_clause_opt", - /* 453 */ "slimit_clause_opt", - /* 454 */ "limit_clause_opt", - /* 455 */ "union_query_expression", - /* 456 */ "query_simple_or_subquery", - /* 457 */ "sort_specification_list", - /* 458 */ "sort_specification", - /* 459 */ "ordering_specification_opt", - /* 460 */ "null_ordering_opt", + /* 160 */ "ALIVE", + /* 161 */ "LIKE", + /* 162 */ "TBNAME", + /* 163 */ "QTAGS", + /* 164 */ "AS", + /* 165 */ "INDEX", + /* 166 */ "FUNCTION", + /* 167 */ "INTERVAL", + /* 168 */ "COUNT", + /* 169 */ "LAST_ROW", + /* 170 */ "TOPIC", + /* 171 */ "WITH", + /* 172 */ "META", + /* 173 */ "CONSUMER", + /* 174 */ "GROUP", + /* 175 */ "DESC", + /* 176 */ "DESCRIBE", + /* 177 */ "RESET", + /* 178 */ "QUERY", + /* 179 */ "CACHE", + /* 180 */ "EXPLAIN", + /* 181 */ "ANALYZE", + /* 182 */ "VERBOSE", + /* 183 */ "NK_BOOL", + /* 184 */ "RATIO", + /* 185 */ "NK_FLOAT", + /* 186 */ "OUTPUTTYPE", + /* 187 */ "AGGREGATE", + /* 188 */ "BUFSIZE", + /* 189 */ "STREAM", + /* 190 */ "INTO", + /* 191 */ "TRIGGER", + /* 192 */ "AT_ONCE", + /* 193 */ "WINDOW_CLOSE", + /* 194 */ "IGNORE", + /* 195 */ "EXPIRED", + /* 196 */ "FILL_HISTORY", + /* 197 */ "SUBTABLE", + /* 198 */ "KILL", + /* 199 */ "CONNECTION", + /* 200 */ "TRANSACTION", + /* 201 */ "BALANCE", + /* 202 */ "VGROUP", + /* 203 */ "MERGE", + /* 204 */ "REDISTRIBUTE", + /* 205 */ "SPLIT", + /* 206 */ "DELETE", + /* 207 */ "INSERT", + /* 208 */ "NULL", + /* 209 */ "NK_QUESTION", + /* 210 */ "NK_ARROW", + /* 211 */ "ROWTS", + /* 212 */ "QSTART", + /* 213 */ "QEND", + /* 214 */ "QDURATION", + /* 215 */ "WSTART", + /* 216 */ "WEND", + /* 217 */ "WDURATION", + /* 218 */ "IROWTS", + /* 219 */ "ISFILLED", + /* 220 */ "CAST", + /* 221 */ "NOW", + /* 222 */ "TODAY", + /* 223 */ "TIMEZONE", + /* 224 */ "CLIENT_VERSION", + /* 225 */ "SERVER_VERSION", + /* 226 */ "SERVER_STATUS", + /* 227 */ "CURRENT_USER", + /* 228 */ "CASE", + /* 229 */ "END", + /* 230 */ "WHEN", + /* 231 */ "THEN", + /* 232 */ "ELSE", + /* 233 */ "BETWEEN", + /* 234 */ "IS", + /* 235 */ "NK_LT", + /* 236 */ "NK_GT", + /* 237 */ "NK_LE", + /* 238 */ "NK_GE", + /* 239 */ "NK_NE", + /* 240 */ "MATCH", + /* 241 */ "NMATCH", + /* 242 */ "CONTAINS", + /* 243 */ "IN", + /* 244 */ "JOIN", + /* 245 */ "INNER", + /* 246 */ "SELECT", + /* 247 */ "DISTINCT", + /* 248 */ "WHERE", + /* 249 */ "PARTITION", + /* 250 */ "BY", + /* 251 */ "SESSION", + /* 252 */ "STATE_WINDOW", + /* 253 */ "EVENT_WINDOW", + /* 254 */ "START", + /* 255 */ "SLIDING", + /* 256 */ "FILL", + /* 257 */ "VALUE", + /* 258 */ "NONE", + /* 259 */ "PREV", + /* 260 */ "LINEAR", + /* 261 */ "NEXT", + /* 262 */ "HAVING", + /* 263 */ "RANGE", + /* 264 */ "EVERY", + /* 265 */ "ORDER", + /* 266 */ "SLIMIT", + /* 267 */ "SOFFSET", + /* 268 */ "LIMIT", + /* 269 */ "OFFSET", + /* 270 */ "ASC", + /* 271 */ "NULLS", + /* 272 */ "ABORT", + /* 273 */ "AFTER", + /* 274 */ "ATTACH", + /* 275 */ "BEFORE", + /* 276 */ "BEGIN", + /* 277 */ "BITAND", + /* 278 */ "BITNOT", + /* 279 */ "BITOR", + /* 280 */ "BLOCKS", + /* 281 */ "CHANGE", + /* 282 */ "COMMA", + /* 283 */ "COMPACT", + /* 284 */ "CONCAT", + /* 285 */ "CONFLICT", + /* 286 */ "COPY", + /* 287 */ "DEFERRED", + /* 288 */ "DELIMITERS", + /* 289 */ "DETACH", + /* 290 */ "DIVIDE", + /* 291 */ "DOT", + /* 292 */ "EACH", + /* 293 */ "FAIL", + /* 294 */ "FILE", + /* 295 */ "FOR", + /* 296 */ "GLOB", + /* 297 */ "ID", + /* 298 */ "IMMEDIATE", + /* 299 */ "IMPORT", + /* 300 */ "INITIALLY", + /* 301 */ "INSTEAD", + /* 302 */ "ISNULL", + /* 303 */ "KEY", + /* 304 */ "MODULES", + /* 305 */ "NK_BITNOT", + /* 306 */ "NK_SEMI", + /* 307 */ "NOTNULL", + /* 308 */ "OF", + /* 309 */ "PLUS", + /* 310 */ "PRIVILEGE", + /* 311 */ "RAISE", + /* 312 */ "REPLACE", + /* 313 */ "RESTRICT", + /* 314 */ "ROW", + /* 315 */ "SEMI", + /* 316 */ "STAR", + /* 317 */ "STATEMENT", + /* 318 */ "STRICT", + /* 319 */ "STRING", + /* 320 */ "TIMES", + /* 321 */ "UPDATE", + /* 322 */ "VALUES", + /* 323 */ "VARIABLE", + /* 324 */ "VIEW", + /* 325 */ "WAL", + /* 326 */ "cmd", + /* 327 */ "account_options", + /* 328 */ "alter_account_options", + /* 329 */ "literal", + /* 330 */ "alter_account_option", + /* 331 */ "user_name", + /* 332 */ "sysinfo_opt", + /* 333 */ "privileges", + /* 334 */ "priv_level", + /* 335 */ "priv_type_list", + /* 336 */ "priv_type", + /* 337 */ "db_name", + /* 338 */ "topic_name", + /* 339 */ "dnode_endpoint", + /* 340 */ "force_opt", + /* 341 */ "not_exists_opt", + /* 342 */ "db_options", + /* 343 */ "exists_opt", + /* 344 */ "alter_db_options", + /* 345 */ "speed_opt", + /* 346 */ "integer_list", + /* 347 */ "variable_list", + /* 348 */ "retention_list", + /* 349 */ "alter_db_option", + /* 350 */ "retention", + /* 351 */ "full_table_name", + /* 352 */ "column_def_list", + /* 353 */ "tags_def_opt", + /* 354 */ "table_options", + /* 355 */ "multi_create_clause", + /* 356 */ "tags_def", + /* 357 */ "multi_drop_clause", + /* 358 */ "alter_table_clause", + /* 359 */ "alter_table_options", + /* 360 */ "column_name", + /* 361 */ "type_name", + /* 362 */ "signed_literal", + /* 363 */ "create_subtable_clause", + /* 364 */ "specific_cols_opt", + /* 365 */ "expression_list", + /* 366 */ "drop_table_clause", + /* 367 */ "col_name_list", + /* 368 */ "table_name", + /* 369 */ "column_def", + /* 370 */ "duration_list", + /* 371 */ "rollup_func_list", + /* 372 */ "alter_table_option", + /* 373 */ "duration_literal", + /* 374 */ "rollup_func_name", + /* 375 */ "function_name", + /* 376 */ "col_name", + /* 377 */ "db_name_cond_opt", + /* 378 */ "like_pattern_opt", + /* 379 */ "table_name_cond", + /* 380 */ "from_db_opt", + /* 381 */ "tag_list_opt", + /* 382 */ "tag_item", + /* 383 */ "column_alias", + /* 384 */ "full_index_name", + /* 385 */ "index_options", + /* 386 */ "index_name", + /* 387 */ "func_list", + /* 388 */ "sliding_opt", + /* 389 */ "sma_stream_opt", + /* 390 */ "func", + /* 391 */ "sma_func_name", + /* 392 */ "query_or_subquery", + /* 393 */ "cgroup_name", + /* 394 */ "analyze_opt", + /* 395 */ "explain_options", + /* 396 */ "agg_func_opt", + /* 397 */ "bufsize_opt", + /* 398 */ "stream_name", + /* 399 */ "stream_options", + /* 400 */ "col_list_opt", + /* 401 */ "subtable_opt", + /* 402 */ "expression", + /* 403 */ "dnode_list", + /* 404 */ "where_clause_opt", + /* 405 */ "signed", + /* 406 */ "literal_func", + /* 407 */ "literal_list", + /* 408 */ "table_alias", + /* 409 */ "expr_or_subquery", + /* 410 */ "pseudo_column", + /* 411 */ "column_reference", + /* 412 */ "function_expression", + /* 413 */ "case_when_expression", + /* 414 */ "star_func", + /* 415 */ "star_func_para_list", + /* 416 */ "noarg_func", + /* 417 */ "other_para_list", + /* 418 */ "star_func_para", + /* 419 */ "when_then_list", + /* 420 */ "case_when_else_opt", + /* 421 */ "common_expression", + /* 422 */ "when_then_expr", + /* 423 */ "predicate", + /* 424 */ "compare_op", + /* 425 */ "in_op", + /* 426 */ "in_predicate_value", + /* 427 */ "boolean_value_expression", + /* 428 */ "boolean_primary", + /* 429 */ "from_clause_opt", + /* 430 */ "table_reference_list", + /* 431 */ "table_reference", + /* 432 */ "table_primary", + /* 433 */ "joined_table", + /* 434 */ "alias_opt", + /* 435 */ "subquery", + /* 436 */ "parenthesized_joined_table", + /* 437 */ "join_type", + /* 438 */ "search_condition", + /* 439 */ "query_specification", + /* 440 */ "set_quantifier_opt", + /* 441 */ "select_list", + /* 442 */ "partition_by_clause_opt", + /* 443 */ "range_opt", + /* 444 */ "every_opt", + /* 445 */ "fill_opt", + /* 446 */ "twindow_clause_opt", + /* 447 */ "group_by_clause_opt", + /* 448 */ "having_clause_opt", + /* 449 */ "select_item", + /* 450 */ "partition_list", + /* 451 */ "partition_item", + /* 452 */ "fill_mode", + /* 453 */ "group_by_list", + /* 454 */ "query_expression", + /* 455 */ "query_simple", + /* 456 */ "order_by_clause_opt", + /* 457 */ "slimit_clause_opt", + /* 458 */ "limit_clause_opt", + /* 459 */ "union_query_expression", + /* 460 */ "query_simple_or_subquery", + /* 461 */ "sort_specification_list", + /* 462 */ "sort_specification", + /* 463 */ "ordering_specification_opt", + /* 464 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -2198,306 +2099,314 @@ static const char *const yyRuleName[] = { /* 242 */ "cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt", /* 243 */ "cmd ::= SHOW VNODES NK_INTEGER", /* 244 */ "cmd ::= SHOW VNODES NK_STRING", - /* 245 */ "db_name_cond_opt ::=", - /* 246 */ "db_name_cond_opt ::= db_name NK_DOT", - /* 247 */ "like_pattern_opt ::=", - /* 248 */ "like_pattern_opt ::= LIKE NK_STRING", - /* 249 */ "table_name_cond ::= table_name", - /* 250 */ "from_db_opt ::=", - /* 251 */ "from_db_opt ::= FROM db_name", - /* 252 */ "tag_list_opt ::=", - /* 253 */ "tag_list_opt ::= tag_item", - /* 254 */ "tag_list_opt ::= tag_list_opt NK_COMMA tag_item", - /* 255 */ "tag_item ::= TBNAME", - /* 256 */ "tag_item ::= QTAGS", - /* 257 */ "tag_item ::= column_name", - /* 258 */ "tag_item ::= column_name column_alias", - /* 259 */ "tag_item ::= column_name AS column_alias", - /* 260 */ "cmd ::= CREATE SMA INDEX not_exists_opt full_table_name ON full_table_name index_options", - /* 261 */ "cmd ::= DROP INDEX exists_opt full_table_name", - /* 262 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", - /* 263 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", - /* 264 */ "func_list ::= func", - /* 265 */ "func_list ::= func_list NK_COMMA func", - /* 266 */ "func ::= sma_func_name NK_LP expression_list NK_RP", - /* 267 */ "sma_func_name ::= function_name", - /* 268 */ "sma_func_name ::= COUNT", - /* 269 */ "sma_func_name ::= FIRST", - /* 270 */ "sma_func_name ::= LAST", - /* 271 */ "sma_func_name ::= LAST_ROW", - /* 272 */ "sma_stream_opt ::=", - /* 273 */ "sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal", - /* 274 */ "sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal", - /* 275 */ "sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal", - /* 276 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery", - /* 277 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", - /* 278 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", - /* 279 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name", - /* 280 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name", - /* 281 */ "cmd ::= DROP TOPIC exists_opt topic_name", - /* 282 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", - /* 283 */ "cmd ::= DESC full_table_name", - /* 284 */ "cmd ::= DESCRIBE full_table_name", - /* 285 */ "cmd ::= RESET QUERY CACHE", - /* 286 */ "cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery", - /* 287 */ "analyze_opt ::=", - /* 288 */ "analyze_opt ::= ANALYZE", - /* 289 */ "explain_options ::=", - /* 290 */ "explain_options ::= explain_options VERBOSE NK_BOOL", - /* 291 */ "explain_options ::= explain_options RATIO NK_FLOAT", - /* 292 */ "cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt", - /* 293 */ "cmd ::= DROP FUNCTION exists_opt function_name", - /* 294 */ "agg_func_opt ::=", - /* 295 */ "agg_func_opt ::= AGGREGATE", - /* 296 */ "bufsize_opt ::=", - /* 297 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", - /* 298 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name tags_def_opt subtable_opt AS query_or_subquery", - /* 299 */ "cmd ::= DROP STREAM exists_opt stream_name", - /* 300 */ "stream_options ::=", - /* 301 */ "stream_options ::= stream_options TRIGGER AT_ONCE", - /* 302 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", - /* 303 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", - /* 304 */ "stream_options ::= stream_options WATERMARK duration_literal", - /* 305 */ "stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER", - /* 306 */ "stream_options ::= stream_options FILL_HISTORY NK_INTEGER", - /* 307 */ "subtable_opt ::=", - /* 308 */ "subtable_opt ::= SUBTABLE NK_LP expression NK_RP", - /* 309 */ "cmd ::= KILL CONNECTION NK_INTEGER", - /* 310 */ "cmd ::= KILL QUERY NK_STRING", - /* 311 */ "cmd ::= KILL TRANSACTION NK_INTEGER", - /* 312 */ "cmd ::= BALANCE VGROUP", - /* 313 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", - /* 314 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", - /* 315 */ "cmd ::= SPLIT VGROUP NK_INTEGER", - /* 316 */ "dnode_list ::= DNODE NK_INTEGER", - /* 317 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", - /* 318 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", - /* 319 */ "cmd ::= query_or_subquery", - /* 320 */ "cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery", - /* 321 */ "cmd ::= INSERT INTO full_table_name query_or_subquery", - /* 322 */ "literal ::= NK_INTEGER", - /* 323 */ "literal ::= NK_FLOAT", - /* 324 */ "literal ::= NK_STRING", - /* 325 */ "literal ::= NK_BOOL", - /* 326 */ "literal ::= TIMESTAMP NK_STRING", - /* 327 */ "literal ::= duration_literal", - /* 328 */ "literal ::= NULL", - /* 329 */ "literal ::= NK_QUESTION", - /* 330 */ "duration_literal ::= NK_VARIABLE", - /* 331 */ "signed ::= NK_INTEGER", - /* 332 */ "signed ::= NK_PLUS NK_INTEGER", - /* 333 */ "signed ::= NK_MINUS NK_INTEGER", - /* 334 */ "signed ::= NK_FLOAT", - /* 335 */ "signed ::= NK_PLUS NK_FLOAT", - /* 336 */ "signed ::= NK_MINUS NK_FLOAT", - /* 337 */ "signed_literal ::= signed", - /* 338 */ "signed_literal ::= NK_STRING", - /* 339 */ "signed_literal ::= NK_BOOL", - /* 340 */ "signed_literal ::= TIMESTAMP NK_STRING", - /* 341 */ "signed_literal ::= duration_literal", - /* 342 */ "signed_literal ::= NULL", - /* 343 */ "signed_literal ::= literal_func", - /* 344 */ "signed_literal ::= NK_QUESTION", - /* 345 */ "literal_list ::= signed_literal", - /* 346 */ "literal_list ::= literal_list NK_COMMA signed_literal", - /* 347 */ "db_name ::= NK_ID", - /* 348 */ "table_name ::= NK_ID", - /* 349 */ "column_name ::= NK_ID", - /* 350 */ "function_name ::= NK_ID", - /* 351 */ "table_alias ::= NK_ID", - /* 352 */ "column_alias ::= NK_ID", - /* 353 */ "user_name ::= NK_ID", - /* 354 */ "topic_name ::= NK_ID", - /* 355 */ "stream_name ::= NK_ID", - /* 356 */ "cgroup_name ::= NK_ID", - /* 357 */ "expr_or_subquery ::= expression", - /* 358 */ "expression ::= literal", - /* 359 */ "expression ::= pseudo_column", - /* 360 */ "expression ::= column_reference", - /* 361 */ "expression ::= function_expression", - /* 362 */ "expression ::= case_when_expression", - /* 363 */ "expression ::= NK_LP expression NK_RP", - /* 364 */ "expression ::= NK_PLUS expr_or_subquery", - /* 365 */ "expression ::= NK_MINUS expr_or_subquery", - /* 366 */ "expression ::= expr_or_subquery NK_PLUS expr_or_subquery", - /* 367 */ "expression ::= expr_or_subquery NK_MINUS expr_or_subquery", - /* 368 */ "expression ::= expr_or_subquery NK_STAR expr_or_subquery", - /* 369 */ "expression ::= expr_or_subquery NK_SLASH expr_or_subquery", - /* 370 */ "expression ::= expr_or_subquery NK_REM expr_or_subquery", - /* 371 */ "expression ::= column_reference NK_ARROW NK_STRING", - /* 372 */ "expression ::= expr_or_subquery NK_BITAND expr_or_subquery", - /* 373 */ "expression ::= expr_or_subquery NK_BITOR expr_or_subquery", - /* 374 */ "expression_list ::= expr_or_subquery", - /* 375 */ "expression_list ::= expression_list NK_COMMA expr_or_subquery", - /* 376 */ "column_reference ::= column_name", - /* 377 */ "column_reference ::= table_name NK_DOT column_name", - /* 378 */ "pseudo_column ::= ROWTS", - /* 379 */ "pseudo_column ::= TBNAME", - /* 380 */ "pseudo_column ::= table_name NK_DOT TBNAME", - /* 381 */ "pseudo_column ::= QSTART", - /* 382 */ "pseudo_column ::= QEND", - /* 383 */ "pseudo_column ::= QDURATION", - /* 384 */ "pseudo_column ::= WSTART", - /* 385 */ "pseudo_column ::= WEND", - /* 386 */ "pseudo_column ::= WDURATION", - /* 387 */ "pseudo_column ::= IROWTS", - /* 388 */ "pseudo_column ::= ISFILLED", - /* 389 */ "pseudo_column ::= QTAGS", - /* 390 */ "function_expression ::= function_name NK_LP expression_list NK_RP", - /* 391 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", - /* 392 */ "function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP", - /* 393 */ "function_expression ::= literal_func", - /* 394 */ "literal_func ::= noarg_func NK_LP NK_RP", - /* 395 */ "literal_func ::= NOW", - /* 396 */ "noarg_func ::= NOW", - /* 397 */ "noarg_func ::= TODAY", - /* 398 */ "noarg_func ::= TIMEZONE", - /* 399 */ "noarg_func ::= DATABASE", - /* 400 */ "noarg_func ::= CLIENT_VERSION", - /* 401 */ "noarg_func ::= SERVER_VERSION", - /* 402 */ "noarg_func ::= SERVER_STATUS", - /* 403 */ "noarg_func ::= CURRENT_USER", - /* 404 */ "noarg_func ::= USER", - /* 405 */ "star_func ::= COUNT", - /* 406 */ "star_func ::= FIRST", - /* 407 */ "star_func ::= LAST", - /* 408 */ "star_func ::= LAST_ROW", - /* 409 */ "star_func_para_list ::= NK_STAR", - /* 410 */ "star_func_para_list ::= other_para_list", - /* 411 */ "other_para_list ::= star_func_para", - /* 412 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", - /* 413 */ "star_func_para ::= expr_or_subquery", - /* 414 */ "star_func_para ::= table_name NK_DOT NK_STAR", - /* 415 */ "case_when_expression ::= CASE when_then_list case_when_else_opt END", - /* 416 */ "case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END", - /* 417 */ "when_then_list ::= when_then_expr", - /* 418 */ "when_then_list ::= when_then_list when_then_expr", - /* 419 */ "when_then_expr ::= WHEN common_expression THEN common_expression", - /* 420 */ "case_when_else_opt ::=", - /* 421 */ "case_when_else_opt ::= ELSE common_expression", - /* 422 */ "predicate ::= expr_or_subquery compare_op expr_or_subquery", - /* 423 */ "predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery", - /* 424 */ "predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery", - /* 425 */ "predicate ::= expr_or_subquery IS NULL", - /* 426 */ "predicate ::= expr_or_subquery IS NOT NULL", - /* 427 */ "predicate ::= expr_or_subquery in_op in_predicate_value", - /* 428 */ "compare_op ::= NK_LT", - /* 429 */ "compare_op ::= NK_GT", - /* 430 */ "compare_op ::= NK_LE", - /* 431 */ "compare_op ::= NK_GE", - /* 432 */ "compare_op ::= NK_NE", - /* 433 */ "compare_op ::= NK_EQ", - /* 434 */ "compare_op ::= LIKE", - /* 435 */ "compare_op ::= NOT LIKE", - /* 436 */ "compare_op ::= MATCH", - /* 437 */ "compare_op ::= NMATCH", - /* 438 */ "compare_op ::= CONTAINS", - /* 439 */ "in_op ::= IN", - /* 440 */ "in_op ::= NOT IN", - /* 441 */ "in_predicate_value ::= NK_LP literal_list NK_RP", - /* 442 */ "boolean_value_expression ::= boolean_primary", - /* 443 */ "boolean_value_expression ::= NOT boolean_primary", - /* 444 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", - /* 445 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", - /* 446 */ "boolean_primary ::= predicate", - /* 447 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 448 */ "common_expression ::= expr_or_subquery", - /* 449 */ "common_expression ::= boolean_value_expression", - /* 450 */ "from_clause_opt ::=", - /* 451 */ "from_clause_opt ::= FROM table_reference_list", - /* 452 */ "table_reference_list ::= table_reference", - /* 453 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 454 */ "table_reference ::= table_primary", - /* 455 */ "table_reference ::= joined_table", - /* 456 */ "table_primary ::= table_name alias_opt", - /* 457 */ "table_primary ::= db_name NK_DOT table_name alias_opt", - /* 458 */ "table_primary ::= subquery alias_opt", - /* 459 */ "table_primary ::= parenthesized_joined_table", - /* 460 */ "alias_opt ::=", - /* 461 */ "alias_opt ::= table_alias", - /* 462 */ "alias_opt ::= AS table_alias", - /* 463 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 464 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 465 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 466 */ "join_type ::=", - /* 467 */ "join_type ::= INNER", - /* 468 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 469 */ "set_quantifier_opt ::=", - /* 470 */ "set_quantifier_opt ::= DISTINCT", - /* 471 */ "set_quantifier_opt ::= ALL", - /* 472 */ "select_list ::= select_item", - /* 473 */ "select_list ::= select_list NK_COMMA select_item", - /* 474 */ "select_item ::= NK_STAR", - /* 475 */ "select_item ::= common_expression", - /* 476 */ "select_item ::= common_expression column_alias", - /* 477 */ "select_item ::= common_expression AS column_alias", - /* 478 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 479 */ "where_clause_opt ::=", - /* 480 */ "where_clause_opt ::= WHERE search_condition", - /* 481 */ "partition_by_clause_opt ::=", - /* 482 */ "partition_by_clause_opt ::= PARTITION BY partition_list", - /* 483 */ "partition_list ::= partition_item", - /* 484 */ "partition_list ::= partition_list NK_COMMA partition_item", - /* 485 */ "partition_item ::= expr_or_subquery", - /* 486 */ "partition_item ::= expr_or_subquery column_alias", - /* 487 */ "partition_item ::= expr_or_subquery AS column_alias", - /* 488 */ "twindow_clause_opt ::=", - /* 489 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", - /* 490 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP", - /* 491 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 492 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 493 */ "twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition", - /* 494 */ "sliding_opt ::=", - /* 495 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 496 */ "fill_opt ::=", - /* 497 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 498 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 499 */ "fill_mode ::= NONE", - /* 500 */ "fill_mode ::= PREV", - /* 501 */ "fill_mode ::= NULL", - /* 502 */ "fill_mode ::= LINEAR", - /* 503 */ "fill_mode ::= NEXT", - /* 504 */ "group_by_clause_opt ::=", - /* 505 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 506 */ "group_by_list ::= expr_or_subquery", - /* 507 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", - /* 508 */ "having_clause_opt ::=", - /* 509 */ "having_clause_opt ::= HAVING search_condition", - /* 510 */ "range_opt ::=", - /* 511 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", - /* 512 */ "every_opt ::=", - /* 513 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", - /* 514 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 515 */ "query_simple ::= query_specification", - /* 516 */ "query_simple ::= union_query_expression", - /* 517 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", - /* 518 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", - /* 519 */ "query_simple_or_subquery ::= query_simple", - /* 520 */ "query_simple_or_subquery ::= subquery", - /* 521 */ "query_or_subquery ::= query_expression", - /* 522 */ "query_or_subquery ::= subquery", - /* 523 */ "order_by_clause_opt ::=", - /* 524 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 525 */ "slimit_clause_opt ::=", - /* 526 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 527 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 528 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 529 */ "limit_clause_opt ::=", - /* 530 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 531 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 532 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 533 */ "subquery ::= NK_LP query_expression NK_RP", - /* 534 */ "subquery ::= NK_LP subquery NK_RP", - /* 535 */ "search_condition ::= common_expression", - /* 536 */ "sort_specification_list ::= sort_specification", - /* 537 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 538 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", - /* 539 */ "ordering_specification_opt ::=", - /* 540 */ "ordering_specification_opt ::= ASC", - /* 541 */ "ordering_specification_opt ::= DESC", - /* 542 */ "null_ordering_opt ::=", - /* 543 */ "null_ordering_opt ::= NULLS FIRST", - /* 544 */ "null_ordering_opt ::= NULLS LAST", + /* 245 */ "cmd ::= SHOW db_name_cond_opt ALIVE", + /* 246 */ "cmd ::= SHOW CLUSTER ALIVE", + /* 247 */ "db_name_cond_opt ::=", + /* 248 */ "db_name_cond_opt ::= db_name NK_DOT", + /* 249 */ "like_pattern_opt ::=", + /* 250 */ "like_pattern_opt ::= LIKE NK_STRING", + /* 251 */ "table_name_cond ::= table_name", + /* 252 */ "from_db_opt ::=", + /* 253 */ "from_db_opt ::= FROM db_name", + /* 254 */ "tag_list_opt ::=", + /* 255 */ "tag_list_opt ::= tag_item", + /* 256 */ "tag_list_opt ::= tag_list_opt NK_COMMA tag_item", + /* 257 */ "tag_item ::= TBNAME", + /* 258 */ "tag_item ::= QTAGS", + /* 259 */ "tag_item ::= column_name", + /* 260 */ "tag_item ::= column_name column_alias", + /* 261 */ "tag_item ::= column_name AS column_alias", + /* 262 */ "cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options", + /* 263 */ "cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP", + /* 264 */ "cmd ::= DROP INDEX exists_opt full_index_name", + /* 265 */ "full_index_name ::= index_name", + /* 266 */ "full_index_name ::= db_name NK_DOT index_name", + /* 267 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", + /* 268 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", + /* 269 */ "func_list ::= func", + /* 270 */ "func_list ::= func_list NK_COMMA func", + /* 271 */ "func ::= sma_func_name NK_LP expression_list NK_RP", + /* 272 */ "sma_func_name ::= function_name", + /* 273 */ "sma_func_name ::= COUNT", + /* 274 */ "sma_func_name ::= FIRST", + /* 275 */ "sma_func_name ::= LAST", + /* 276 */ "sma_func_name ::= LAST_ROW", + /* 277 */ "sma_stream_opt ::=", + /* 278 */ "sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal", + /* 279 */ "sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal", + /* 280 */ "sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal", + /* 281 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery", + /* 282 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", + /* 283 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", + /* 284 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name", + /* 285 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name", + /* 286 */ "cmd ::= DROP TOPIC exists_opt topic_name", + /* 287 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", + /* 288 */ "cmd ::= DESC full_table_name", + /* 289 */ "cmd ::= DESCRIBE full_table_name", + /* 290 */ "cmd ::= RESET QUERY CACHE", + /* 291 */ "cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery", + /* 292 */ "analyze_opt ::=", + /* 293 */ "analyze_opt ::= ANALYZE", + /* 294 */ "explain_options ::=", + /* 295 */ "explain_options ::= explain_options VERBOSE NK_BOOL", + /* 296 */ "explain_options ::= explain_options RATIO NK_FLOAT", + /* 297 */ "cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt", + /* 298 */ "cmd ::= DROP FUNCTION exists_opt function_name", + /* 299 */ "agg_func_opt ::=", + /* 300 */ "agg_func_opt ::= AGGREGATE", + /* 301 */ "bufsize_opt ::=", + /* 302 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", + /* 303 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tags_def_opt subtable_opt AS query_or_subquery", + /* 304 */ "cmd ::= DROP STREAM exists_opt stream_name", + /* 305 */ "col_list_opt ::=", + /* 306 */ "col_list_opt ::= NK_LP col_name_list NK_RP", + /* 307 */ "stream_options ::=", + /* 308 */ "stream_options ::= stream_options TRIGGER AT_ONCE", + /* 309 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", + /* 310 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", + /* 311 */ "stream_options ::= stream_options WATERMARK duration_literal", + /* 312 */ "stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER", + /* 313 */ "stream_options ::= stream_options FILL_HISTORY NK_INTEGER", + /* 314 */ "subtable_opt ::=", + /* 315 */ "subtable_opt ::= SUBTABLE NK_LP expression NK_RP", + /* 316 */ "cmd ::= KILL CONNECTION NK_INTEGER", + /* 317 */ "cmd ::= KILL QUERY NK_STRING", + /* 318 */ "cmd ::= KILL TRANSACTION NK_INTEGER", + /* 319 */ "cmd ::= BALANCE VGROUP", + /* 320 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", + /* 321 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", + /* 322 */ "cmd ::= SPLIT VGROUP NK_INTEGER", + /* 323 */ "dnode_list ::= DNODE NK_INTEGER", + /* 324 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", + /* 325 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", + /* 326 */ "cmd ::= query_or_subquery", + /* 327 */ "cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery", + /* 328 */ "cmd ::= INSERT INTO full_table_name query_or_subquery", + /* 329 */ "literal ::= NK_INTEGER", + /* 330 */ "literal ::= NK_FLOAT", + /* 331 */ "literal ::= NK_STRING", + /* 332 */ "literal ::= NK_BOOL", + /* 333 */ "literal ::= TIMESTAMP NK_STRING", + /* 334 */ "literal ::= duration_literal", + /* 335 */ "literal ::= NULL", + /* 336 */ "literal ::= NK_QUESTION", + /* 337 */ "duration_literal ::= NK_VARIABLE", + /* 338 */ "signed ::= NK_INTEGER", + /* 339 */ "signed ::= NK_PLUS NK_INTEGER", + /* 340 */ "signed ::= NK_MINUS NK_INTEGER", + /* 341 */ "signed ::= NK_FLOAT", + /* 342 */ "signed ::= NK_PLUS NK_FLOAT", + /* 343 */ "signed ::= NK_MINUS NK_FLOAT", + /* 344 */ "signed_literal ::= signed", + /* 345 */ "signed_literal ::= NK_STRING", + /* 346 */ "signed_literal ::= NK_BOOL", + /* 347 */ "signed_literal ::= TIMESTAMP NK_STRING", + /* 348 */ "signed_literal ::= duration_literal", + /* 349 */ "signed_literal ::= NULL", + /* 350 */ "signed_literal ::= literal_func", + /* 351 */ "signed_literal ::= NK_QUESTION", + /* 352 */ "literal_list ::= signed_literal", + /* 353 */ "literal_list ::= literal_list NK_COMMA signed_literal", + /* 354 */ "db_name ::= NK_ID", + /* 355 */ "table_name ::= NK_ID", + /* 356 */ "column_name ::= NK_ID", + /* 357 */ "function_name ::= NK_ID", + /* 358 */ "table_alias ::= NK_ID", + /* 359 */ "column_alias ::= NK_ID", + /* 360 */ "user_name ::= NK_ID", + /* 361 */ "topic_name ::= NK_ID", + /* 362 */ "stream_name ::= NK_ID", + /* 363 */ "cgroup_name ::= NK_ID", + /* 364 */ "index_name ::= NK_ID", + /* 365 */ "expr_or_subquery ::= expression", + /* 366 */ "expression ::= literal", + /* 367 */ "expression ::= pseudo_column", + /* 368 */ "expression ::= column_reference", + /* 369 */ "expression ::= function_expression", + /* 370 */ "expression ::= case_when_expression", + /* 371 */ "expression ::= NK_LP expression NK_RP", + /* 372 */ "expression ::= NK_PLUS expr_or_subquery", + /* 373 */ "expression ::= NK_MINUS expr_or_subquery", + /* 374 */ "expression ::= expr_or_subquery NK_PLUS expr_or_subquery", + /* 375 */ "expression ::= expr_or_subquery NK_MINUS expr_or_subquery", + /* 376 */ "expression ::= expr_or_subquery NK_STAR expr_or_subquery", + /* 377 */ "expression ::= expr_or_subquery NK_SLASH expr_or_subquery", + /* 378 */ "expression ::= expr_or_subquery NK_REM expr_or_subquery", + /* 379 */ "expression ::= column_reference NK_ARROW NK_STRING", + /* 380 */ "expression ::= expr_or_subquery NK_BITAND expr_or_subquery", + /* 381 */ "expression ::= expr_or_subquery NK_BITOR expr_or_subquery", + /* 382 */ "expression_list ::= expr_or_subquery", + /* 383 */ "expression_list ::= expression_list NK_COMMA expr_or_subquery", + /* 384 */ "column_reference ::= column_name", + /* 385 */ "column_reference ::= table_name NK_DOT column_name", + /* 386 */ "pseudo_column ::= ROWTS", + /* 387 */ "pseudo_column ::= TBNAME", + /* 388 */ "pseudo_column ::= table_name NK_DOT TBNAME", + /* 389 */ "pseudo_column ::= QSTART", + /* 390 */ "pseudo_column ::= QEND", + /* 391 */ "pseudo_column ::= QDURATION", + /* 392 */ "pseudo_column ::= WSTART", + /* 393 */ "pseudo_column ::= WEND", + /* 394 */ "pseudo_column ::= WDURATION", + /* 395 */ "pseudo_column ::= IROWTS", + /* 396 */ "pseudo_column ::= ISFILLED", + /* 397 */ "pseudo_column ::= QTAGS", + /* 398 */ "function_expression ::= function_name NK_LP expression_list NK_RP", + /* 399 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", + /* 400 */ "function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP", + /* 401 */ "function_expression ::= literal_func", + /* 402 */ "literal_func ::= noarg_func NK_LP NK_RP", + /* 403 */ "literal_func ::= NOW", + /* 404 */ "noarg_func ::= NOW", + /* 405 */ "noarg_func ::= TODAY", + /* 406 */ "noarg_func ::= TIMEZONE", + /* 407 */ "noarg_func ::= DATABASE", + /* 408 */ "noarg_func ::= CLIENT_VERSION", + /* 409 */ "noarg_func ::= SERVER_VERSION", + /* 410 */ "noarg_func ::= SERVER_STATUS", + /* 411 */ "noarg_func ::= CURRENT_USER", + /* 412 */ "noarg_func ::= USER", + /* 413 */ "star_func ::= COUNT", + /* 414 */ "star_func ::= FIRST", + /* 415 */ "star_func ::= LAST", + /* 416 */ "star_func ::= LAST_ROW", + /* 417 */ "star_func_para_list ::= NK_STAR", + /* 418 */ "star_func_para_list ::= other_para_list", + /* 419 */ "other_para_list ::= star_func_para", + /* 420 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", + /* 421 */ "star_func_para ::= expr_or_subquery", + /* 422 */ "star_func_para ::= table_name NK_DOT NK_STAR", + /* 423 */ "case_when_expression ::= CASE when_then_list case_when_else_opt END", + /* 424 */ "case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END", + /* 425 */ "when_then_list ::= when_then_expr", + /* 426 */ "when_then_list ::= when_then_list when_then_expr", + /* 427 */ "when_then_expr ::= WHEN common_expression THEN common_expression", + /* 428 */ "case_when_else_opt ::=", + /* 429 */ "case_when_else_opt ::= ELSE common_expression", + /* 430 */ "predicate ::= expr_or_subquery compare_op expr_or_subquery", + /* 431 */ "predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery", + /* 432 */ "predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery", + /* 433 */ "predicate ::= expr_or_subquery IS NULL", + /* 434 */ "predicate ::= expr_or_subquery IS NOT NULL", + /* 435 */ "predicate ::= expr_or_subquery in_op in_predicate_value", + /* 436 */ "compare_op ::= NK_LT", + /* 437 */ "compare_op ::= NK_GT", + /* 438 */ "compare_op ::= NK_LE", + /* 439 */ "compare_op ::= NK_GE", + /* 440 */ "compare_op ::= NK_NE", + /* 441 */ "compare_op ::= NK_EQ", + /* 442 */ "compare_op ::= LIKE", + /* 443 */ "compare_op ::= NOT LIKE", + /* 444 */ "compare_op ::= MATCH", + /* 445 */ "compare_op ::= NMATCH", + /* 446 */ "compare_op ::= CONTAINS", + /* 447 */ "in_op ::= IN", + /* 448 */ "in_op ::= NOT IN", + /* 449 */ "in_predicate_value ::= NK_LP literal_list NK_RP", + /* 450 */ "boolean_value_expression ::= boolean_primary", + /* 451 */ "boolean_value_expression ::= NOT boolean_primary", + /* 452 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 453 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 454 */ "boolean_primary ::= predicate", + /* 455 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 456 */ "common_expression ::= expr_or_subquery", + /* 457 */ "common_expression ::= boolean_value_expression", + /* 458 */ "from_clause_opt ::=", + /* 459 */ "from_clause_opt ::= FROM table_reference_list", + /* 460 */ "table_reference_list ::= table_reference", + /* 461 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 462 */ "table_reference ::= table_primary", + /* 463 */ "table_reference ::= joined_table", + /* 464 */ "table_primary ::= table_name alias_opt", + /* 465 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 466 */ "table_primary ::= subquery alias_opt", + /* 467 */ "table_primary ::= parenthesized_joined_table", + /* 468 */ "alias_opt ::=", + /* 469 */ "alias_opt ::= table_alias", + /* 470 */ "alias_opt ::= AS table_alias", + /* 471 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 472 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 473 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 474 */ "join_type ::=", + /* 475 */ "join_type ::= INNER", + /* 476 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 477 */ "set_quantifier_opt ::=", + /* 478 */ "set_quantifier_opt ::= DISTINCT", + /* 479 */ "set_quantifier_opt ::= ALL", + /* 480 */ "select_list ::= select_item", + /* 481 */ "select_list ::= select_list NK_COMMA select_item", + /* 482 */ "select_item ::= NK_STAR", + /* 483 */ "select_item ::= common_expression", + /* 484 */ "select_item ::= common_expression column_alias", + /* 485 */ "select_item ::= common_expression AS column_alias", + /* 486 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 487 */ "where_clause_opt ::=", + /* 488 */ "where_clause_opt ::= WHERE search_condition", + /* 489 */ "partition_by_clause_opt ::=", + /* 490 */ "partition_by_clause_opt ::= PARTITION BY partition_list", + /* 491 */ "partition_list ::= partition_item", + /* 492 */ "partition_list ::= partition_list NK_COMMA partition_item", + /* 493 */ "partition_item ::= expr_or_subquery", + /* 494 */ "partition_item ::= expr_or_subquery column_alias", + /* 495 */ "partition_item ::= expr_or_subquery AS column_alias", + /* 496 */ "twindow_clause_opt ::=", + /* 497 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", + /* 498 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP", + /* 499 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 500 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 501 */ "twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition", + /* 502 */ "sliding_opt ::=", + /* 503 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 504 */ "fill_opt ::=", + /* 505 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 506 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 507 */ "fill_mode ::= NONE", + /* 508 */ "fill_mode ::= PREV", + /* 509 */ "fill_mode ::= NULL", + /* 510 */ "fill_mode ::= LINEAR", + /* 511 */ "fill_mode ::= NEXT", + /* 512 */ "group_by_clause_opt ::=", + /* 513 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 514 */ "group_by_list ::= expr_or_subquery", + /* 515 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", + /* 516 */ "having_clause_opt ::=", + /* 517 */ "having_clause_opt ::= HAVING search_condition", + /* 518 */ "range_opt ::=", + /* 519 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", + /* 520 */ "every_opt ::=", + /* 521 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", + /* 522 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 523 */ "query_simple ::= query_specification", + /* 524 */ "query_simple ::= union_query_expression", + /* 525 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", + /* 526 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", + /* 527 */ "query_simple_or_subquery ::= query_simple", + /* 528 */ "query_simple_or_subquery ::= subquery", + /* 529 */ "query_or_subquery ::= query_expression", + /* 530 */ "query_or_subquery ::= subquery", + /* 531 */ "order_by_clause_opt ::=", + /* 532 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 533 */ "slimit_clause_opt ::=", + /* 534 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 535 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 536 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 537 */ "limit_clause_opt ::=", + /* 538 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 539 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 540 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 541 */ "subquery ::= NK_LP query_expression NK_RP", + /* 542 */ "subquery ::= NK_LP subquery NK_RP", + /* 543 */ "search_condition ::= common_expression", + /* 544 */ "sort_specification_list ::= sort_specification", + /* 545 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 546 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", + /* 547 */ "ordering_specification_opt ::=", + /* 548 */ "ordering_specification_opt ::= ASC", + /* 549 */ "ordering_specification_opt ::= DESC", + /* 550 */ "null_ordering_opt ::=", + /* 551 */ "null_ordering_opt ::= NULLS FIRST", + /* 552 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -2624,194 +2533,197 @@ static void yy_destructor( */ /********* Begin destructor definitions ***************************************/ /* Default NON-TERMINAL Destructor */ - case 325: /* cmd */ - case 328: /* literal */ - case 341: /* db_options */ - case 343: /* alter_db_options */ - case 349: /* retention */ - case 350: /* full_table_name */ - case 353: /* table_options */ - case 357: /* alter_table_clause */ - case 358: /* alter_table_options */ - case 361: /* signed_literal */ - case 362: /* create_subtable_clause */ - case 365: /* drop_table_clause */ - case 368: /* column_def */ - case 372: /* duration_literal */ - case 373: /* rollup_func_name */ - case 375: /* col_name */ - case 376: /* db_name_cond_opt */ - case 377: /* like_pattern_opt */ - case 378: /* table_name_cond */ - case 379: /* from_db_opt */ - case 381: /* tag_item */ - case 383: /* index_options */ - case 385: /* sliding_opt */ - case 386: /* sma_stream_opt */ - case 387: /* func */ - case 389: /* query_or_subquery */ - case 392: /* explain_options */ - case 396: /* stream_options */ - case 397: /* subtable_opt */ - case 398: /* expression */ - case 400: /* where_clause_opt */ - case 401: /* signed */ - case 402: /* literal_func */ - case 405: /* expr_or_subquery */ - case 406: /* pseudo_column */ - case 407: /* column_reference */ - case 408: /* function_expression */ - case 409: /* case_when_expression */ - case 414: /* star_func_para */ - case 416: /* case_when_else_opt */ - case 417: /* common_expression */ - case 418: /* when_then_expr */ - case 419: /* predicate */ - case 422: /* in_predicate_value */ - case 423: /* boolean_value_expression */ - case 424: /* boolean_primary */ - case 425: /* from_clause_opt */ - case 426: /* table_reference_list */ - case 427: /* table_reference */ - case 428: /* table_primary */ - case 429: /* joined_table */ - case 431: /* subquery */ - case 432: /* parenthesized_joined_table */ - case 434: /* search_condition */ - case 435: /* query_specification */ - case 439: /* range_opt */ - case 440: /* every_opt */ - case 441: /* fill_opt */ - case 442: /* twindow_clause_opt */ - case 444: /* having_clause_opt */ - case 445: /* select_item */ - case 447: /* partition_item */ - case 450: /* query_expression */ - case 451: /* query_simple */ - case 453: /* slimit_clause_opt */ - case 454: /* limit_clause_opt */ - case 455: /* union_query_expression */ - case 456: /* query_simple_or_subquery */ - case 458: /* sort_specification */ + case 326: /* cmd */ + case 329: /* literal */ + case 342: /* db_options */ + case 344: /* alter_db_options */ + case 350: /* retention */ + case 351: /* full_table_name */ + case 354: /* table_options */ + case 358: /* alter_table_clause */ + case 359: /* alter_table_options */ + case 362: /* signed_literal */ + case 363: /* create_subtable_clause */ + case 366: /* drop_table_clause */ + case 369: /* column_def */ + case 373: /* duration_literal */ + case 374: /* rollup_func_name */ + case 376: /* col_name */ + case 377: /* db_name_cond_opt */ + case 378: /* like_pattern_opt */ + case 379: /* table_name_cond */ + case 380: /* from_db_opt */ + case 382: /* tag_item */ + case 384: /* full_index_name */ + case 385: /* index_options */ + case 388: /* sliding_opt */ + case 389: /* sma_stream_opt */ + case 390: /* func */ + case 392: /* query_or_subquery */ + case 395: /* explain_options */ + case 399: /* stream_options */ + case 401: /* subtable_opt */ + case 402: /* expression */ + case 404: /* where_clause_opt */ + case 405: /* signed */ + case 406: /* literal_func */ + case 409: /* expr_or_subquery */ + case 410: /* pseudo_column */ + case 411: /* column_reference */ + case 412: /* function_expression */ + case 413: /* case_when_expression */ + case 418: /* star_func_para */ + case 420: /* case_when_else_opt */ + case 421: /* common_expression */ + case 422: /* when_then_expr */ + case 423: /* predicate */ + case 426: /* in_predicate_value */ + case 427: /* boolean_value_expression */ + case 428: /* boolean_primary */ + case 429: /* from_clause_opt */ + case 430: /* table_reference_list */ + case 431: /* table_reference */ + case 432: /* table_primary */ + case 433: /* joined_table */ + case 435: /* subquery */ + case 436: /* parenthesized_joined_table */ + case 438: /* search_condition */ + case 439: /* query_specification */ + case 443: /* range_opt */ + case 444: /* every_opt */ + case 445: /* fill_opt */ + case 446: /* twindow_clause_opt */ + case 448: /* having_clause_opt */ + case 449: /* select_item */ + case 451: /* partition_item */ + case 454: /* query_expression */ + case 455: /* query_simple */ + case 457: /* slimit_clause_opt */ + case 458: /* limit_clause_opt */ + case 459: /* union_query_expression */ + case 460: /* query_simple_or_subquery */ + case 462: /* sort_specification */ { - nodesDestroyNode((yypminor->yy600)); + nodesDestroyNode((yypminor->yy602)); } break; - case 326: /* account_options */ - case 327: /* alter_account_options */ - case 329: /* alter_account_option */ - case 344: /* speed_opt */ - case 394: /* bufsize_opt */ + case 327: /* account_options */ + case 328: /* alter_account_options */ + case 330: /* alter_account_option */ + case 345: /* speed_opt */ + case 397: /* bufsize_opt */ { } break; - case 330: /* user_name */ - case 333: /* priv_level */ - case 336: /* db_name */ - case 337: /* topic_name */ - case 338: /* dnode_endpoint */ - case 359: /* column_name */ - case 367: /* table_name */ - case 374: /* function_name */ - case 382: /* column_alias */ - case 388: /* sma_func_name */ - case 390: /* cgroup_name */ - case 395: /* stream_name */ - case 404: /* table_alias */ - case 410: /* star_func */ - case 412: /* noarg_func */ - case 430: /* alias_opt */ + case 331: /* user_name */ + case 334: /* priv_level */ + case 337: /* db_name */ + case 338: /* topic_name */ + case 339: /* dnode_endpoint */ + case 360: /* column_name */ + case 368: /* table_name */ + case 375: /* function_name */ + case 383: /* column_alias */ + case 386: /* index_name */ + case 391: /* sma_func_name */ + case 393: /* cgroup_name */ + case 398: /* stream_name */ + case 408: /* table_alias */ + case 414: /* star_func */ + case 416: /* noarg_func */ + case 434: /* alias_opt */ { } break; - case 331: /* sysinfo_opt */ + case 332: /* sysinfo_opt */ { } break; - case 332: /* privileges */ - case 334: /* priv_type_list */ - case 335: /* priv_type */ + case 333: /* privileges */ + case 335: /* priv_type_list */ + case 336: /* priv_type */ { } break; - case 339: /* force_opt */ - case 340: /* not_exists_opt */ - case 342: /* exists_opt */ - case 391: /* analyze_opt */ - case 393: /* agg_func_opt */ - case 436: /* set_quantifier_opt */ + case 340: /* force_opt */ + case 341: /* not_exists_opt */ + case 343: /* exists_opt */ + case 394: /* analyze_opt */ + case 396: /* agg_func_opt */ + case 440: /* set_quantifier_opt */ { } break; - case 345: /* integer_list */ - case 346: /* variable_list */ - case 347: /* retention_list */ - case 351: /* column_def_list */ - case 352: /* tags_def_opt */ - case 354: /* multi_create_clause */ - case 355: /* tags_def */ - case 356: /* multi_drop_clause */ - case 363: /* specific_cols_opt */ - case 364: /* expression_list */ - case 366: /* col_name_list */ - case 369: /* duration_list */ - case 370: /* rollup_func_list */ - case 380: /* tag_list_opt */ - case 384: /* func_list */ - case 399: /* dnode_list */ - case 403: /* literal_list */ - case 411: /* star_func_para_list */ - case 413: /* other_para_list */ - case 415: /* when_then_list */ - case 437: /* select_list */ - case 438: /* partition_by_clause_opt */ - case 443: /* group_by_clause_opt */ - case 446: /* partition_list */ - case 449: /* group_by_list */ - case 452: /* order_by_clause_opt */ - case 457: /* sort_specification_list */ + case 346: /* integer_list */ + case 347: /* variable_list */ + case 348: /* retention_list */ + case 352: /* column_def_list */ + case 353: /* tags_def_opt */ + case 355: /* multi_create_clause */ + case 356: /* tags_def */ + case 357: /* multi_drop_clause */ + case 364: /* specific_cols_opt */ + case 365: /* expression_list */ + case 367: /* col_name_list */ + case 370: /* duration_list */ + case 371: /* rollup_func_list */ + case 381: /* tag_list_opt */ + case 387: /* func_list */ + case 400: /* col_list_opt */ + case 403: /* dnode_list */ + case 407: /* literal_list */ + case 415: /* star_func_para_list */ + case 417: /* other_para_list */ + case 419: /* when_then_list */ + case 441: /* select_list */ + case 442: /* partition_by_clause_opt */ + case 447: /* group_by_clause_opt */ + case 450: /* partition_list */ + case 453: /* group_by_list */ + case 456: /* order_by_clause_opt */ + case 461: /* sort_specification_list */ { - nodesDestroyList((yypminor->yy601)); + nodesDestroyList((yypminor->yy874)); } break; - case 348: /* alter_db_option */ - case 371: /* alter_table_option */ + case 349: /* alter_db_option */ + case 372: /* alter_table_option */ { } break; - case 360: /* type_name */ + case 361: /* type_name */ { } break; - case 420: /* compare_op */ - case 421: /* in_op */ + case 424: /* compare_op */ + case 425: /* in_op */ { } break; - case 433: /* join_type */ + case 437: /* join_type */ { } break; - case 448: /* fill_mode */ + case 452: /* fill_mode */ { } break; - case 459: /* ordering_specification_opt */ + case 463: /* ordering_specification_opt */ { } break; - case 460: /* null_ordering_opt */ + case 464: /* null_ordering_opt */ { } @@ -2939,18 +2851,15 @@ static YYACTIONTYPE yy_find_shift_action( do{ i = yy_shift_ofst[stateno]; assert( i>=0 ); - assert( i<=YY_ACTTAB_COUNT ); - assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); + /* assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); */ assert( iLookAhead!=YYNOCODE ); assert( iLookAhead < YYNTOKEN ); i += iLookAhead; - assert( i<(int)YY_NLOOKAHEAD ); - if( yy_lookahead[i]!=iLookAhead ){ + if( i>=YY_NLOOKAHEAD || yy_lookahead[i]!=iLookAhead ){ #ifdef YYFALLBACK YYCODETYPE iFallback; /* Fallback token */ - assert( iLookAhead %s\n", @@ -2965,8 +2874,16 @@ static YYACTIONTYPE yy_find_shift_action( #ifdef YYWILDCARD { int j = i - iLookAhead + YYWILDCARD; - assert( j<(int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])) ); - if( yy_lookahead[j]==YYWILDCARD && iLookAhead>0 ){ + if( +#if YY_SHIFT_MIN+YYWILDCARD<0 + j>=0 && +#endif +#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT + j0 + ){ #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", @@ -2980,7 +2897,6 @@ static YYACTIONTYPE yy_find_shift_action( #endif /* YYWILDCARD */ return yy_default[stateno]; }else{ - assert( i>=0 && iyytos; #ifndef NDEBUG if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - yysize = yyRuleInfoNRhs[yyruleno]; + yysize = yyRuleInfo[yyruleno].nrhs; if( yysize ){ - fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", + fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n", yyTracePrompt, - yyruleno, yyRuleName[yyruleno], - yyrulenoyytos - yypParser->yystack)>yypParser->yyhwm ){ yypParser->yyhwm++; @@ -4288,11 +3663,11 @@ static YYACTIONTYPE yy_reduce( YYMINORTYPE yylhsminor; case 0: /* cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,326,&yymsp[0].minor); + yy_destructor(yypParser,327,&yymsp[0].minor); break; case 1: /* cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,327,&yymsp[0].minor); + yy_destructor(yypParser,328,&yymsp[0].minor); break; case 2: /* account_options ::= */ { } @@ -4306,20 +3681,20 @@ static YYACTIONTYPE yy_reduce( case 9: /* account_options ::= account_options USERS literal */ yytestcase(yyruleno==9); case 10: /* account_options ::= account_options CONNS literal */ yytestcase(yyruleno==10); case 11: /* account_options ::= account_options STATE literal */ yytestcase(yyruleno==11); -{ yy_destructor(yypParser,326,&yymsp[-2].minor); +{ yy_destructor(yypParser,327,&yymsp[-2].minor); { } - yy_destructor(yypParser,328,&yymsp[0].minor); + yy_destructor(yypParser,329,&yymsp[0].minor); } break; case 12: /* alter_account_options ::= alter_account_option */ -{ yy_destructor(yypParser,329,&yymsp[0].minor); +{ yy_destructor(yypParser,330,&yymsp[0].minor); { } } break; case 13: /* alter_account_options ::= alter_account_options alter_account_option */ -{ yy_destructor(yypParser,327,&yymsp[-1].minor); +{ yy_destructor(yypParser,328,&yymsp[-1].minor); { } - yy_destructor(yypParser,329,&yymsp[0].minor); + yy_destructor(yypParser,330,&yymsp[0].minor); } break; case 14: /* alter_account_option ::= PASS literal */ @@ -4333,81 +3708,81 @@ static YYACTIONTYPE yy_reduce( case 22: /* alter_account_option ::= CONNS literal */ yytestcase(yyruleno==22); case 23: /* alter_account_option ::= STATE literal */ yytestcase(yyruleno==23); { } - yy_destructor(yypParser,328,&yymsp[0].minor); + yy_destructor(yypParser,329,&yymsp[0].minor); break; case 24: /* cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ -{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-3].minor.yy77, &yymsp[-1].minor.yy0, yymsp[0].minor.yy287); } +{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-3].minor.yy179, &yymsp[-1].minor.yy0, yymsp[0].minor.yy113); } break; case 25: /* cmd ::= ALTER USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy77, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy179, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } break; case 26: /* cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy77, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy179, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } break; case 27: /* cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy77, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy179, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } break; case 28: /* cmd ::= DROP USER user_name */ -{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy77); } +{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy179); } break; case 29: /* sysinfo_opt ::= */ -{ yymsp[1].minor.yy287 = 1; } +{ yymsp[1].minor.yy113 = 1; } break; case 30: /* sysinfo_opt ::= SYSINFO NK_INTEGER */ -{ yymsp[-1].minor.yy287 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } +{ yymsp[-1].minor.yy113 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } break; case 31: /* cmd ::= GRANT privileges ON priv_level TO user_name */ -{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-4].minor.yy717, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy77); } +{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-4].minor.yy159, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy179); } break; case 32: /* cmd ::= REVOKE privileges ON priv_level FROM user_name */ -{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-4].minor.yy717, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy77); } +{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-4].minor.yy159, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy179); } break; case 33: /* privileges ::= ALL */ -{ yymsp[0].minor.yy717 = PRIVILEGE_TYPE_ALL; } +{ yymsp[0].minor.yy159 = PRIVILEGE_TYPE_ALL; } break; case 34: /* privileges ::= priv_type_list */ case 36: /* priv_type_list ::= priv_type */ yytestcase(yyruleno==36); -{ yylhsminor.yy717 = yymsp[0].minor.yy717; } - yymsp[0].minor.yy717 = yylhsminor.yy717; +{ yylhsminor.yy159 = yymsp[0].minor.yy159; } + yymsp[0].minor.yy159 = yylhsminor.yy159; break; case 35: /* privileges ::= SUBSCRIBE */ -{ yymsp[0].minor.yy717 = PRIVILEGE_TYPE_SUBSCRIBE; } +{ yymsp[0].minor.yy159 = PRIVILEGE_TYPE_SUBSCRIBE; } break; case 37: /* priv_type_list ::= priv_type_list NK_COMMA priv_type */ -{ yylhsminor.yy717 = yymsp[-2].minor.yy717 | yymsp[0].minor.yy717; } - yymsp[-2].minor.yy717 = yylhsminor.yy717; +{ yylhsminor.yy159 = yymsp[-2].minor.yy159 | yymsp[0].minor.yy159; } + yymsp[-2].minor.yy159 = yylhsminor.yy159; break; case 38: /* priv_type ::= READ */ -{ yymsp[0].minor.yy717 = PRIVILEGE_TYPE_READ; } +{ yymsp[0].minor.yy159 = PRIVILEGE_TYPE_READ; } break; case 39: /* priv_type ::= WRITE */ -{ yymsp[0].minor.yy717 = PRIVILEGE_TYPE_WRITE; } +{ yymsp[0].minor.yy159 = PRIVILEGE_TYPE_WRITE; } break; case 40: /* priv_level ::= NK_STAR NK_DOT NK_STAR */ -{ yylhsminor.yy77 = yymsp[-2].minor.yy0; } - yymsp[-2].minor.yy77 = yylhsminor.yy77; +{ yylhsminor.yy179 = yymsp[-2].minor.yy0; } + yymsp[-2].minor.yy179 = yylhsminor.yy179; break; case 41: /* priv_level ::= db_name NK_DOT NK_STAR */ -{ yylhsminor.yy77 = yymsp[-2].minor.yy77; } - yymsp[-2].minor.yy77 = yylhsminor.yy77; +{ yylhsminor.yy179 = yymsp[-2].minor.yy179; } + yymsp[-2].minor.yy179 = yylhsminor.yy179; break; case 42: /* priv_level ::= topic_name */ - case 267: /* sma_func_name ::= function_name */ yytestcase(yyruleno==267); - case 461: /* alias_opt ::= table_alias */ yytestcase(yyruleno==461); -{ yylhsminor.yy77 = yymsp[0].minor.yy77; } - yymsp[0].minor.yy77 = yylhsminor.yy77; + case 272: /* sma_func_name ::= function_name */ yytestcase(yyruleno==272); + case 469: /* alias_opt ::= table_alias */ yytestcase(yyruleno==469); +{ yylhsminor.yy179 = yymsp[0].minor.yy179; } + yymsp[0].minor.yy179 = yylhsminor.yy179; break; case 43: /* cmd ::= CREATE DNODE dnode_endpoint */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy77, NULL); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy179, NULL); } break; case 44: /* cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy0); } break; case 45: /* cmd ::= DROP DNODE NK_INTEGER force_opt */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy841); } +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy767); } break; case 46: /* cmd ::= DROP DNODE dnode_endpoint force_opt */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy77, yymsp[0].minor.yy841); } +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy179, yymsp[0].minor.yy767); } break; case 47: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, NULL); } @@ -4424,49 +3799,50 @@ static YYACTIONTYPE yy_reduce( case 51: /* dnode_endpoint ::= NK_STRING */ case 52: /* dnode_endpoint ::= NK_ID */ yytestcase(yyruleno==52); case 53: /* dnode_endpoint ::= NK_IPTOKEN */ yytestcase(yyruleno==53); - case 268: /* sma_func_name ::= COUNT */ yytestcase(yyruleno==268); - case 269: /* sma_func_name ::= FIRST */ yytestcase(yyruleno==269); - case 270: /* sma_func_name ::= LAST */ yytestcase(yyruleno==270); - case 271: /* sma_func_name ::= LAST_ROW */ yytestcase(yyruleno==271); - case 347: /* db_name ::= NK_ID */ yytestcase(yyruleno==347); - case 348: /* table_name ::= NK_ID */ yytestcase(yyruleno==348); - case 349: /* column_name ::= NK_ID */ yytestcase(yyruleno==349); - case 350: /* function_name ::= NK_ID */ yytestcase(yyruleno==350); - case 351: /* table_alias ::= NK_ID */ yytestcase(yyruleno==351); - case 352: /* column_alias ::= NK_ID */ yytestcase(yyruleno==352); - case 353: /* user_name ::= NK_ID */ yytestcase(yyruleno==353); - case 354: /* topic_name ::= NK_ID */ yytestcase(yyruleno==354); - case 355: /* stream_name ::= NK_ID */ yytestcase(yyruleno==355); - case 356: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==356); - case 396: /* noarg_func ::= NOW */ yytestcase(yyruleno==396); - case 397: /* noarg_func ::= TODAY */ yytestcase(yyruleno==397); - case 398: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==398); - case 399: /* noarg_func ::= DATABASE */ yytestcase(yyruleno==399); - case 400: /* noarg_func ::= CLIENT_VERSION */ yytestcase(yyruleno==400); - case 401: /* noarg_func ::= SERVER_VERSION */ yytestcase(yyruleno==401); - case 402: /* noarg_func ::= SERVER_STATUS */ yytestcase(yyruleno==402); - case 403: /* noarg_func ::= CURRENT_USER */ yytestcase(yyruleno==403); - case 404: /* noarg_func ::= USER */ yytestcase(yyruleno==404); - case 405: /* star_func ::= COUNT */ yytestcase(yyruleno==405); - case 406: /* star_func ::= FIRST */ yytestcase(yyruleno==406); - case 407: /* star_func ::= LAST */ yytestcase(yyruleno==407); - case 408: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==408); -{ yylhsminor.yy77 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy77 = yylhsminor.yy77; + case 273: /* sma_func_name ::= COUNT */ yytestcase(yyruleno==273); + case 274: /* sma_func_name ::= FIRST */ yytestcase(yyruleno==274); + case 275: /* sma_func_name ::= LAST */ yytestcase(yyruleno==275); + case 276: /* sma_func_name ::= LAST_ROW */ yytestcase(yyruleno==276); + case 354: /* db_name ::= NK_ID */ yytestcase(yyruleno==354); + case 355: /* table_name ::= NK_ID */ yytestcase(yyruleno==355); + case 356: /* column_name ::= NK_ID */ yytestcase(yyruleno==356); + case 357: /* function_name ::= NK_ID */ yytestcase(yyruleno==357); + case 358: /* table_alias ::= NK_ID */ yytestcase(yyruleno==358); + case 359: /* column_alias ::= NK_ID */ yytestcase(yyruleno==359); + case 360: /* user_name ::= NK_ID */ yytestcase(yyruleno==360); + case 361: /* topic_name ::= NK_ID */ yytestcase(yyruleno==361); + case 362: /* stream_name ::= NK_ID */ yytestcase(yyruleno==362); + case 363: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==363); + case 364: /* index_name ::= NK_ID */ yytestcase(yyruleno==364); + case 404: /* noarg_func ::= NOW */ yytestcase(yyruleno==404); + case 405: /* noarg_func ::= TODAY */ yytestcase(yyruleno==405); + case 406: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==406); + case 407: /* noarg_func ::= DATABASE */ yytestcase(yyruleno==407); + case 408: /* noarg_func ::= CLIENT_VERSION */ yytestcase(yyruleno==408); + case 409: /* noarg_func ::= SERVER_VERSION */ yytestcase(yyruleno==409); + case 410: /* noarg_func ::= SERVER_STATUS */ yytestcase(yyruleno==410); + case 411: /* noarg_func ::= CURRENT_USER */ yytestcase(yyruleno==411); + case 412: /* noarg_func ::= USER */ yytestcase(yyruleno==412); + case 413: /* star_func ::= COUNT */ yytestcase(yyruleno==413); + case 414: /* star_func ::= FIRST */ yytestcase(yyruleno==414); + case 415: /* star_func ::= LAST */ yytestcase(yyruleno==415); + case 416: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==416); +{ yylhsminor.yy179 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy179 = yylhsminor.yy179; break; case 54: /* force_opt ::= */ case 73: /* not_exists_opt ::= */ yytestcase(yyruleno==73); case 75: /* exists_opt ::= */ yytestcase(yyruleno==75); - case 287: /* analyze_opt ::= */ yytestcase(yyruleno==287); - case 294: /* agg_func_opt ::= */ yytestcase(yyruleno==294); - case 469: /* set_quantifier_opt ::= */ yytestcase(yyruleno==469); -{ yymsp[1].minor.yy841 = false; } + case 292: /* analyze_opt ::= */ yytestcase(yyruleno==292); + case 299: /* agg_func_opt ::= */ yytestcase(yyruleno==299); + case 477: /* set_quantifier_opt ::= */ yytestcase(yyruleno==477); +{ yymsp[1].minor.yy767 = false; } break; case 55: /* force_opt ::= FORCE */ - case 288: /* analyze_opt ::= ANALYZE */ yytestcase(yyruleno==288); - case 295: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==295); - case 470: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==470); -{ yymsp[0].minor.yy841 = true; } + case 293: /* analyze_opt ::= ANALYZE */ yytestcase(yyruleno==293); + case 300: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==300); + case 478: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==478); +{ yymsp[0].minor.yy767 = true; } break; case 56: /* cmd ::= ALTER LOCAL NK_STRING */ { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[0].minor.yy0, NULL); } @@ -4499,206 +3875,206 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_MNODE_STMT, &yymsp[0].minor.yy0); } break; case 66: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ -{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy841, &yymsp[-1].minor.yy77, yymsp[0].minor.yy600); } +{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy767, &yymsp[-1].minor.yy179, yymsp[0].minor.yy602); } break; case 67: /* cmd ::= DROP DATABASE exists_opt db_name */ -{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy841, &yymsp[0].minor.yy77); } +{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy767, &yymsp[0].minor.yy179); } break; case 68: /* cmd ::= USE db_name */ -{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy77); } +{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy179); } break; case 69: /* cmd ::= ALTER DATABASE db_name alter_db_options */ -{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy77, yymsp[0].minor.yy600); } +{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy179, yymsp[0].minor.yy602); } break; case 70: /* cmd ::= FLUSH DATABASE db_name */ -{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy77); } +{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy179); } break; case 71: /* cmd ::= TRIM DATABASE db_name speed_opt */ -{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy77, yymsp[0].minor.yy248); } +{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy179, yymsp[0].minor.yy820); } break; case 72: /* not_exists_opt ::= IF NOT EXISTS */ -{ yymsp[-2].minor.yy841 = true; } +{ yymsp[-2].minor.yy767 = true; } break; case 74: /* exists_opt ::= IF EXISTS */ -{ yymsp[-1].minor.yy841 = true; } +{ yymsp[-1].minor.yy767 = true; } break; case 76: /* db_options ::= */ -{ yymsp[1].minor.yy600 = createDefaultDatabaseOptions(pCxt); } +{ yymsp[1].minor.yy602 = createDefaultDatabaseOptions(pCxt); } break; case 77: /* db_options ::= db_options BUFFER NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 78: /* db_options ::= db_options CACHEMODEL NK_STRING */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 79: /* db_options ::= db_options CACHESIZE NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 80: /* db_options ::= db_options COMP NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_COMP, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_COMP, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 81: /* db_options ::= db_options DURATION NK_INTEGER */ case 82: /* db_options ::= db_options DURATION NK_VARIABLE */ yytestcase(yyruleno==82); -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 83: /* db_options ::= db_options MAXROWS NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 84: /* db_options ::= db_options MINROWS NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 85: /* db_options ::= db_options KEEP integer_list */ case 86: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==86); -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_KEEP, yymsp[0].minor.yy601); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_KEEP, yymsp[0].minor.yy874); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 87: /* db_options ::= db_options PAGES NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 88: /* db_options ::= db_options PAGESIZE NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 89: /* db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 90: /* db_options ::= db_options PRECISION NK_STRING */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 91: /* db_options ::= db_options REPLICA NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 92: /* db_options ::= db_options VGROUPS NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 93: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 94: /* db_options ::= db_options RETENTIONS retention_list */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_RETENTIONS, yymsp[0].minor.yy601); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_RETENTIONS, yymsp[0].minor.yy874); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 95: /* db_options ::= db_options SCHEMALESS NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 96: /* db_options ::= db_options WAL_LEVEL NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_WAL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 97: /* db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 98: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 99: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-3].minor.yy600, DB_OPTION_WAL_RETENTION_PERIOD, &t); + yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-3].minor.yy602, DB_OPTION_WAL_RETENTION_PERIOD, &t); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; case 100: /* db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 101: /* db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-3].minor.yy600, DB_OPTION_WAL_RETENTION_SIZE, &t); + yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-3].minor.yy602, DB_OPTION_WAL_RETENTION_SIZE, &t); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; case 102: /* db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 103: /* db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 104: /* db_options ::= db_options STT_TRIGGER NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 105: /* db_options ::= db_options TABLE_PREFIX NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_TABLE_PREFIX, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_TABLE_PREFIX, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 106: /* db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_TABLE_SUFFIX, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setDatabaseOption(pCxt, yymsp[-2].minor.yy602, DB_OPTION_TABLE_SUFFIX, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 107: /* alter_db_options ::= alter_db_option */ -{ yylhsminor.yy600 = createAlterDatabaseOptions(pCxt); yylhsminor.yy600 = setAlterDatabaseOption(pCxt, yylhsminor.yy600, &yymsp[0].minor.yy661); } - yymsp[0].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = createAlterDatabaseOptions(pCxt); yylhsminor.yy602 = setAlterDatabaseOption(pCxt, yylhsminor.yy602, &yymsp[0].minor.yy845); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; case 108: /* alter_db_options ::= alter_db_options alter_db_option */ -{ yylhsminor.yy600 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy600, &yymsp[0].minor.yy661); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy602, &yymsp[0].minor.yy845); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; case 109: /* alter_db_option ::= BUFFER NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy845.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy845.val = yymsp[0].minor.yy0; } break; case 110: /* alter_db_option ::= CACHEMODEL NK_STRING */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy845.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy845.val = yymsp[0].minor.yy0; } break; case 111: /* alter_db_option ::= CACHESIZE NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy845.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy845.val = yymsp[0].minor.yy0; } break; case 112: /* alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy845.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy845.val = yymsp[0].minor.yy0; } break; case 113: /* alter_db_option ::= KEEP integer_list */ case 114: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==114); -{ yymsp[-1].minor.yy661.type = DB_OPTION_KEEP; yymsp[-1].minor.yy661.pList = yymsp[0].minor.yy601; } +{ yymsp[-1].minor.yy845.type = DB_OPTION_KEEP; yymsp[-1].minor.yy845.pList = yymsp[0].minor.yy874; } break; case 115: /* alter_db_option ::= PAGES NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_PAGES; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy845.type = DB_OPTION_PAGES; yymsp[-1].minor.yy845.val = yymsp[0].minor.yy0; } break; case 116: /* alter_db_option ::= REPLICA NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy845.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy845.val = yymsp[0].minor.yy0; } break; case 117: /* alter_db_option ::= WAL_LEVEL NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_WAL; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy845.type = DB_OPTION_WAL; yymsp[-1].minor.yy845.val = yymsp[0].minor.yy0; } break; case 118: /* alter_db_option ::= STT_TRIGGER NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy845.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy845.val = yymsp[0].minor.yy0; } break; case 119: /* integer_list ::= NK_INTEGER */ -{ yylhsminor.yy601 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy601 = yylhsminor.yy601; +{ yylhsminor.yy874 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy874 = yylhsminor.yy874; break; case 120: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ - case 317: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==317); -{ yylhsminor.yy601 = addNodeToList(pCxt, yymsp[-2].minor.yy601, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy601 = yylhsminor.yy601; + case 324: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==324); +{ yylhsminor.yy874 = addNodeToList(pCxt, yymsp[-2].minor.yy874, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy874 = yylhsminor.yy874; break; case 121: /* variable_list ::= NK_VARIABLE */ -{ yylhsminor.yy601 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy601 = yylhsminor.yy601; +{ yylhsminor.yy874 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy874 = yylhsminor.yy874; break; case 122: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ -{ yylhsminor.yy601 = addNodeToList(pCxt, yymsp[-2].minor.yy601, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy601 = yylhsminor.yy601; +{ yylhsminor.yy874 = addNodeToList(pCxt, yymsp[-2].minor.yy874, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy874 = yylhsminor.yy874; break; case 123: /* retention_list ::= retention */ case 145: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==145); @@ -4706,288 +4082,290 @@ static YYACTIONTYPE yy_reduce( case 155: /* column_def_list ::= column_def */ yytestcase(yyruleno==155); case 199: /* rollup_func_list ::= rollup_func_name */ yytestcase(yyruleno==199); case 204: /* col_name_list ::= col_name */ yytestcase(yyruleno==204); - case 253: /* tag_list_opt ::= tag_item */ yytestcase(yyruleno==253); - case 264: /* func_list ::= func */ yytestcase(yyruleno==264); - case 345: /* literal_list ::= signed_literal */ yytestcase(yyruleno==345); - case 411: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==411); - case 417: /* when_then_list ::= when_then_expr */ yytestcase(yyruleno==417); - case 472: /* select_list ::= select_item */ yytestcase(yyruleno==472); - case 483: /* partition_list ::= partition_item */ yytestcase(yyruleno==483); - case 536: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==536); -{ yylhsminor.yy601 = createNodeList(pCxt, yymsp[0].minor.yy600); } - yymsp[0].minor.yy601 = yylhsminor.yy601; + case 255: /* tag_list_opt ::= tag_item */ yytestcase(yyruleno==255); + case 269: /* func_list ::= func */ yytestcase(yyruleno==269); + case 352: /* literal_list ::= signed_literal */ yytestcase(yyruleno==352); + case 419: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==419); + case 425: /* when_then_list ::= when_then_expr */ yytestcase(yyruleno==425); + case 480: /* select_list ::= select_item */ yytestcase(yyruleno==480); + case 491: /* partition_list ::= partition_item */ yytestcase(yyruleno==491); + case 544: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==544); +{ yylhsminor.yy874 = createNodeList(pCxt, yymsp[0].minor.yy602); } + yymsp[0].minor.yy874 = yylhsminor.yy874; break; case 124: /* retention_list ::= retention_list NK_COMMA retention */ case 156: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==156); case 200: /* rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ yytestcase(yyruleno==200); case 205: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==205); - case 254: /* tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ yytestcase(yyruleno==254); - case 265: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==265); - case 346: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==346); - case 412: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==412); - case 473: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==473); - case 484: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==484); - case 537: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==537); -{ yylhsminor.yy601 = addNodeToList(pCxt, yymsp[-2].minor.yy601, yymsp[0].minor.yy600); } - yymsp[-2].minor.yy601 = yylhsminor.yy601; + case 256: /* tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ yytestcase(yyruleno==256); + case 270: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==270); + case 353: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==353); + case 420: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==420); + case 481: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==481); + case 492: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==492); + case 545: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==545); +{ yylhsminor.yy874 = addNodeToList(pCxt, yymsp[-2].minor.yy874, yymsp[0].minor.yy602); } + yymsp[-2].minor.yy874 = yylhsminor.yy874; break; case 125: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ -{ yylhsminor.yy600 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 126: /* speed_opt ::= */ - case 296: /* bufsize_opt ::= */ yytestcase(yyruleno==296); -{ yymsp[1].minor.yy248 = 0; } + case 301: /* bufsize_opt ::= */ yytestcase(yyruleno==301); +{ yymsp[1].minor.yy820 = 0; } break; case 127: /* speed_opt ::= MAX_SPEED NK_INTEGER */ - case 297: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==297); -{ yymsp[-1].minor.yy248 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } + case 302: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==302); +{ yymsp[-1].minor.yy820 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } break; case 128: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ case 130: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==130); -{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy841, yymsp[-5].minor.yy600, yymsp[-3].minor.yy601, yymsp[-1].minor.yy601, yymsp[0].minor.yy600); } +{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy767, yymsp[-5].minor.yy602, yymsp[-3].minor.yy874, yymsp[-1].minor.yy874, yymsp[0].minor.yy602); } break; case 129: /* cmd ::= CREATE TABLE multi_create_clause */ -{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy601); } +{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy874); } break; case 131: /* cmd ::= DROP TABLE multi_drop_clause */ -{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy601); } +{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy874); } break; case 132: /* cmd ::= DROP STABLE exists_opt full_table_name */ -{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy841, yymsp[0].minor.yy600); } +{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy767, yymsp[0].minor.yy602); } break; case 133: /* cmd ::= ALTER TABLE alter_table_clause */ - case 319: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==319); -{ pCxt->pRootNode = yymsp[0].minor.yy600; } + case 326: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==326); +{ pCxt->pRootNode = yymsp[0].minor.yy602; } break; case 134: /* cmd ::= ALTER STABLE alter_table_clause */ -{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy600); } +{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy602); } break; case 135: /* alter_table_clause ::= full_table_name alter_table_options */ -{ yylhsminor.yy600 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy602, yymsp[0].minor.yy602); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; case 136: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ -{ yylhsminor.yy600 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy77, yymsp[0].minor.yy888); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy602, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy179, yymsp[0].minor.yy394); } + yymsp[-4].minor.yy602 = yylhsminor.yy602; break; case 137: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ -{ yylhsminor.yy600 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy600, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy77); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy602, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy179); } + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; case 138: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ -{ yylhsminor.yy600 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy77, yymsp[0].minor.yy888); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy602, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy179, yymsp[0].minor.yy394); } + yymsp[-4].minor.yy602 = yylhsminor.yy602; break; case 139: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ -{ yylhsminor.yy600 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy77, &yymsp[0].minor.yy77); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy602, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy179, &yymsp[0].minor.yy179); } + yymsp[-4].minor.yy602 = yylhsminor.yy602; break; case 140: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ -{ yylhsminor.yy600 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy77, yymsp[0].minor.yy888); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy602, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy179, yymsp[0].minor.yy394); } + yymsp[-4].minor.yy602 = yylhsminor.yy602; break; case 141: /* alter_table_clause ::= full_table_name DROP TAG column_name */ -{ yylhsminor.yy600 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy600, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy77); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy602, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy179); } + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; case 142: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ -{ yylhsminor.yy600 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy77, yymsp[0].minor.yy888); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy602, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy179, yymsp[0].minor.yy394); } + yymsp[-4].minor.yy602 = yylhsminor.yy602; break; case 143: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ -{ yylhsminor.yy600 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy77, &yymsp[0].minor.yy77); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy602, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy179, &yymsp[0].minor.yy179); } + yymsp[-4].minor.yy602 = yylhsminor.yy602; break; case 144: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ -{ yylhsminor.yy600 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy600, &yymsp[-2].minor.yy77, yymsp[0].minor.yy600); } - yymsp[-5].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy602, &yymsp[-2].minor.yy179, yymsp[0].minor.yy602); } + yymsp[-5].minor.yy602 = yylhsminor.yy602; break; case 146: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ case 149: /* multi_drop_clause ::= multi_drop_clause drop_table_clause */ yytestcase(yyruleno==149); - case 418: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==418); -{ yylhsminor.yy601 = addNodeToList(pCxt, yymsp[-1].minor.yy601, yymsp[0].minor.yy600); } - yymsp[-1].minor.yy601 = yylhsminor.yy601; + case 426: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==426); +{ yylhsminor.yy874 = addNodeToList(pCxt, yymsp[-1].minor.yy874, yymsp[0].minor.yy602); } + yymsp[-1].minor.yy874 = yylhsminor.yy874; break; case 147: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ -{ yylhsminor.yy600 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy841, yymsp[-8].minor.yy600, yymsp[-6].minor.yy600, yymsp[-5].minor.yy601, yymsp[-2].minor.yy601, yymsp[0].minor.yy600); } - yymsp[-9].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy767, yymsp[-8].minor.yy602, yymsp[-6].minor.yy602, yymsp[-5].minor.yy874, yymsp[-2].minor.yy874, yymsp[0].minor.yy602); } + yymsp[-9].minor.yy602 = yylhsminor.yy602; break; case 150: /* drop_table_clause ::= exists_opt full_table_name */ -{ yylhsminor.yy600 = createDropTableClause(pCxt, yymsp[-1].minor.yy841, yymsp[0].minor.yy600); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = createDropTableClause(pCxt, yymsp[-1].minor.yy767, yymsp[0].minor.yy602); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; case 151: /* specific_cols_opt ::= */ case 182: /* tags_def_opt ::= */ yytestcase(yyruleno==182); - case 252: /* tag_list_opt ::= */ yytestcase(yyruleno==252); - case 481: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==481); - case 504: /* group_by_clause_opt ::= */ yytestcase(yyruleno==504); - case 523: /* order_by_clause_opt ::= */ yytestcase(yyruleno==523); -{ yymsp[1].minor.yy601 = NULL; } + case 254: /* tag_list_opt ::= */ yytestcase(yyruleno==254); + case 305: /* col_list_opt ::= */ yytestcase(yyruleno==305); + case 489: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==489); + case 512: /* group_by_clause_opt ::= */ yytestcase(yyruleno==512); + case 531: /* order_by_clause_opt ::= */ yytestcase(yyruleno==531); +{ yymsp[1].minor.yy874 = NULL; } break; case 152: /* specific_cols_opt ::= NK_LP col_name_list NK_RP */ -{ yymsp[-2].minor.yy601 = yymsp[-1].minor.yy601; } + case 306: /* col_list_opt ::= NK_LP col_name_list NK_RP */ yytestcase(yyruleno==306); +{ yymsp[-2].minor.yy874 = yymsp[-1].minor.yy874; } break; case 153: /* full_table_name ::= table_name */ -{ yylhsminor.yy600 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy77, NULL); } - yymsp[0].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy179, NULL); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; case 154: /* full_table_name ::= db_name NK_DOT table_name */ -{ yylhsminor.yy600 = createRealTableNode(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy77, NULL); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = createRealTableNode(pCxt, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy179, NULL); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 157: /* column_def ::= column_name type_name */ -{ yylhsminor.yy600 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy77, yymsp[0].minor.yy888, NULL); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy179, yymsp[0].minor.yy394, NULL); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; case 158: /* column_def ::= column_name type_name COMMENT NK_STRING */ -{ yylhsminor.yy600 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy77, yymsp[-2].minor.yy888, &yymsp[0].minor.yy0); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy179, yymsp[-2].minor.yy394, &yymsp[0].minor.yy0); } + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; case 159: /* type_name ::= BOOL */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_BOOL); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_BOOL); } break; case 160: /* type_name ::= TINYINT */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_TINYINT); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_TINYINT); } break; case 161: /* type_name ::= SMALLINT */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_SMALLINT); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_SMALLINT); } break; case 162: /* type_name ::= INT */ case 163: /* type_name ::= INTEGER */ yytestcase(yyruleno==163); -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_INT); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_INT); } break; case 164: /* type_name ::= BIGINT */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_BIGINT); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_BIGINT); } break; case 165: /* type_name ::= FLOAT */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_FLOAT); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_FLOAT); } break; case 166: /* type_name ::= DOUBLE */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_DOUBLE); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_DOUBLE); } break; case 167: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy888 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy394 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } break; case 168: /* type_name ::= TIMESTAMP */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } break; case 169: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy888 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy394 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } break; case 170: /* type_name ::= TINYINT UNSIGNED */ -{ yymsp[-1].minor.yy888 = createDataType(TSDB_DATA_TYPE_UTINYINT); } +{ yymsp[-1].minor.yy394 = createDataType(TSDB_DATA_TYPE_UTINYINT); } break; case 171: /* type_name ::= SMALLINT UNSIGNED */ -{ yymsp[-1].minor.yy888 = createDataType(TSDB_DATA_TYPE_USMALLINT); } +{ yymsp[-1].minor.yy394 = createDataType(TSDB_DATA_TYPE_USMALLINT); } break; case 172: /* type_name ::= INT UNSIGNED */ -{ yymsp[-1].minor.yy888 = createDataType(TSDB_DATA_TYPE_UINT); } +{ yymsp[-1].minor.yy394 = createDataType(TSDB_DATA_TYPE_UINT); } break; case 173: /* type_name ::= BIGINT UNSIGNED */ -{ yymsp[-1].minor.yy888 = createDataType(TSDB_DATA_TYPE_UBIGINT); } +{ yymsp[-1].minor.yy394 = createDataType(TSDB_DATA_TYPE_UBIGINT); } break; case 174: /* type_name ::= JSON */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_JSON); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_JSON); } break; case 175: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy888 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy394 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } break; case 176: /* type_name ::= MEDIUMBLOB */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } break; case 177: /* type_name ::= BLOB */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_BLOB); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_BLOB); } break; case 178: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy888 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy394 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } break; case 179: /* type_name ::= DECIMAL */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +{ yymsp[0].minor.yy394 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 180: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy888 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +{ yymsp[-3].minor.yy394 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 181: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ -{ yymsp[-5].minor.yy888 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +{ yymsp[-5].minor.yy394 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 183: /* tags_def_opt ::= tags_def */ - case 410: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==410); -{ yylhsminor.yy601 = yymsp[0].minor.yy601; } - yymsp[0].minor.yy601 = yylhsminor.yy601; + case 418: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==418); +{ yylhsminor.yy874 = yymsp[0].minor.yy874; } + yymsp[0].minor.yy874 = yylhsminor.yy874; break; case 184: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ -{ yymsp[-3].minor.yy601 = yymsp[-1].minor.yy601; } +{ yymsp[-3].minor.yy874 = yymsp[-1].minor.yy874; } break; case 185: /* table_options ::= */ -{ yymsp[1].minor.yy600 = createDefaultTableOptions(pCxt); } +{ yymsp[1].minor.yy602 = createDefaultTableOptions(pCxt); } break; case 186: /* table_options ::= table_options COMMENT NK_STRING */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setTableOption(pCxt, yymsp[-2].minor.yy602, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 187: /* table_options ::= table_options MAX_DELAY duration_list */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy601); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setTableOption(pCxt, yymsp[-2].minor.yy602, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy874); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 188: /* table_options ::= table_options WATERMARK duration_list */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy601); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setTableOption(pCxt, yymsp[-2].minor.yy602, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy874); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 189: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-4].minor.yy600, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy601); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setTableOption(pCxt, yymsp[-4].minor.yy602, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy874); } + yymsp[-4].minor.yy602 = yylhsminor.yy602; break; case 190: /* table_options ::= table_options TTL NK_INTEGER */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setTableOption(pCxt, yymsp[-2].minor.yy602, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 191: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-4].minor.yy600, TABLE_OPTION_SMA, yymsp[-1].minor.yy601); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setTableOption(pCxt, yymsp[-4].minor.yy602, TABLE_OPTION_SMA, yymsp[-1].minor.yy874); } + yymsp[-4].minor.yy602 = yylhsminor.yy602; break; case 192: /* table_options ::= table_options DELETE_MARK duration_list */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy601); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setTableOption(pCxt, yymsp[-2].minor.yy602, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy874); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; case 193: /* alter_table_options ::= alter_table_option */ -{ yylhsminor.yy600 = createAlterTableOptions(pCxt); yylhsminor.yy600 = setTableOption(pCxt, yylhsminor.yy600, yymsp[0].minor.yy661.type, &yymsp[0].minor.yy661.val); } - yymsp[0].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = createAlterTableOptions(pCxt); yylhsminor.yy602 = setTableOption(pCxt, yylhsminor.yy602, yymsp[0].minor.yy845.type, &yymsp[0].minor.yy845.val); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; case 194: /* alter_table_options ::= alter_table_options alter_table_option */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-1].minor.yy600, yymsp[0].minor.yy661.type, &yymsp[0].minor.yy661.val); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = setTableOption(pCxt, yymsp[-1].minor.yy602, yymsp[0].minor.yy845.type, &yymsp[0].minor.yy845.val); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; case 195: /* alter_table_option ::= COMMENT NK_STRING */ -{ yymsp[-1].minor.yy661.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy845.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy845.val = yymsp[0].minor.yy0; } break; case 196: /* alter_table_option ::= TTL NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy845.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy845.val = yymsp[0].minor.yy0; } break; case 197: /* duration_list ::= duration_literal */ - case 374: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==374); -{ yylhsminor.yy601 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); } - yymsp[0].minor.yy601 = yylhsminor.yy601; + case 382: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==382); +{ yylhsminor.yy874 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy602)); } + yymsp[0].minor.yy874 = yylhsminor.yy874; break; case 198: /* duration_list ::= duration_list NK_COMMA duration_literal */ - case 375: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==375); -{ yylhsminor.yy601 = addNodeToList(pCxt, yymsp[-2].minor.yy601, releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); } - yymsp[-2].minor.yy601 = yylhsminor.yy601; + case 383: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==383); +{ yylhsminor.yy874 = addNodeToList(pCxt, yymsp[-2].minor.yy874, releaseRawExprNode(pCxt, yymsp[0].minor.yy602)); } + yymsp[-2].minor.yy874 = yylhsminor.yy874; break; case 201: /* rollup_func_name ::= function_name */ -{ yylhsminor.yy600 = createFunctionNode(pCxt, &yymsp[0].minor.yy77, NULL); } - yymsp[0].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy602 = createFunctionNode(pCxt, &yymsp[0].minor.yy179, NULL); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; case 202: /* rollup_func_name ::= FIRST */ case 203: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==203); - case 256: /* tag_item ::= QTAGS */ yytestcase(yyruleno==256); -{ yylhsminor.yy600 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } - yymsp[0].minor.yy600 = yylhsminor.yy600; + case 258: /* tag_item ::= QTAGS */ yytestcase(yyruleno==258); +{ yylhsminor.yy602 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; case 206: /* col_name ::= column_name */ - case 257: /* tag_item ::= column_name */ yytestcase(yyruleno==257); -{ yylhsminor.yy600 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy77); } - yymsp[0].minor.yy600 = yylhsminor.yy600; + case 259: /* tag_item ::= column_name */ yytestcase(yyruleno==259); +{ yylhsminor.yy602 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy179); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; case 207: /* cmd ::= SHOW DNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT); } @@ -5002,13 +4380,13 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT); } break; case 211: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy600, yymsp[0].minor.yy600, OP_TYPE_LIKE); } +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy602, yymsp[0].minor.yy602, OP_TYPE_LIKE); } break; case 212: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy600, yymsp[0].minor.yy600, OP_TYPE_LIKE); } +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy602, yymsp[0].minor.yy602, OP_TYPE_LIKE); } break; case 213: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy600, NULL, OP_TYPE_LIKE); } +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy602, NULL, OP_TYPE_LIKE); } break; case 214: /* cmd ::= SHOW MNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT); } @@ -5020,7 +4398,7 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT); } break; case 217: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy600, yymsp[-1].minor.yy600, OP_TYPE_EQUAL); } +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy602, yymsp[-1].minor.yy602, OP_TYPE_EQUAL); } break; case 218: /* cmd ::= SHOW STREAMS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT); } @@ -5039,13 +4417,13 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCES_STMT); } break; case 224: /* cmd ::= SHOW CREATE DATABASE db_name */ -{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy77); } +{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy179); } break; case 225: /* cmd ::= SHOW CREATE TABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy600); } +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy602); } break; case 226: /* cmd ::= SHOW CREATE STABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy600); } +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy602); } break; case 227: /* cmd ::= SHOW QUERIES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT); } @@ -5064,7 +4442,7 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT); } break; case 233: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy600); } +{ pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy602); } break; case 234: /* cmd ::= SHOW BNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_BNODES_STMT); } @@ -5079,7 +4457,7 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TRANSACTIONS_STMT); } break; case 238: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ -{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy600); } +{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy602); } break; case 239: /* cmd ::= SHOW CONSUMERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONSUMERS_STMT); } @@ -5088,10 +4466,10 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT); } break; case 241: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy600, yymsp[-1].minor.yy600, OP_TYPE_EQUAL); } +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy602, yymsp[-1].minor.yy602, OP_TYPE_EQUAL); } break; case 242: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy600, yymsp[0].minor.yy600, yymsp[-3].minor.yy601); } +{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy602, yymsp[0].minor.yy602, yymsp[-3].minor.yy874); } break; case 243: /* cmd ::= SHOW VNODES NK_INTEGER */ { pCxt->pRootNode = createShowVnodesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0), NULL); } @@ -5099,747 +4477,764 @@ static YYACTIONTYPE yy_reduce( case 244: /* cmd ::= SHOW VNODES NK_STRING */ { pCxt->pRootNode = createShowVnodesStmt(pCxt, NULL, createValueNode(pCxt, TSDB_DATA_TYPE_VARCHAR, &yymsp[0].minor.yy0)); } break; - case 245: /* db_name_cond_opt ::= */ - case 250: /* from_db_opt ::= */ yytestcase(yyruleno==250); -{ yymsp[1].minor.yy600 = createDefaultDatabaseCondValue(pCxt); } + case 245: /* cmd ::= SHOW db_name_cond_opt ALIVE */ +{ pCxt->pRootNode = createShowAliveStmt(pCxt, yymsp[-1].minor.yy602, QUERY_NODE_SHOW_DB_ALIVE_STMT); } + break; + case 246: /* cmd ::= SHOW CLUSTER ALIVE */ +{ pCxt->pRootNode = createShowAliveStmt(pCxt, NULL, QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT); } + break; + case 247: /* db_name_cond_opt ::= */ + case 252: /* from_db_opt ::= */ yytestcase(yyruleno==252); +{ yymsp[1].minor.yy602 = createDefaultDatabaseCondValue(pCxt); } + break; + case 248: /* db_name_cond_opt ::= db_name NK_DOT */ +{ yylhsminor.yy602 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy179); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; - case 246: /* db_name_cond_opt ::= db_name NK_DOT */ -{ yylhsminor.yy600 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy77); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + case 249: /* like_pattern_opt ::= */ + case 314: /* subtable_opt ::= */ yytestcase(yyruleno==314); + case 428: /* case_when_else_opt ::= */ yytestcase(yyruleno==428); + case 458: /* from_clause_opt ::= */ yytestcase(yyruleno==458); + case 487: /* where_clause_opt ::= */ yytestcase(yyruleno==487); + case 496: /* twindow_clause_opt ::= */ yytestcase(yyruleno==496); + case 502: /* sliding_opt ::= */ yytestcase(yyruleno==502); + case 504: /* fill_opt ::= */ yytestcase(yyruleno==504); + case 516: /* having_clause_opt ::= */ yytestcase(yyruleno==516); + case 518: /* range_opt ::= */ yytestcase(yyruleno==518); + case 520: /* every_opt ::= */ yytestcase(yyruleno==520); + case 533: /* slimit_clause_opt ::= */ yytestcase(yyruleno==533); + case 537: /* limit_clause_opt ::= */ yytestcase(yyruleno==537); +{ yymsp[1].minor.yy602 = NULL; } break; - case 247: /* like_pattern_opt ::= */ - case 307: /* subtable_opt ::= */ yytestcase(yyruleno==307); - case 420: /* case_when_else_opt ::= */ yytestcase(yyruleno==420); - case 450: /* from_clause_opt ::= */ yytestcase(yyruleno==450); - case 479: /* where_clause_opt ::= */ yytestcase(yyruleno==479); - case 488: /* twindow_clause_opt ::= */ yytestcase(yyruleno==488); - case 494: /* sliding_opt ::= */ yytestcase(yyruleno==494); - case 496: /* fill_opt ::= */ yytestcase(yyruleno==496); - case 508: /* having_clause_opt ::= */ yytestcase(yyruleno==508); - case 510: /* range_opt ::= */ yytestcase(yyruleno==510); - case 512: /* every_opt ::= */ yytestcase(yyruleno==512); - case 525: /* slimit_clause_opt ::= */ yytestcase(yyruleno==525); - case 529: /* limit_clause_opt ::= */ yytestcase(yyruleno==529); -{ yymsp[1].minor.yy600 = NULL; } + case 250: /* like_pattern_opt ::= LIKE NK_STRING */ +{ yymsp[-1].minor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } break; - case 248: /* like_pattern_opt ::= LIKE NK_STRING */ -{ yymsp[-1].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + case 251: /* table_name_cond ::= table_name */ +{ yylhsminor.yy602 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy179); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; - case 249: /* table_name_cond ::= table_name */ -{ yylhsminor.yy600 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy77); } - yymsp[0].minor.yy600 = yylhsminor.yy600; + case 253: /* from_db_opt ::= FROM db_name */ +{ yymsp[-1].minor.yy602 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy179); } break; - case 251: /* from_db_opt ::= FROM db_name */ -{ yymsp[-1].minor.yy600 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy77); } + case 257: /* tag_item ::= TBNAME */ +{ yylhsminor.yy602 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; - case 255: /* tag_item ::= TBNAME */ -{ yylhsminor.yy600 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; + case 260: /* tag_item ::= column_name column_alias */ +{ yylhsminor.yy602 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy179), &yymsp[0].minor.yy179); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; - case 258: /* tag_item ::= column_name column_alias */ -{ yylhsminor.yy600 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy77), &yymsp[0].minor.yy77); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + case 261: /* tag_item ::= column_name AS column_alias */ +{ yylhsminor.yy602 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy179), &yymsp[0].minor.yy179); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 259: /* tag_item ::= column_name AS column_alias */ -{ yylhsminor.yy600 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy77), &yymsp[0].minor.yy77); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 262: /* cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy767, yymsp[-3].minor.yy602, yymsp[-1].minor.yy602, NULL, yymsp[0].minor.yy602); } break; - case 260: /* cmd ::= CREATE SMA INDEX not_exists_opt full_table_name ON full_table_name index_options */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy841, yymsp[-3].minor.yy600, yymsp[-1].minor.yy600, NULL, yymsp[0].minor.yy600); } + case 263: /* cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_NORMAL, yymsp[-6].minor.yy767, yymsp[-5].minor.yy602, yymsp[-3].minor.yy602, yymsp[-1].minor.yy874, NULL); } break; - case 261: /* cmd ::= DROP INDEX exists_opt full_table_name */ -{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy841, yymsp[0].minor.yy600); } + case 264: /* cmd ::= DROP INDEX exists_opt full_index_name */ +{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy767, yymsp[0].minor.yy602); } break; - case 262: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ -{ yymsp[-9].minor.yy600 = createIndexOption(pCxt, yymsp[-7].minor.yy601, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), NULL, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } + case 265: /* full_index_name ::= index_name */ +{ yylhsminor.yy602 = createRealTableNodeForIndexName(pCxt, NULL, &yymsp[0].minor.yy179); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; - case 263: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ -{ yymsp[-11].minor.yy600 = createIndexOption(pCxt, yymsp[-9].minor.yy601, releaseRawExprNode(pCxt, yymsp[-5].minor.yy600), releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } + case 266: /* full_index_name ::= db_name NK_DOT index_name */ +{ yylhsminor.yy602 = createRealTableNodeForIndexName(pCxt, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy179); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 266: /* func ::= sma_func_name NK_LP expression_list NK_RP */ -{ yylhsminor.yy600 = createFunctionNode(pCxt, &yymsp[-3].minor.yy77, yymsp[-1].minor.yy601); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + case 267: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ +{ yymsp[-9].minor.yy602 = createIndexOption(pCxt, yymsp[-7].minor.yy874, releaseRawExprNode(pCxt, yymsp[-3].minor.yy602), NULL, yymsp[-1].minor.yy602, yymsp[0].minor.yy602); } break; - case 272: /* sma_stream_opt ::= */ - case 300: /* stream_options ::= */ yytestcase(yyruleno==300); -{ yymsp[1].minor.yy600 = createStreamOptions(pCxt); } + case 268: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ +{ yymsp[-11].minor.yy602 = createIndexOption(pCxt, yymsp[-9].minor.yy874, releaseRawExprNode(pCxt, yymsp[-5].minor.yy602), releaseRawExprNode(pCxt, yymsp[-3].minor.yy602), yymsp[-1].minor.yy602, yymsp[0].minor.yy602); } break; - case 273: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ - case 304: /* stream_options ::= stream_options WATERMARK duration_literal */ yytestcase(yyruleno==304); -{ ((SStreamOptions*)yymsp[-2].minor.yy600)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = yymsp[-2].minor.yy600; } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 271: /* func ::= sma_func_name NK_LP expression_list NK_RP */ +{ yylhsminor.yy602 = createFunctionNode(pCxt, &yymsp[-3].minor.yy179, yymsp[-1].minor.yy874); } + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; - case 274: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy600)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = yymsp[-2].minor.yy600; } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 277: /* sma_stream_opt ::= */ + case 307: /* stream_options ::= */ yytestcase(yyruleno==307); +{ yymsp[1].minor.yy602 = createStreamOptions(pCxt); } break; - case 275: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy600)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = yymsp[-2].minor.yy600; } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 278: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ + case 311: /* stream_options ::= stream_options WATERMARK duration_literal */ yytestcase(yyruleno==311); +{ ((SStreamOptions*)yymsp[-2].minor.yy602)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy602); yylhsminor.yy602 = yymsp[-2].minor.yy602; } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 276: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ -{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy841, &yymsp[-2].minor.yy77, yymsp[0].minor.yy600); } + case 279: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy602)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy602); yylhsminor.yy602 = yymsp[-2].minor.yy602; } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 277: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy841, &yymsp[-3].minor.yy77, &yymsp[0].minor.yy77, false); } + case 280: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy602)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy602); yylhsminor.yy602 = yymsp[-2].minor.yy602; } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 278: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy841, &yymsp[-5].minor.yy77, &yymsp[0].minor.yy77, true); } + case 281: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ +{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy767, &yymsp[-2].minor.yy179, yymsp[0].minor.yy602); } break; - case 279: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-4].minor.yy841, &yymsp[-3].minor.yy77, yymsp[0].minor.yy600, false); } + case 282: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy767, &yymsp[-3].minor.yy179, &yymsp[0].minor.yy179, false); } break; - case 280: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-6].minor.yy841, &yymsp[-5].minor.yy77, yymsp[0].minor.yy600, true); } + case 283: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy767, &yymsp[-5].minor.yy179, &yymsp[0].minor.yy179, true); } break; - case 281: /* cmd ::= DROP TOPIC exists_opt topic_name */ -{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy841, &yymsp[0].minor.yy77); } + case 284: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-4].minor.yy767, &yymsp[-3].minor.yy179, yymsp[0].minor.yy602, false); } break; - case 282: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ -{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy841, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy77); } + case 285: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-6].minor.yy767, &yymsp[-5].minor.yy179, yymsp[0].minor.yy602, true); } break; - case 283: /* cmd ::= DESC full_table_name */ - case 284: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==284); -{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy600); } + case 286: /* cmd ::= DROP TOPIC exists_opt topic_name */ +{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy767, &yymsp[0].minor.yy179); } break; - case 285: /* cmd ::= RESET QUERY CACHE */ + case 287: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ +{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy767, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy179); } + break; + case 288: /* cmd ::= DESC full_table_name */ + case 289: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==289); +{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy602); } + break; + case 290: /* cmd ::= RESET QUERY CACHE */ { pCxt->pRootNode = createResetQueryCacheStmt(pCxt); } break; - case 286: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ -{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy841, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } + case 291: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ +{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy767, yymsp[-1].minor.yy602, yymsp[0].minor.yy602); } break; - case 289: /* explain_options ::= */ -{ yymsp[1].minor.yy600 = createDefaultExplainOptions(pCxt); } + case 294: /* explain_options ::= */ +{ yymsp[1].minor.yy602 = createDefaultExplainOptions(pCxt); } break; - case 290: /* explain_options ::= explain_options VERBOSE NK_BOOL */ -{ yylhsminor.yy600 = setExplainVerbose(pCxt, yymsp[-2].minor.yy600, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 295: /* explain_options ::= explain_options VERBOSE NK_BOOL */ +{ yylhsminor.yy602 = setExplainVerbose(pCxt, yymsp[-2].minor.yy602, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 291: /* explain_options ::= explain_options RATIO NK_FLOAT */ -{ yylhsminor.yy600 = setExplainRatio(pCxt, yymsp[-2].minor.yy600, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 296: /* explain_options ::= explain_options RATIO NK_FLOAT */ +{ yylhsminor.yy602 = setExplainRatio(pCxt, yymsp[-2].minor.yy602, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 292: /* cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ -{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy841, yymsp[-8].minor.yy841, &yymsp[-5].minor.yy77, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy888, yymsp[0].minor.yy248); } + case 297: /* cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ +{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy767, yymsp[-8].minor.yy767, &yymsp[-5].minor.yy179, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy394, yymsp[0].minor.yy820); } break; - case 293: /* cmd ::= DROP FUNCTION exists_opt function_name */ -{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy841, &yymsp[0].minor.yy77); } + case 298: /* cmd ::= DROP FUNCTION exists_opt function_name */ +{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy767, &yymsp[0].minor.yy179); } break; - case 298: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name tags_def_opt subtable_opt AS query_or_subquery */ -{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-8].minor.yy841, &yymsp[-7].minor.yy77, yymsp[-4].minor.yy600, yymsp[-6].minor.yy600, yymsp[-3].minor.yy601, yymsp[-2].minor.yy600, yymsp[0].minor.yy600); } + case 303: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tags_def_opt subtable_opt AS query_or_subquery */ +{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-9].minor.yy767, &yymsp[-8].minor.yy179, yymsp[-5].minor.yy602, yymsp[-7].minor.yy602, yymsp[-3].minor.yy874, yymsp[-2].minor.yy602, yymsp[0].minor.yy602, yymsp[-4].minor.yy874); } break; - case 299: /* cmd ::= DROP STREAM exists_opt stream_name */ -{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy841, &yymsp[0].minor.yy77); } + case 304: /* cmd ::= DROP STREAM exists_opt stream_name */ +{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy767, &yymsp[0].minor.yy179); } break; - case 301: /* stream_options ::= stream_options TRIGGER AT_ONCE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy600)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy600 = yymsp[-2].minor.yy600; } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 308: /* stream_options ::= stream_options TRIGGER AT_ONCE */ +{ ((SStreamOptions*)yymsp[-2].minor.yy602)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy602 = yymsp[-2].minor.yy602; } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 302: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy600)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy600 = yymsp[-2].minor.yy600; } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 309: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ +{ ((SStreamOptions*)yymsp[-2].minor.yy602)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy602 = yymsp[-2].minor.yy602; } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 303: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ -{ ((SStreamOptions*)yymsp[-3].minor.yy600)->triggerType = STREAM_TRIGGER_MAX_DELAY; ((SStreamOptions*)yymsp[-3].minor.yy600)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = yymsp[-3].minor.yy600; } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + case 310: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ +{ ((SStreamOptions*)yymsp[-3].minor.yy602)->triggerType = STREAM_TRIGGER_MAX_DELAY; ((SStreamOptions*)yymsp[-3].minor.yy602)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy602); yylhsminor.yy602 = yymsp[-3].minor.yy602; } + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; - case 305: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ -{ ((SStreamOptions*)yymsp[-3].minor.yy600)->ignoreExpired = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy600 = yymsp[-3].minor.yy600; } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + case 312: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ +{ ((SStreamOptions*)yymsp[-3].minor.yy602)->ignoreExpired = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy602 = yymsp[-3].minor.yy602; } + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; - case 306: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ -{ ((SStreamOptions*)yymsp[-2].minor.yy600)->fillHistory = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy600 = yymsp[-2].minor.yy600; } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 313: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ +{ ((SStreamOptions*)yymsp[-2].minor.yy602)->fillHistory = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy602 = yymsp[-2].minor.yy602; } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 308: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ - case 495: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==495); - case 513: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==513); -{ yymsp[-3].minor.yy600 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy600); } + case 315: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ + case 503: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==503); + case 521: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==521); +{ yymsp[-3].minor.yy602 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy602); } break; - case 309: /* cmd ::= KILL CONNECTION NK_INTEGER */ + case 316: /* cmd ::= KILL CONNECTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &yymsp[0].minor.yy0); } break; - case 310: /* cmd ::= KILL QUERY NK_STRING */ + case 317: /* cmd ::= KILL QUERY NK_STRING */ { pCxt->pRootNode = createKillQueryStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 311: /* cmd ::= KILL TRANSACTION NK_INTEGER */ + case 318: /* cmd ::= KILL TRANSACTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_TRANSACTION_STMT, &yymsp[0].minor.yy0); } break; - case 312: /* cmd ::= BALANCE VGROUP */ + case 319: /* cmd ::= BALANCE VGROUP */ { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); } break; - case 313: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + case 320: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 314: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ -{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy601); } + case 321: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ +{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy874); } break; - case 315: /* cmd ::= SPLIT VGROUP NK_INTEGER */ + case 322: /* cmd ::= SPLIT VGROUP NK_INTEGER */ { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 316: /* dnode_list ::= DNODE NK_INTEGER */ -{ yymsp[-1].minor.yy601 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - break; - case 318: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ -{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } - break; - case 320: /* cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ -{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-4].minor.yy600, yymsp[-2].minor.yy601, yymsp[0].minor.yy600); } - break; - case 321: /* cmd ::= INSERT INTO full_table_name query_or_subquery */ -{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-1].minor.yy600, NULL, yymsp[0].minor.yy600); } - break; - case 322: /* literal ::= NK_INTEGER */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 323: /* literal ::= NK_FLOAT */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 324: /* literal ::= NK_STRING */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 325: /* literal ::= NK_BOOL */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 326: /* literal ::= TIMESTAMP NK_STRING */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; - break; - case 327: /* literal ::= duration_literal */ - case 337: /* signed_literal ::= signed */ yytestcase(yyruleno==337); - case 357: /* expr_or_subquery ::= expression */ yytestcase(yyruleno==357); - case 358: /* expression ::= literal */ yytestcase(yyruleno==358); - case 359: /* expression ::= pseudo_column */ yytestcase(yyruleno==359); - case 360: /* expression ::= column_reference */ yytestcase(yyruleno==360); - case 361: /* expression ::= function_expression */ yytestcase(yyruleno==361); - case 362: /* expression ::= case_when_expression */ yytestcase(yyruleno==362); - case 393: /* function_expression ::= literal_func */ yytestcase(yyruleno==393); - case 442: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==442); - case 446: /* boolean_primary ::= predicate */ yytestcase(yyruleno==446); - case 448: /* common_expression ::= expr_or_subquery */ yytestcase(yyruleno==448); - case 449: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==449); - case 452: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==452); - case 454: /* table_reference ::= table_primary */ yytestcase(yyruleno==454); - case 455: /* table_reference ::= joined_table */ yytestcase(yyruleno==455); - case 459: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==459); - case 515: /* query_simple ::= query_specification */ yytestcase(yyruleno==515); - case 516: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==516); - case 519: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==519); - case 521: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==521); -{ yylhsminor.yy600 = yymsp[0].minor.yy600; } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 328: /* literal ::= NULL */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 329: /* literal ::= NK_QUESTION */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 330: /* duration_literal ::= NK_VARIABLE */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 331: /* signed ::= NK_INTEGER */ -{ yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 332: /* signed ::= NK_PLUS NK_INTEGER */ -{ yymsp[-1].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } - break; - case 333: /* signed ::= NK_MINUS NK_INTEGER */ + case 323: /* dnode_list ::= DNODE NK_INTEGER */ +{ yymsp[-1].minor.yy874 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + break; + case 325: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ +{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy602, yymsp[0].minor.yy602); } + break; + case 327: /* cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ +{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-4].minor.yy602, yymsp[-2].minor.yy874, yymsp[0].minor.yy602); } + break; + case 328: /* cmd ::= INSERT INTO full_table_name query_or_subquery */ +{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-1].minor.yy602, NULL, yymsp[0].minor.yy602); } + break; + case 329: /* literal ::= NK_INTEGER */ +{ yylhsminor.yy602 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 330: /* literal ::= NK_FLOAT */ +{ yylhsminor.yy602 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 331: /* literal ::= NK_STRING */ +{ yylhsminor.yy602 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 332: /* literal ::= NK_BOOL */ +{ yylhsminor.yy602 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 333: /* literal ::= TIMESTAMP NK_STRING */ +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; + break; + case 334: /* literal ::= duration_literal */ + case 344: /* signed_literal ::= signed */ yytestcase(yyruleno==344); + case 365: /* expr_or_subquery ::= expression */ yytestcase(yyruleno==365); + case 366: /* expression ::= literal */ yytestcase(yyruleno==366); + case 367: /* expression ::= pseudo_column */ yytestcase(yyruleno==367); + case 368: /* expression ::= column_reference */ yytestcase(yyruleno==368); + case 369: /* expression ::= function_expression */ yytestcase(yyruleno==369); + case 370: /* expression ::= case_when_expression */ yytestcase(yyruleno==370); + case 401: /* function_expression ::= literal_func */ yytestcase(yyruleno==401); + case 450: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==450); + case 454: /* boolean_primary ::= predicate */ yytestcase(yyruleno==454); + case 456: /* common_expression ::= expr_or_subquery */ yytestcase(yyruleno==456); + case 457: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==457); + case 460: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==460); + case 462: /* table_reference ::= table_primary */ yytestcase(yyruleno==462); + case 463: /* table_reference ::= joined_table */ yytestcase(yyruleno==463); + case 467: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==467); + case 523: /* query_simple ::= query_specification */ yytestcase(yyruleno==523); + case 524: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==524); + case 527: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==527); + case 529: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==529); +{ yylhsminor.yy602 = yymsp[0].minor.yy602; } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 335: /* literal ::= NULL */ +{ yylhsminor.yy602 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 336: /* literal ::= NK_QUESTION */ +{ yylhsminor.yy602 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 337: /* duration_literal ::= NK_VARIABLE */ +{ yylhsminor.yy602 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 338: /* signed ::= NK_INTEGER */ +{ yylhsminor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 339: /* signed ::= NK_PLUS NK_INTEGER */ +{ yymsp[-1].minor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } + break; + case 340: /* signed ::= NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); + yylhsminor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; - case 334: /* signed ::= NK_FLOAT */ -{ yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; + case 341: /* signed ::= NK_FLOAT */ +{ yylhsminor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; - case 335: /* signed ::= NK_PLUS NK_FLOAT */ -{ yymsp[-1].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + case 342: /* signed ::= NK_PLUS NK_FLOAT */ +{ yymsp[-1].minor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } break; - case 336: /* signed ::= NK_MINUS NK_FLOAT */ + case 343: /* signed ::= NK_MINUS NK_FLOAT */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); + yylhsminor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; - break; - case 338: /* signed_literal ::= NK_STRING */ -{ yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 339: /* signed_literal ::= NK_BOOL */ -{ yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 340: /* signed_literal ::= TIMESTAMP NK_STRING */ -{ yymsp[-1].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } - break; - case 341: /* signed_literal ::= duration_literal */ - case 343: /* signed_literal ::= literal_func */ yytestcase(yyruleno==343); - case 413: /* star_func_para ::= expr_or_subquery */ yytestcase(yyruleno==413); - case 475: /* select_item ::= common_expression */ yytestcase(yyruleno==475); - case 485: /* partition_item ::= expr_or_subquery */ yytestcase(yyruleno==485); - case 520: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==520); - case 522: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==522); - case 535: /* search_condition ::= common_expression */ yytestcase(yyruleno==535); -{ yylhsminor.yy600 = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 342: /* signed_literal ::= NULL */ -{ yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 344: /* signed_literal ::= NK_QUESTION */ -{ yylhsminor.yy600 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 363: /* expression ::= NK_LP expression NK_RP */ - case 447: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==447); - case 534: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==534); -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 364: /* expression ::= NK_PLUS expr_or_subquery */ + yymsp[-1].minor.yy602 = yylhsminor.yy602; + break; + case 345: /* signed_literal ::= NK_STRING */ +{ yylhsminor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 346: /* signed_literal ::= NK_BOOL */ +{ yylhsminor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 347: /* signed_literal ::= TIMESTAMP NK_STRING */ +{ yymsp[-1].minor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } + break; + case 348: /* signed_literal ::= duration_literal */ + case 350: /* signed_literal ::= literal_func */ yytestcase(yyruleno==350); + case 421: /* star_func_para ::= expr_or_subquery */ yytestcase(yyruleno==421); + case 483: /* select_item ::= common_expression */ yytestcase(yyruleno==483); + case 493: /* partition_item ::= expr_or_subquery */ yytestcase(yyruleno==493); + case 528: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==528); + case 530: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==530); + case 543: /* search_condition ::= common_expression */ yytestcase(yyruleno==543); +{ yylhsminor.yy602 = releaseRawExprNode(pCxt, yymsp[0].minor.yy602); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 349: /* signed_literal ::= NULL */ +{ yylhsminor.yy602 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 351: /* signed_literal ::= NK_QUESTION */ +{ yylhsminor.yy602 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 371: /* expression ::= NK_LP expression NK_RP */ + case 455: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==455); + case 542: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==542); +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy602)); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; + break; + case 372: /* expression ::= NK_PLUS expr_or_subquery */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy602)); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; - case 365: /* expression ::= NK_MINUS expr_or_subquery */ + case 373: /* expression ::= NK_MINUS expr_or_subquery */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy600), NULL)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy602), NULL)); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; - case 366: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ + case 374: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 367: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ + case 375: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 368: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ + case 376: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 369: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ + case 377: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 370: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ + case 378: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 371: /* expression ::= column_reference NK_ARROW NK_STRING */ + case 379: /* expression ::= column_reference NK_ARROW NK_STRING */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 372: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ + case 380: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 373: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ + case 381: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 376: /* column_reference ::= column_name */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy77, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy77)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 377: /* column_reference ::= table_name NK_DOT column_name */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy77, createColumnNode(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy77)); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 378: /* pseudo_column ::= ROWTS */ - case 379: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==379); - case 381: /* pseudo_column ::= QSTART */ yytestcase(yyruleno==381); - case 382: /* pseudo_column ::= QEND */ yytestcase(yyruleno==382); - case 383: /* pseudo_column ::= QDURATION */ yytestcase(yyruleno==383); - case 384: /* pseudo_column ::= WSTART */ yytestcase(yyruleno==384); - case 385: /* pseudo_column ::= WEND */ yytestcase(yyruleno==385); - case 386: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==386); - case 387: /* pseudo_column ::= IROWTS */ yytestcase(yyruleno==387); - case 388: /* pseudo_column ::= ISFILLED */ yytestcase(yyruleno==388); - case 389: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==389); - case 395: /* literal_func ::= NOW */ yytestcase(yyruleno==395); -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 380: /* pseudo_column ::= table_name NK_DOT TBNAME */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy77)))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 390: /* function_expression ::= function_name NK_LP expression_list NK_RP */ - case 391: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==391); -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy77, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy77, yymsp[-1].minor.yy601)); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; - break; - case 392: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), yymsp[-1].minor.yy888)); } - yymsp[-5].minor.yy600 = yylhsminor.yy600; - break; - case 394: /* literal_func ::= noarg_func NK_LP NK_RP */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy77, NULL)); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 409: /* star_func_para_list ::= NK_STAR */ -{ yylhsminor.yy601 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy601 = yylhsminor.yy601; - break; - case 414: /* star_func_para ::= table_name NK_DOT NK_STAR */ - case 478: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==478); -{ yylhsminor.yy600 = createColumnNode(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 415: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy601, yymsp[-1].minor.yy600)); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; - break; - case 416: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), yymsp[-2].minor.yy601, yymsp[-1].minor.yy600)); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; - break; - case 419: /* when_then_expr ::= WHEN common_expression THEN common_expression */ -{ yymsp[-3].minor.yy600 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); } - break; - case 421: /* case_when_else_opt ::= ELSE common_expression */ -{ yymsp[-1].minor.yy600 = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); } - break; - case 422: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ - case 427: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==427); + yymsp[-2].minor.yy602 = yylhsminor.yy602; + break; + case 384: /* column_reference ::= column_name */ +{ yylhsminor.yy602 = createRawExprNode(pCxt, &yymsp[0].minor.yy179, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy179)); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 385: /* column_reference ::= table_name NK_DOT column_name */ +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy179, createColumnNode(pCxt, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy179)); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; + break; + case 386: /* pseudo_column ::= ROWTS */ + case 387: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==387); + case 389: /* pseudo_column ::= QSTART */ yytestcase(yyruleno==389); + case 390: /* pseudo_column ::= QEND */ yytestcase(yyruleno==390); + case 391: /* pseudo_column ::= QDURATION */ yytestcase(yyruleno==391); + case 392: /* pseudo_column ::= WSTART */ yytestcase(yyruleno==392); + case 393: /* pseudo_column ::= WEND */ yytestcase(yyruleno==393); + case 394: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==394); + case 395: /* pseudo_column ::= IROWTS */ yytestcase(yyruleno==395); + case 396: /* pseudo_column ::= ISFILLED */ yytestcase(yyruleno==396); + case 397: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==397); + case 403: /* literal_func ::= NOW */ yytestcase(yyruleno==403); +{ yylhsminor.yy602 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } + yymsp[0].minor.yy602 = yylhsminor.yy602; + break; + case 388: /* pseudo_column ::= table_name NK_DOT TBNAME */ +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy179)))); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; + break; + case 398: /* function_expression ::= function_name NK_LP expression_list NK_RP */ + case 399: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==399); +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy179, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy179, yymsp[-1].minor.yy874)); } + yymsp[-3].minor.yy602 = yylhsminor.yy602; + break; + case 400: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy602), yymsp[-1].minor.yy394)); } + yymsp[-5].minor.yy602 = yylhsminor.yy602; + break; + case 402: /* literal_func ::= noarg_func NK_LP NK_RP */ +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy179, NULL)); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; + break; + case 417: /* star_func_para_list ::= NK_STAR */ +{ yylhsminor.yy874 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy874 = yylhsminor.yy874; + break; + case 422: /* star_func_para ::= table_name NK_DOT NK_STAR */ + case 486: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==486); +{ yylhsminor.yy602 = createColumnNode(pCxt, &yymsp[-2].minor.yy179, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; + break; + case 423: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy874, yymsp[-1].minor.yy602)); } + yymsp[-3].minor.yy602 = yylhsminor.yy602; + break; + case 424: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy602), yymsp[-2].minor.yy874, yymsp[-1].minor.yy602)); } + yymsp[-4].minor.yy602 = yylhsminor.yy602; + break; + case 427: /* when_then_expr ::= WHEN common_expression THEN common_expression */ +{ yymsp[-3].minor.yy602 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602)); } + break; + case 429: /* case_when_else_opt ::= ELSE common_expression */ +{ yymsp[-1].minor.yy602 = releaseRawExprNode(pCxt, yymsp[0].minor.yy602); } + break; + case 430: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ + case 435: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==435); { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy666, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy290, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 423: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ + case 431: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy600), releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy602), releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; + yymsp[-4].minor.yy602 = yylhsminor.yy602; break; - case 424: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ + case 432: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy600), releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy602), releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-5].minor.yy600 = yylhsminor.yy600; + yymsp[-5].minor.yy602 = yylhsminor.yy602; break; - case 425: /* predicate ::= expr_or_subquery IS NULL */ + case 433: /* predicate ::= expr_or_subquery IS NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), NULL)); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 426: /* predicate ::= expr_or_subquery IS NOT NULL */ + case 434: /* predicate ::= expr_or_subquery IS NOT NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy602), NULL)); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; - case 428: /* compare_op ::= NK_LT */ -{ yymsp[0].minor.yy666 = OP_TYPE_LOWER_THAN; } + case 436: /* compare_op ::= NK_LT */ +{ yymsp[0].minor.yy290 = OP_TYPE_LOWER_THAN; } break; - case 429: /* compare_op ::= NK_GT */ -{ yymsp[0].minor.yy666 = OP_TYPE_GREATER_THAN; } + case 437: /* compare_op ::= NK_GT */ +{ yymsp[0].minor.yy290 = OP_TYPE_GREATER_THAN; } break; - case 430: /* compare_op ::= NK_LE */ -{ yymsp[0].minor.yy666 = OP_TYPE_LOWER_EQUAL; } + case 438: /* compare_op ::= NK_LE */ +{ yymsp[0].minor.yy290 = OP_TYPE_LOWER_EQUAL; } break; - case 431: /* compare_op ::= NK_GE */ -{ yymsp[0].minor.yy666 = OP_TYPE_GREATER_EQUAL; } + case 439: /* compare_op ::= NK_GE */ +{ yymsp[0].minor.yy290 = OP_TYPE_GREATER_EQUAL; } break; - case 432: /* compare_op ::= NK_NE */ -{ yymsp[0].minor.yy666 = OP_TYPE_NOT_EQUAL; } + case 440: /* compare_op ::= NK_NE */ +{ yymsp[0].minor.yy290 = OP_TYPE_NOT_EQUAL; } break; - case 433: /* compare_op ::= NK_EQ */ -{ yymsp[0].minor.yy666 = OP_TYPE_EQUAL; } + case 441: /* compare_op ::= NK_EQ */ +{ yymsp[0].minor.yy290 = OP_TYPE_EQUAL; } break; - case 434: /* compare_op ::= LIKE */ -{ yymsp[0].minor.yy666 = OP_TYPE_LIKE; } + case 442: /* compare_op ::= LIKE */ +{ yymsp[0].minor.yy290 = OP_TYPE_LIKE; } break; - case 435: /* compare_op ::= NOT LIKE */ -{ yymsp[-1].minor.yy666 = OP_TYPE_NOT_LIKE; } + case 443: /* compare_op ::= NOT LIKE */ +{ yymsp[-1].minor.yy290 = OP_TYPE_NOT_LIKE; } break; - case 436: /* compare_op ::= MATCH */ -{ yymsp[0].minor.yy666 = OP_TYPE_MATCH; } + case 444: /* compare_op ::= MATCH */ +{ yymsp[0].minor.yy290 = OP_TYPE_MATCH; } break; - case 437: /* compare_op ::= NMATCH */ -{ yymsp[0].minor.yy666 = OP_TYPE_NMATCH; } + case 445: /* compare_op ::= NMATCH */ +{ yymsp[0].minor.yy290 = OP_TYPE_NMATCH; } break; - case 438: /* compare_op ::= CONTAINS */ -{ yymsp[0].minor.yy666 = OP_TYPE_JSON_CONTAINS; } + case 446: /* compare_op ::= CONTAINS */ +{ yymsp[0].minor.yy290 = OP_TYPE_JSON_CONTAINS; } break; - case 439: /* in_op ::= IN */ -{ yymsp[0].minor.yy666 = OP_TYPE_IN; } + case 447: /* in_op ::= IN */ +{ yymsp[0].minor.yy290 = OP_TYPE_IN; } break; - case 440: /* in_op ::= NOT IN */ -{ yymsp[-1].minor.yy666 = OP_TYPE_NOT_IN; } + case 448: /* in_op ::= NOT IN */ +{ yymsp[-1].minor.yy290 = OP_TYPE_NOT_IN; } break; - case 441: /* in_predicate_value ::= NK_LP literal_list NK_RP */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy601)); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 449: /* in_predicate_value ::= NK_LP literal_list NK_RP */ +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy874)); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 443: /* boolean_value_expression ::= NOT boolean_primary */ + case 451: /* boolean_value_expression ::= NOT boolean_primary */ { - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy600), NULL)); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy602), NULL)); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; - case 444: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + case 452: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 445: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + case 453: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy602); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy602); + yylhsminor.yy602 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 451: /* from_clause_opt ::= FROM table_reference_list */ - case 480: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==480); - case 509: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==509); -{ yymsp[-1].minor.yy600 = yymsp[0].minor.yy600; } + case 459: /* from_clause_opt ::= FROM table_reference_list */ + case 488: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==488); + case 517: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==517); +{ yymsp[-1].minor.yy602 = yymsp[0].minor.yy602; } break; - case 453: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -{ yylhsminor.yy600 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy600, yymsp[0].minor.yy600, NULL); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 461: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ yylhsminor.yy602 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy602, yymsp[0].minor.yy602, NULL); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 456: /* table_primary ::= table_name alias_opt */ -{ yylhsminor.yy600 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy77, &yymsp[0].minor.yy77); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + case 464: /* table_primary ::= table_name alias_opt */ +{ yylhsminor.yy602 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy179, &yymsp[0].minor.yy179); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; - case 457: /* table_primary ::= db_name NK_DOT table_name alias_opt */ -{ yylhsminor.yy600 = createRealTableNode(pCxt, &yymsp[-3].minor.yy77, &yymsp[-1].minor.yy77, &yymsp[0].minor.yy77); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + case 465: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ yylhsminor.yy602 = createRealTableNode(pCxt, &yymsp[-3].minor.yy179, &yymsp[-1].minor.yy179, &yymsp[0].minor.yy179); } + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; - case 458: /* table_primary ::= subquery alias_opt */ -{ yylhsminor.yy600 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600), &yymsp[0].minor.yy77); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + case 466: /* table_primary ::= subquery alias_opt */ +{ yylhsminor.yy602 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy602), &yymsp[0].minor.yy179); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; - case 460: /* alias_opt ::= */ -{ yymsp[1].minor.yy77 = nil_token; } + case 468: /* alias_opt ::= */ +{ yymsp[1].minor.yy179 = nil_token; } break; - case 462: /* alias_opt ::= AS table_alias */ -{ yymsp[-1].minor.yy77 = yymsp[0].minor.yy77; } + case 470: /* alias_opt ::= AS table_alias */ +{ yymsp[-1].minor.yy179 = yymsp[0].minor.yy179; } break; - case 463: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - case 464: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==464); -{ yymsp[-2].minor.yy600 = yymsp[-1].minor.yy600; } + case 471: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + case 472: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==472); +{ yymsp[-2].minor.yy602 = yymsp[-1].minor.yy602; } break; - case 465: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -{ yylhsminor.yy600 = createJoinTableNode(pCxt, yymsp[-4].minor.yy560, yymsp[-5].minor.yy600, yymsp[-2].minor.yy600, yymsp[0].minor.yy600); } - yymsp[-5].minor.yy600 = yylhsminor.yy600; + case 473: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ yylhsminor.yy602 = createJoinTableNode(pCxt, yymsp[-4].minor.yy42, yymsp[-5].minor.yy602, yymsp[-2].minor.yy602, yymsp[0].minor.yy602); } + yymsp[-5].minor.yy602 = yylhsminor.yy602; break; - case 466: /* join_type ::= */ -{ yymsp[1].minor.yy560 = JOIN_TYPE_INNER; } + case 474: /* join_type ::= */ +{ yymsp[1].minor.yy42 = JOIN_TYPE_INNER; } break; - case 467: /* join_type ::= INNER */ -{ yymsp[0].minor.yy560 = JOIN_TYPE_INNER; } + case 475: /* join_type ::= INNER */ +{ yymsp[0].minor.yy42 = JOIN_TYPE_INNER; } break; - case 468: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 476: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { - yymsp[-11].minor.yy600 = createSelectStmt(pCxt, yymsp[-10].minor.yy841, yymsp[-9].minor.yy601, yymsp[-8].minor.yy600); - yymsp[-11].minor.yy600 = addWhereClause(pCxt, yymsp[-11].minor.yy600, yymsp[-7].minor.yy600); - yymsp[-11].minor.yy600 = addPartitionByClause(pCxt, yymsp[-11].minor.yy600, yymsp[-6].minor.yy601); - yymsp[-11].minor.yy600 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy600, yymsp[-2].minor.yy600); - yymsp[-11].minor.yy600 = addGroupByClause(pCxt, yymsp[-11].minor.yy600, yymsp[-1].minor.yy601); - yymsp[-11].minor.yy600 = addHavingClause(pCxt, yymsp[-11].minor.yy600, yymsp[0].minor.yy600); - yymsp[-11].minor.yy600 = addRangeClause(pCxt, yymsp[-11].minor.yy600, yymsp[-5].minor.yy600); - yymsp[-11].minor.yy600 = addEveryClause(pCxt, yymsp[-11].minor.yy600, yymsp[-4].minor.yy600); - yymsp[-11].minor.yy600 = addFillClause(pCxt, yymsp[-11].minor.yy600, yymsp[-3].minor.yy600); + yymsp[-11].minor.yy602 = createSelectStmt(pCxt, yymsp[-10].minor.yy767, yymsp[-9].minor.yy874, yymsp[-8].minor.yy602); + yymsp[-11].minor.yy602 = addWhereClause(pCxt, yymsp[-11].minor.yy602, yymsp[-7].minor.yy602); + yymsp[-11].minor.yy602 = addPartitionByClause(pCxt, yymsp[-11].minor.yy602, yymsp[-6].minor.yy874); + yymsp[-11].minor.yy602 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy602, yymsp[-2].minor.yy602); + yymsp[-11].minor.yy602 = addGroupByClause(pCxt, yymsp[-11].minor.yy602, yymsp[-1].minor.yy874); + yymsp[-11].minor.yy602 = addHavingClause(pCxt, yymsp[-11].minor.yy602, yymsp[0].minor.yy602); + yymsp[-11].minor.yy602 = addRangeClause(pCxt, yymsp[-11].minor.yy602, yymsp[-5].minor.yy602); + yymsp[-11].minor.yy602 = addEveryClause(pCxt, yymsp[-11].minor.yy602, yymsp[-4].minor.yy602); + yymsp[-11].minor.yy602 = addFillClause(pCxt, yymsp[-11].minor.yy602, yymsp[-3].minor.yy602); } break; - case 471: /* set_quantifier_opt ::= ALL */ -{ yymsp[0].minor.yy841 = false; } + case 479: /* set_quantifier_opt ::= ALL */ +{ yymsp[0].minor.yy767 = false; } break; - case 474: /* select_item ::= NK_STAR */ -{ yylhsminor.yy600 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; + case 482: /* select_item ::= NK_STAR */ +{ yylhsminor.yy602 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy602 = yylhsminor.yy602; break; - case 476: /* select_item ::= common_expression column_alias */ - case 486: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==486); -{ yylhsminor.yy600 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600), &yymsp[0].minor.yy77); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + case 484: /* select_item ::= common_expression column_alias */ + case 494: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==494); +{ yylhsminor.yy602 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy602), &yymsp[0].minor.yy179); } + yymsp[-1].minor.yy602 = yylhsminor.yy602; break; - case 477: /* select_item ::= common_expression AS column_alias */ - case 487: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==487); -{ yylhsminor.yy600 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), &yymsp[0].minor.yy77); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 485: /* select_item ::= common_expression AS column_alias */ + case 495: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==495); +{ yylhsminor.yy602 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), &yymsp[0].minor.yy179); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 482: /* partition_by_clause_opt ::= PARTITION BY partition_list */ - case 505: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==505); - case 524: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==524); -{ yymsp[-2].minor.yy601 = yymsp[0].minor.yy601; } + case 490: /* partition_by_clause_opt ::= PARTITION BY partition_list */ + case 513: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==513); + case 532: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==532); +{ yymsp[-2].minor.yy874 = yymsp[0].minor.yy874; } break; - case 489: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ -{ yymsp[-5].minor.yy600 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); } + case 497: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ +{ yymsp[-5].minor.yy602 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy602), releaseRawExprNode(pCxt, yymsp[-1].minor.yy602)); } break; - case 490: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ -{ yymsp[-3].minor.yy600 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); } + case 498: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ +{ yymsp[-3].minor.yy602 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy602)); } break; - case 491: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-5].minor.yy600 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), NULL, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } + case 499: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-5].minor.yy602 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy602), NULL, yymsp[-1].minor.yy602, yymsp[0].minor.yy602); } break; - case 492: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-7].minor.yy600 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy600), releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } + case 500: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-7].minor.yy602 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy602), releaseRawExprNode(pCxt, yymsp[-3].minor.yy602), yymsp[-1].minor.yy602, yymsp[0].minor.yy602); } break; - case 493: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ -{ yymsp[-6].minor.yy600 = createEventWindowNode(pCxt, yymsp[-3].minor.yy600, yymsp[0].minor.yy600); } + case 501: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ +{ yymsp[-6].minor.yy602 = createEventWindowNode(pCxt, yymsp[-3].minor.yy602, yymsp[0].minor.yy602); } break; - case 497: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -{ yymsp[-3].minor.yy600 = createFillNode(pCxt, yymsp[-1].minor.yy798, NULL); } + case 505: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ yymsp[-3].minor.yy602 = createFillNode(pCxt, yymsp[-1].minor.yy324, NULL); } break; - case 498: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ -{ yymsp[-5].minor.yy600 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy601)); } + case 506: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ +{ yymsp[-5].minor.yy602 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy874)); } break; - case 499: /* fill_mode ::= NONE */ -{ yymsp[0].minor.yy798 = FILL_MODE_NONE; } + case 507: /* fill_mode ::= NONE */ +{ yymsp[0].minor.yy324 = FILL_MODE_NONE; } break; - case 500: /* fill_mode ::= PREV */ -{ yymsp[0].minor.yy798 = FILL_MODE_PREV; } + case 508: /* fill_mode ::= PREV */ +{ yymsp[0].minor.yy324 = FILL_MODE_PREV; } break; - case 501: /* fill_mode ::= NULL */ -{ yymsp[0].minor.yy798 = FILL_MODE_NULL; } + case 509: /* fill_mode ::= NULL */ +{ yymsp[0].minor.yy324 = FILL_MODE_NULL; } break; - case 502: /* fill_mode ::= LINEAR */ -{ yymsp[0].minor.yy798 = FILL_MODE_LINEAR; } + case 510: /* fill_mode ::= LINEAR */ +{ yymsp[0].minor.yy324 = FILL_MODE_LINEAR; } break; - case 503: /* fill_mode ::= NEXT */ -{ yymsp[0].minor.yy798 = FILL_MODE_NEXT; } + case 511: /* fill_mode ::= NEXT */ +{ yymsp[0].minor.yy324 = FILL_MODE_NEXT; } break; - case 506: /* group_by_list ::= expr_or_subquery */ -{ yylhsminor.yy601 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); } - yymsp[0].minor.yy601 = yylhsminor.yy601; + case 514: /* group_by_list ::= expr_or_subquery */ +{ yylhsminor.yy874 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } + yymsp[0].minor.yy874 = yylhsminor.yy874; break; - case 507: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ -{ yylhsminor.yy601 = addNodeToList(pCxt, yymsp[-2].minor.yy601, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); } - yymsp[-2].minor.yy601 = yylhsminor.yy601; + case 515: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ +{ yylhsminor.yy874 = addNodeToList(pCxt, yymsp[-2].minor.yy874, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy602))); } + yymsp[-2].minor.yy874 = yylhsminor.yy874; break; - case 511: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ -{ yymsp[-5].minor.yy600 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); } + case 519: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ +{ yymsp[-5].minor.yy602 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy602), releaseRawExprNode(pCxt, yymsp[-1].minor.yy602)); } break; - case 514: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 522: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ { - yylhsminor.yy600 = addOrderByClause(pCxt, yymsp[-3].minor.yy600, yymsp[-2].minor.yy601); - yylhsminor.yy600 = addSlimitClause(pCxt, yylhsminor.yy600, yymsp[-1].minor.yy600); - yylhsminor.yy600 = addLimitClause(pCxt, yylhsminor.yy600, yymsp[0].minor.yy600); + yylhsminor.yy602 = addOrderByClause(pCxt, yymsp[-3].minor.yy602, yymsp[-2].minor.yy874); + yylhsminor.yy602 = addSlimitClause(pCxt, yylhsminor.yy602, yymsp[-1].minor.yy602); + yylhsminor.yy602 = addLimitClause(pCxt, yylhsminor.yy602, yymsp[0].minor.yy602); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; - case 517: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ -{ yylhsminor.yy600 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy600, yymsp[0].minor.yy600); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + case 525: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ +{ yylhsminor.yy602 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy602, yymsp[0].minor.yy602); } + yymsp[-3].minor.yy602 = yylhsminor.yy602; break; - case 518: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ -{ yylhsminor.yy600 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy600, yymsp[0].minor.yy600); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 526: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ +{ yylhsminor.yy602 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy602, yymsp[0].minor.yy602); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 526: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 530: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==530); -{ yymsp[-1].minor.yy600 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + case 534: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 538: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==538); +{ yymsp[-1].minor.yy602 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 527: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 531: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==531); -{ yymsp[-3].minor.yy600 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 535: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 539: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==539); +{ yymsp[-3].minor.yy602 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 528: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 532: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==532); -{ yymsp[-3].minor.yy600 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 536: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 540: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==540); +{ yymsp[-3].minor.yy602 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 533: /* subquery ::= NK_LP query_expression NK_RP */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy600); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 541: /* subquery ::= NK_LP query_expression NK_RP */ +{ yylhsminor.yy602 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy602); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 538: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ -{ yylhsminor.yy600 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), yymsp[-1].minor.yy32, yymsp[0].minor.yy385); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 546: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ +{ yylhsminor.yy602 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy602), yymsp[-1].minor.yy878, yymsp[0].minor.yy487); } + yymsp[-2].minor.yy602 = yylhsminor.yy602; break; - case 539: /* ordering_specification_opt ::= */ -{ yymsp[1].minor.yy32 = ORDER_ASC; } + case 547: /* ordering_specification_opt ::= */ +{ yymsp[1].minor.yy878 = ORDER_ASC; } break; - case 540: /* ordering_specification_opt ::= ASC */ -{ yymsp[0].minor.yy32 = ORDER_ASC; } + case 548: /* ordering_specification_opt ::= ASC */ +{ yymsp[0].minor.yy878 = ORDER_ASC; } break; - case 541: /* ordering_specification_opt ::= DESC */ -{ yymsp[0].minor.yy32 = ORDER_DESC; } + case 549: /* ordering_specification_opt ::= DESC */ +{ yymsp[0].minor.yy878 = ORDER_DESC; } break; - case 542: /* null_ordering_opt ::= */ -{ yymsp[1].minor.yy385 = NULL_ORDER_DEFAULT; } + case 550: /* null_ordering_opt ::= */ +{ yymsp[1].minor.yy487 = NULL_ORDER_DEFAULT; } break; - case 543: /* null_ordering_opt ::= NULLS FIRST */ -{ yymsp[-1].minor.yy385 = NULL_ORDER_FIRST; } + case 551: /* null_ordering_opt ::= NULLS FIRST */ +{ yymsp[-1].minor.yy487 = NULL_ORDER_FIRST; } break; - case 544: /* null_ordering_opt ::= NULLS LAST */ -{ yymsp[-1].minor.yy385 = NULL_ORDER_LAST; } + case 552: /* null_ordering_opt ::= NULLS LAST */ +{ yymsp[-1].minor.yy487 = NULL_ORDER_LAST; } break; default: break; /********** End reduce actions ************************************************/ }; - assert( yyrulenocreateTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_COLS, TSDB_SYSTEM_TABLE, 2) + .addColumn("table_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) + .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) + .done(); mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_PRIVILEGES, TSDB_SYSTEM_TABLE, 2) .addColumn("user_name", TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN) .addColumn("privilege", TSDB_DATA_TYPE_BINARY, 10) diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 9cc55a7cd55a0060635e9a9044ac076ce5a77119..0d46b63278722bbeaaff35d2aca72953a81b5994 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -33,7 +33,6 @@ std::unique_ptr g_mockCatalogService; class TableBuilder : public ITableBuilder { public: virtual TableBuilder& addColumn(const string& name, int8_t type, int32_t bytes) { - assert(colId_ <= schema()->tableInfo.numOfTags + schema()->tableInfo.numOfColumns); SSchema* col = schema()->schema + (colId_ - 1); col->type = type; col->colId = colId_++; @@ -427,7 +426,7 @@ class MockCatalogServiceImpl { int32_t copyTableSchemaMeta(const string& db, const string& tbname, std::unique_ptr* dst) const { STableMeta* src = getTableSchemaMeta(db, tbname); if (nullptr == src) { - return TSDB_CODE_TSC_INVALID_TABLE_NAME; + return TSDB_CODE_PAR_TABLE_NOT_EXIST; } int32_t len = sizeof(STableMeta) + sizeof(SSchema) * (src->tableInfo.numOfTags + src->tableInfo.numOfColumns); dst->reset((STableMeta*)taosMemoryCalloc(1, len)); diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index 8bdea07150ab3d6d521d4332183819c1144cd3b5..8286ae72e19b767fdddf6a882e370261ebfc4ccb 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -755,14 +755,23 @@ TEST_F(ParserInitialCTest, createStream) { }; auto setCreateStreamReq = [&](const char* pStream, const char* pSrcDb, const char* pSql, const char* pDstStb, - int8_t igExists = 0, int8_t triggerType = STREAM_TRIGGER_AT_ONCE, int64_t maxDelay = 0, - int64_t watermark = 0, int8_t igExpired = STREAM_DEFAULT_IGNORE_EXPIRED, - int8_t fillHistory = STREAM_DEFAULT_FILL_HISTORY) { + int8_t createStb = STREAM_CREATE_STABLE_TRUE, int8_t igExists = 0) { snprintf(expect.name, sizeof(expect.name), "0.%s", pStream); snprintf(expect.sourceDB, sizeof(expect.sourceDB), "0.%s", pSrcDb); snprintf(expect.targetStbFullName, sizeof(expect.targetStbFullName), "0.test.%s", pDstStb); expect.igExists = igExists; expect.sql = strdup(pSql); + expect.createStb = createStb; + expect.triggerType = STREAM_TRIGGER_AT_ONCE; + expect.maxDelay = 0; + expect.watermark = 0; + expect.fillHistory = STREAM_DEFAULT_FILL_HISTORY; + expect.igExpired = STREAM_DEFAULT_IGNORE_EXPIRED; + }; + + auto setStreamOptions = [&](int8_t triggerType = STREAM_TRIGGER_AT_ONCE, int64_t maxDelay = 0, int64_t watermark = 0, + int8_t igExpired = STREAM_DEFAULT_IGNORE_EXPIRED, + int8_t fillHistory = STREAM_DEFAULT_FILL_HISTORY) { expect.triggerType = triggerType; expect.maxDelay = maxDelay; expect.watermark = watermark; @@ -813,19 +822,22 @@ TEST_F(ParserInitialCTest, createStream) { ASSERT_EQ(pField->flags, pExpectField->flags); } } + ASSERT_EQ(req.checkpointFreq, expect.checkpointFreq); + ASSERT_EQ(req.createStb, expect.createStb); tFreeSCMCreateStreamReq(&req); }); - setCreateStreamReq("s1", "test", "create stream s1 into st1 as select count(*) from t1 interval(10s)", "st1"); - run("CREATE STREAM s1 INTO st1 AS SELECT COUNT(*) FROM t1 INTERVAL(10S)"); + setCreateStreamReq("s1", "test", "create stream s1 into st3 as select count(*) from t1 interval(10s)", "st3"); + run("CREATE STREAM s1 INTO st3 AS SELECT COUNT(*) FROM t1 INTERVAL(10S)"); clearCreateStreamReq(); setCreateStreamReq( "s1", "test", - "create stream if not exists s1 trigger max_delay 20s watermark 10s ignore expired 0 fill_history 1 into st1 " + "create stream if not exists s1 trigger max_delay 20s watermark 10s ignore expired 0 fill_history 1 into st3 " "as select count(*) from t1 interval(10s)", - "st1", 1, STREAM_TRIGGER_MAX_DELAY, 20 * MILLISECOND_PER_SECOND, 10 * MILLISECOND_PER_SECOND, 0, 1); - run("CREATE STREAM IF NOT EXISTS s1 TRIGGER MAX_DELAY 20s WATERMARK 10s IGNORE EXPIRED 0 FILL_HISTORY 1 INTO st1 AS " + "st3", 1, 1); + setStreamOptions(STREAM_TRIGGER_MAX_DELAY, 20 * MILLISECOND_PER_SECOND, 10 * MILLISECOND_PER_SECOND, 0, 1); + run("CREATE STREAM IF NOT EXISTS s1 TRIGGER MAX_DELAY 20s WATERMARK 10s IGNORE EXPIRED 0 FILL_HISTORY 1 INTO st3 AS " "SELECT COUNT(*) " "FROM t1 INTERVAL(10S)"); clearCreateStreamReq(); @@ -839,6 +851,11 @@ TEST_F(ParserInitialCTest, createStream) { run("CREATE STREAM s1 INTO st3 TAGS(tname VARCHAR(10), id INT) SUBTABLE(CONCAT('new-', tname)) " "AS SELECT _WSTART wstart, COUNT(*) cnt FROM st1 PARTITION BY TBNAME tname, tag1 id INTERVAL(10S)"); clearCreateStreamReq(); + + setCreateStreamReq("s1", "test", "create stream s1 into st1 as select max(c1), c2 from t1 interval(10s)", "st1", + STREAM_CREATE_STABLE_FALSE); + run("CREATE STREAM s1 INTO st1 AS SELECT MAX(c1), c2 FROM t1 INTERVAL(10S)"); + clearCreateStreamReq(); } TEST_F(ParserInitialCTest, createStreamSemanticCheck) { diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 4e1e98c9c026b116e9cf67d5324b53a003a65192..fef5bd654e04bee0618aa8d25fd2e2c6d46e1b39 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -545,6 +545,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, pAgg->hasLastRow = pSelect->hasLastRowFunc; pAgg->hasLast = pSelect->hasLastFunc; pAgg->hasTimeLineFunc = pSelect->hasTimeLineFunc; + pAgg->hasGroupKeyOptimized = false; pAgg->onlyHasKeepOrderFunc = pSelect->onlyHasKeepOrderFunc; pAgg->node.groupAction = getGroupAction(pCxt, pSelect); pAgg->node.requireDataOrder = getRequireDataOrder(pAgg->hasTimeLineFunc, pSelect); @@ -1358,6 +1359,7 @@ static int32_t createSetOpLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetO } if (TSDB_CODE_SUCCESS == code) { + pSetOp->precision = pSetOperator->precision; *pLogicNode = (SLogicNode*)pSetOp; } else { nodesDestroyNode((SNode*)pSetOp); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index e1687fc3a5e3136fb34893c2af6994c4c4893586..b47f0bc043644a66dfb07cc37d0b48aea940efbc 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1284,18 +1284,14 @@ static int32_t smaIndexOptFindSmaFunc(SNode* pQueryFunc, SNodeList* pSmaFuncs) { return -1; } -static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, SNodeList** pOutput, - int32_t* pWStrartIndex) { +static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, + SNodeList** pOutput) { SNodeList* pCols = NULL; SNode* pFunc = NULL; int32_t code = TSDB_CODE_SUCCESS; int32_t index = 0; int32_t smaFuncIndex = -1; - *pWStrartIndex = -1; FOREACH(pFunc, pFuncs) { - if (FUNCTION_TYPE_WSTART == ((SFunctionNode*)pFunc)->funcType) { - *pWStrartIndex = index; - } smaFuncIndex = smaIndexOptFindSmaFunc(pFunc, pSmaFuncs); if (smaFuncIndex < 0) { break; @@ -1317,8 +1313,7 @@ static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNo return code; } -static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList** pCols, - int32_t* pWStrartIndex) { +static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList** pCols) { SWindowLogicNode* pWindow = (SWindowLogicNode*)pScan->node.pParent; if (!smaIndexOptEqualInterval(pScan, pWindow, pIndex)) { return TSDB_CODE_SUCCESS; @@ -1326,36 +1321,34 @@ static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo SNodeList* pSmaFuncs = NULL; int32_t code = nodesStringToList(pIndex->expr, &pSmaFuncs); if (TSDB_CODE_SUCCESS == code) { - code = smaIndexOptCreateSmaCols(pWindow->pFuncs, pIndex->dstTbUid, pSmaFuncs, pCols, pWStrartIndex); + code = smaIndexOptCreateSmaCols(pWindow->pFuncs, pIndex->dstTbUid, pSmaFuncs, pCols); } nodesDestroyList(pSmaFuncs); return code; } static int32_t smaIndexOptApplyIndex(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, - SNodeList* pSmaCols, int32_t wstrartIndex) { + SNodeList* pSmaCols) { SLogicNode* pSmaScan = NULL; int32_t code = smaIndexOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); if (TSDB_CODE_SUCCESS == code) { code = replaceLogicNode(pLogicSubplan, pScan->node.pParent, pSmaScan); } + if (TSDB_CODE_SUCCESS == code) { + nodesDestroyNode((SNode*)pScan->node.pParent); + } return code; } -static void smaIndexOptDestroySmaIndex(void* p) { taosMemoryFree(((STableIndexInfo*)p)->expr); } - static int32_t smaIndexOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan) { int32_t code = TSDB_CODE_SUCCESS; int32_t nindexes = taosArrayGetSize(pScan->pSmaIndexes); for (int32_t i = 0; i < nindexes; ++i) { STableIndexInfo* pIndex = taosArrayGet(pScan->pSmaIndexes, i); SNodeList* pSmaCols = NULL; - int32_t wstrartIndex = -1; - code = smaIndexOptCouldApplyIndex(pScan, pIndex, &pSmaCols, &wstrartIndex); + code = smaIndexOptCouldApplyIndex(pScan, pIndex, &pSmaCols); if (TSDB_CODE_SUCCESS == code && NULL != pSmaCols) { - code = smaIndexOptApplyIndex(pLogicSubplan, pScan, pIndex, pSmaCols, wstrartIndex); - taosArrayDestroyEx(pScan->pSmaIndexes, smaIndexOptDestroySmaIndex); - pScan->pSmaIndexes = NULL; + code = smaIndexOptApplyIndex(pLogicSubplan, pScan, pIndex, pSmaCols); pCxt->optimized = true; break; } @@ -1544,6 +1537,11 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub code = adjustLogicNodeDataRequirement((SLogicNode*)pScan, pNode->resultDataOrder); } if (TSDB_CODE_SUCCESS == code) { + if (QUERY_NODE_LOGIC_PLAN_AGG == pNode->pParent->type) { + SAggLogicNode* pParent = (SAggLogicNode*)(pNode->pParent); + pParent->hasGroupKeyOptimized = true; + } + NODES_CLEAR_LIST(pNode->pChildren); nodesDestroyNode((SNode*)pNode); } @@ -1569,6 +1567,8 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub break; } } + pAgg->hasGroupKeyOptimized = true; + NODES_DESTORY_LIST(pAgg->pGroupKeys); if (TSDB_CODE_SUCCESS == code && start >= 0) { code = partTagsRewriteGroupTagsToFuncs(pScan->pGroupTags, start, pAgg); @@ -1577,6 +1577,7 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub if (TSDB_CODE_SUCCESS == code) { code = partTagsOptRebuildTbanme(pScan->pGroupTags); } + pCxt->optimized = true; return code; } @@ -2380,6 +2381,9 @@ static bool tagScanOptShouldBeOptimized(SLogicNode* pNode) { if (pScan->hasNormalCols) { return false; } + if (pScan->tableType == TSDB_SYSTEM_TABLE) { + return false; + } if (NULL == pNode->pParent || QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode->pParent) || 1 != LIST_LENGTH(pNode->pParent->pChildren)) { return false; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 78ae3c1c3b9bbf68066e53f32385f4c726a2d94d..f83704be8783ac0f65c02823a3ace1b1031685b7 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -609,7 +609,8 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pScan->accountId = pCxt->pPlanCxt->acctId; pScan->sysInfo = pCxt->pPlanCxt->sysInfo; if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TABLES) || - 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TAGS)) { + 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TAGS) || + 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_COLS)) { vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); } else { pSubplan->execNode.nodeId = MNODE_HANDLE; @@ -872,6 +873,7 @@ static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, } pAgg->mergeDataBlock = (GROUP_ACTION_KEEP == pAggLogicNode->node.groupAction ? false : true); + pAgg->groupKeyOptimized = pAggLogicNode->hasGroupKeyOptimized; SNodeList* pPrecalcExprs = NULL; SNodeList* pGroupKeys = NULL; diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index f6b1babf9501e12122bdcf97e8260251c348a617..6208fc172aa4fd9a55b259f794e0d9e266bfabea 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -333,13 +333,23 @@ static bool stbSplHasPartTbname(SNodeList* pPartKeys) { return false; } -static bool stbSplIsPartTableAgg(SAggLogicNode* pAgg) { - if (NULL != pAgg->pGroupKeys) { - return stbSplHasPartTbname(pAgg->pGroupKeys); +static bool stbSplNotSystemScan(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + return SCAN_TYPE_SYSTEM_TABLE != ((SScanLogicNode*)pNode)->scanType; + } else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { + return stbSplNotSystemScan((SLogicNode*)nodesListGetNode(pNode->pChildren, 0)); + } else { + return true; } +} + +static bool stbSplIsPartTableAgg(SAggLogicNode* pAgg) { if (1 != LIST_LENGTH(pAgg->node.pChildren)) { return false; } + if (NULL != pAgg->pGroupKeys) { + return stbSplHasPartTbname(pAgg->pGroupKeys) && stbSplNotSystemScan((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0)); + } return stbSplHasPartTbname(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0))); } diff --git a/source/libs/planner/test/planOtherTest.cpp b/source/libs/planner/test/planOtherTest.cpp index 4741d241b5ac8b5541b6f92d08dbe1dd26151099..3a12d62340f1744db5d351d7494e2e181b512d76 100644 --- a/source/libs/planner/test/planOtherTest.cpp +++ b/source/libs/planner/test/planOtherTest.cpp @@ -30,7 +30,7 @@ TEST_F(PlanOtherTest, createTopic) { TEST_F(PlanOtherTest, createStream) { useDb("root", "test"); - run("create stream if not exists s1 trigger window_close watermark 10s into st1 as select count(*) from t1 " + run("create stream if not exists s1 trigger window_close watermark 10s into st3 as select count(*) from t1 " "interval(10s)"); run("CREATE STREAM s1 INTO st3 TAGS(tname VARCHAR(10), id INT) SUBTABLE(CONCAT('new-', tname)) " @@ -43,9 +43,9 @@ TEST_F(PlanOtherTest, createStream) { TEST_F(PlanOtherTest, createStreamUseSTable) { useDb("root", "test"); - run("CREATE STREAM IF NOT EXISTS s1 into st1 as SELECT COUNT(*) FROM st1 INTERVAL(10s)"); + run("CREATE STREAM IF NOT EXISTS s1 into st3 as SELECT COUNT(*) FROM st1 INTERVAL(10s)"); - run("CREATE STREAM IF NOT EXISTS s1 into st1 as SELECT COUNT(*) FROM st1 PARTITION BY TBNAME INTERVAL(10s)"); + run("CREATE STREAM IF NOT EXISTS s1 into st3 as SELECT COUNT(*) FROM st1 PARTITION BY TBNAME INTERVAL(10s)"); } TEST_F(PlanOtherTest, createSmaIndex) { diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index c74882cf23dac901eab09b779afd56cbfb55dcaa..d8cd5f4e751ede58190a4458eb0e348e63210b00 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -116,8 +116,6 @@ int32_t cleanupTaskQueue() { } static void execHelper(struct SSchedMsg* pSchedMsg) { - assert(pSchedMsg != NULL && pSchedMsg->ahandle != NULL); - __async_exec_fn_t execFn = (__async_exec_fn_t)pSchedMsg->ahandle; int32_t code = execFn(pSchedMsg->thandle); if (code != 0 && pSchedMsg->msg != NULL) { @@ -126,8 +124,6 @@ static void execHelper(struct SSchedMsg* pSchedMsg) { } int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) { - assert(execFn != NULL); - SSchedMsg schedMsg = {0}; schedMsg.fp = execHelper; schedMsg.ahandle = execFn; @@ -138,7 +134,10 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) } void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { - assert(pMsgBody != NULL); + if (NULL == pMsgBody) { + return; + } + taosMemoryFreeClear(pMsgBody->target.dbFName); taosMemoryFreeClear(pMsgBody->msgInfo.pData); if (pMsgBody->paramFreeFp) { @@ -243,7 +242,8 @@ void destroyQueryExecRes(SExecResult* pRes) { break; } case TDMT_VND_SUBMIT: { - tFreeSSubmitRsp((SSubmitRsp*)pRes->res); + tDestroySSubmitRsp2((SSubmitRsp2*)pRes->res, TSDB_MSG_FLG_DECODE); + taosMemoryFreeClear(pRes->res); break; } case TDMT_SCH_QUERY: @@ -393,7 +393,7 @@ char* parseTagDatatoJson(void* p) { } else if (pTagVal->nData == 0) { value = cJSON_CreateString(""); } else { - ASSERT(0); + goto end; } cJSON_AddItemToObject(json, tagJsonKey, value); @@ -412,7 +412,7 @@ char* parseTagDatatoJson(void* p) { } cJSON_AddItemToObject(json, tagJsonKey, value); } else { - ASSERT(0); + goto end; } } string = cJSON_PrintUnformatted(json); @@ -447,7 +447,6 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { return TSDB_CODE_SUCCESS; } - void freeVgInfo(SDBVgInfo* vgInfo) { if (NULL == vgInfo) { return; @@ -459,7 +458,6 @@ void freeVgInfo(SDBVgInfo* vgInfo) { taosMemoryFreeClear(vgInfo); } - int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { if (NULL == pSrc) { *pDst = NULL; @@ -498,3 +496,53 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { return TSDB_CODE_SUCCESS; } + +int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst) { + if (NULL == pSrc) { + *pDst = NULL; + return TSDB_CODE_SUCCESS; + } + + *pDst = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == *pDst) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + (*pDst)->flags = pSrc->flags; + if (pSrc->name) { + (*pDst)->name = strdup(pSrc->name); + } + (*pDst)->uid = pSrc->uid; + (*pDst)->ctime = pSrc->ctime; + (*pDst)->ttl = pSrc->ttl; + (*pDst)->commentLen = pSrc->commentLen; + if (pSrc->comment) { + (*pDst)->comment = strdup(pSrc->comment); + } + (*pDst)->type = pSrc->type; + + if (pSrc->type == TSDB_CHILD_TABLE) { + if (pSrc->ctb.stbName) { + (*pDst)->ctb.stbName = strdup(pSrc->ctb.stbName); + } + (*pDst)->ctb.tagNum = pSrc->ctb.tagNum; + (*pDst)->ctb.suid = pSrc->ctb.suid; + if (pSrc->ctb.tagName) { + (*pDst)->ctb.tagName = taosArrayDup(pSrc->ctb.tagName, NULL); + } + STag* pTag = (STag*)pSrc->ctb.pTag; + if (pTag) { + (*pDst)->ctb.pTag = taosMemoryMalloc(pTag->len); + memcpy((*pDst)->ctb.pTag, pTag, pTag->len); + } + } else { + (*pDst)->ntb.schemaRow.nCols = pSrc->ntb.schemaRow.nCols; + (*pDst)->ntb.schemaRow.version = pSrc->ntb.schemaRow.nCols; + if (pSrc->ntb.schemaRow.nCols > 0 && pSrc->ntb.schemaRow.pSchema) { + (*pDst)->ntb.schemaRow.pSchema = taosMemoryMalloc(pSrc->ntb.schemaRow.nCols * sizeof(SSchema)); + memcpy((*pDst)->ntb.schemaRow.pSchema, pSrc->ntb.schemaRow.pSchema, pSrc->ntb.schemaRow.nCols * sizeof(SSchema)); + } + } + + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index eb6091d60583ecbd07fbb202870d97dce4577d91..66b8e48eb1a41d5669e008d3296359f208e0a19f 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -114,14 +114,14 @@ typedef struct SQWTaskStatus { typedef struct SQWTaskCtx { SRWLatch lock; int8_t phase; + int8_t inFetch; int8_t taskType; int8_t explain; int8_t needFetch; int8_t localExec; int32_t msgType; - int32_t fetchType; - int32_t execId; int32_t level; + uint64_t sId; bool queryGotData; bool queryRsped; @@ -221,8 +221,16 @@ typedef struct SQWorkerMgmt { #define QW_GET_PHASE(ctx) atomic_load_8(&(ctx)->phase) #define QW_SET_PHASE(ctx, _value) \ do { \ - if ((_value) != QW_PHASE_PRE_FETCH && (_value) != QW_PHASE_POST_FETCH) { \ - atomic_store_8(&(ctx)->phase, _value); \ + switch (_value) { \ + case QW_PHASE_PRE_FETCH: \ + ctx->inFetch = 1; \ + break; \ + case QW_PHASE_POST_FETCH: \ + ctx->inFetch = 0; \ + break; \ + default: \ + atomic_store_8(&(ctx)->phase, _value); \ + break; \ } \ } while (0) @@ -230,6 +238,7 @@ typedef struct SQWorkerMgmt { #define QW_UPDATE_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code) #define QW_QUERY_RUNNING(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY) +#define QW_FETCH_RUNNING(ctx) ((ctx)->inFetch) #define QW_SET_QTID(id, qId, tId, eId) \ do { \ @@ -316,34 +325,34 @@ typedef struct SQWorkerMgmt { #define QW_LOCK(type, _lock) \ do { \ if (QW_READ == (type)) { \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before read lock"); \ QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRLockLatch(_lock); \ QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) > 0); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value after read lock"); \ } else { \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before write lock"); \ QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWLockLatch(_lock); \ QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value after write lock"); \ } \ } while (0) #define QW_UNLOCK(type, _lock) \ do { \ if (QW_READ == (type)) { \ - assert(atomic_load_32((_lock)) > 0); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value before read unlock"); \ QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRUnLockLatch(_lock); \ QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after read unlock"); \ } else { \ - assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value before write unlock"); \ QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWUnLockLatch(_lock); \ QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after write unlock"); \ } \ } while (0) diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index db6e5b19fb51cda2a5bce77ed169b9ff2e69b05b..7a755cd36f46f87674096bae5b970a137c4e4655 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -124,11 +124,11 @@ void qwDbgDumpTasksInfo(SQWorker *mgmt) { void *key = taosHashGetKey(pIter, NULL); QW_GET_QTID(key, qId, tId, eId); - QW_TASK_DLOG("%p lock:%x, phase:%d, type:%d, explain:%d, needFetch:%d, localExec:%d, msgType:%d, fetchType:%d, " - "execId:%x, level:%d, queryGotData:%d, queryRsped:%d, queryEnd:%d, queryContinue:%d, queryInQueue:%d, " + QW_TASK_DLOG("%p lock:%x, phase:%d, type:%d, explain:%d, needFetch:%d, localExec:%d, msgType:%d, " + "sId:%" PRId64 ", level:%d, queryGotData:%d, queryRsped:%d, queryEnd:%d, queryContinue:%d, queryInQueue:%d, " "rspCode:%x, affectedRows:%" PRId64 ", taskHandle:%p, sinkHandle:%p, tbFName:%s, sver:%d, tver:%d, events:%d,%d,%d,%d,%d", ctx, ctx->lock, ctx->phase, ctx->taskType, ctx->explain, ctx->needFetch, ctx->localExec, ctx->msgType, - ctx->fetchType, ctx->execId, ctx->level, ctx->queryGotData, ctx->queryRsped, ctx->queryEnd, ctx->queryContinue, + ctx->sId, ctx->level, ctx->queryGotData, ctx->queryRsped, ctx->queryEnd, ctx->queryContinue, ctx->queryInQueue, ctx->rspCode, ctx->affectedRows, ctx->taskHandle, ctx->sinkHandle, ctx->tbInfo.tbFName, ctx->tbInfo.sversion, ctx->tbInfo.tversion, ctx->events[QW_EVENT_CANCEL], ctx->events[QW_EVENT_READY], ctx->events[QW_EVENT_FETCH], ctx->events[QW_EVENT_DROP], ctx->events[QW_EVENT_CQUERY]); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 81f73b1226227d79a1c11cbaa1d162bd462c6bf1..2f4e8001df1aeca484b6898f1ae6a94d82af44c1 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -147,7 +147,6 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { size_t numOfResBlock = taosArrayGetSize(pResList); for (int32_t j = 0; j < numOfResBlock; ++j) { SSDataBlock *pRes = taosArrayGetP(pResList, j); - ASSERT(pRes->info.rows > 0); SInputData inputData = {.pData = pRes}; code = dsPutDataBlock(sinkHandle, &inputData, &qcontinue); @@ -202,6 +201,15 @@ _return: QW_RET(code); } +bool qwTaskNotInExec(SQWTaskCtx *ctx) { + qTaskInfo_t taskHandle = ctx->taskHandle; + if (NULL == taskHandle || !qTaskIsExecuting(taskHandle)) { + return true; + } + + return false; +} + int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) { int32_t taskNum = 0; @@ -508,16 +516,10 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp } if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - if (QW_PHASE_POST_FETCH == phase) { - QW_TASK_WLOG("drop received at wrong phase %s", qwPhaseStr(phase)); - QW_ERR_JRET(TSDB_CODE_APP_ERROR); + if (QW_PHASE_POST_FETCH != phase || qwTaskNotInExec(ctx)) { + QW_ERR_JRET(qwDropTask(QW_FPARAMS())); + QW_ERR_JRET(ctx->rspCode); } - - // qwBuildAndSendDropRsp(&ctx->ctrlConnInfo, code); - // QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); - - QW_ERR_JRET(qwDropTask(QW_FPARAMS())); - QW_ERR_JRET(ctx->rspCode); } if (ctx->rspCode) { @@ -580,6 +582,7 @@ int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); ctx->ctrlConnInfo = qwMsg->connInfo; + ctx->sId = sId; ctx->phase = -1; QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_INIT)); @@ -670,7 +673,7 @@ _return: qwMsg->connInfo = ctx->dataConnInfo; QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); - qwBuildAndSendFetchRsp(ctx->fetchType, &qwMsg->connInfo, rsp, dataLen, code); + qwBuildAndSendFetchRsp(ctx->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code); rsp = NULL; QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), @@ -688,6 +691,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { void *rsp = NULL; int32_t dataLen = 0; bool queryStop = false; + bool qComplete = false; do { ctx = NULL; @@ -712,17 +716,18 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { } if (rsp) { - bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); + qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); if (qComplete) { atomic_store_8((int8_t *)&ctx->queryEnd, true); + atomic_store_8((int8_t *)&ctx->queryContinue, 0); } qwMsg->connInfo = ctx->dataConnInfo; QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); - qwBuildAndSendFetchRsp(ctx->fetchType, &qwMsg->connInfo, rsp, dataLen, code); + qwBuildAndSendFetchRsp(ctx->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code); rsp = NULL; QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, @@ -744,14 +749,13 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { rsp = NULL; qwMsg->connInfo = ctx->dataConnInfo; - qwBuildAndSendFetchRsp(ctx->fetchType, &qwMsg->connInfo, NULL, 0, code); + qwBuildAndSendFetchRsp(ctx->msgType + 1, &qwMsg->connInfo, NULL, 0, code); QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), 0); } QW_LOCK(QW_WRITE, &ctx->lock); - if ((queryStop && (0 == atomic_load_8((int8_t *)&ctx->queryContinue))) || code || - 0 == atomic_load_8((int8_t *)&ctx->queryContinue)) { + if (qComplete || (queryStop && (0 == atomic_load_8((int8_t *)&ctx->queryContinue))) || code) { // Note: query is not running anymore QW_SET_PHASE(ctx, 0); QW_UNLOCK(QW_WRITE, &ctx->lock); @@ -1178,8 +1182,9 @@ void qWorkerStopAllTasks(void *qWorkerMgmt) { QW_DLOG("start to stop all tasks, taskNum:%d", taosHashGetSize(mgmt->ctxHash)); - uint64_t qId, tId; + uint64_t qId, tId, sId; int32_t eId; + int64_t rId = 0; void *pIter = taosHashIterate(mgmt->ctxHash, NULL); while (pIter) { SQWTaskCtx *ctx = (SQWTaskCtx *)pIter; @@ -1188,6 +1193,8 @@ void qWorkerStopAllTasks(void *qWorkerMgmt) { QW_LOCK(QW_WRITE, &ctx->lock); + sId = ctx->sId; + QW_TASK_DLOG_E("start to force stop task"); if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP) || QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { @@ -1200,9 +1207,11 @@ void qWorkerStopAllTasks(void *qWorkerMgmt) { if (QW_QUERY_RUNNING(ctx)) { qwKillTaskHandle(ctx, TSDB_CODE_VND_STOPPED); - } else if (!QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { + } else if (QW_FETCH_RUNNING(ctx)) { QW_UPDATE_RSP_CODE(ctx, TSDB_CODE_VND_STOPPED); - QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); + QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); + } else { + qwDropTask(QW_FPARAMS()); } QW_UNLOCK(QW_WRITE, &ctx->lock); diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 0b85675c83dbde077ae54aaf737121336afd2dd1..8278da0ed0f4b7b02ab7e95558bad7f0d9d21fa8 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -53,9 +53,6 @@ typedef struct SScalarCtx { #define SCL_IS_COMPARISON_OPERATOR(_opType) ((_opType) >= OP_TYPE_GREATER_THAN && (_opType) < OP_TYPE_IS_NOT_UNKNOWN) #define SCL_DOWNGRADE_DATETYPE(_type) \ ((_type) == TSDB_DATA_TYPE_BIGINT || TSDB_DATA_TYPE_DOUBLE == (_type) || (_type) == TSDB_DATA_TYPE_UBIGINT) -#define SCL_NO_NEED_CONVERT_COMPARISION(_ltype, _rtype, _optr) \ - (IS_NUMERIC_TYPE(_ltype) && IS_NUMERIC_TYPE(_rtype) && \ - ((_optr) >= OP_TYPE_GREATER_THAN && (_optr) <= OP_TYPE_NOT_EQUAL)) #define sclFatal(...) qFatal(__VA_ARGS__) #define sclError(...) qError(__VA_ARGS__) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 3795f5ecb9ff60b19f7d589873536a5ade302767..74d555af7773cf10d7e542ac2df277bce476bac6 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -90,7 +90,7 @@ rangeCompFunc gRangeCompare[] = {filterRangeCompee, filterRangeCompei, filterRan int8_t filterGetRangeCompFuncFromOptrs(uint8_t optr, uint8_t optr2) { if (optr2) { - assert(optr2 == OP_TYPE_LOWER_THAN || optr2 == OP_TYPE_LOWER_EQUAL); + ASSERT(optr2 == OP_TYPE_LOWER_THAN || optr2 == OP_TYPE_LOWER_EQUAL); if (optr == OP_TYPE_GREATER_THAN) { if (optr2 == OP_TYPE_LOWER_THAN) { @@ -130,9 +130,9 @@ __compar_fn_t gDataCompare[] = {compareInt32Val, compareFloatVal, compareDoubleVal, compareLenPrefixedStr, - compareStrPatternMatch, + comparestrPatternMatch, compareChkInString, - compareWStrPatternMatch, + comparewcsPatternMatch, compareLenPrefixedWStr, compareUint8Val, compareUint16Val, @@ -142,15 +142,17 @@ __compar_fn_t gDataCompare[] = {compareInt32Val, setChkInBytes2, setChkInBytes4, setChkInBytes8, - compareStrRegexCompMatch, - compareStrRegexCompNMatch, + comparestrRegexMatch, + comparestrRegexNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, setChkNotInBytes8, compareChkNotInString, - compareStrPatternNotMatch, - compareWStrPatternNotMatch}; + comparestrPatternNMatch, + comparewcsPatternNMatch, + comparewcsRegexMatch, + comparewcsRegexNMatch,}; __compar_fn_t gInt8SignCompare[] = {compareInt8Val, compareInt8Int16, compareInt8Int32, compareInt8Int64, compareInt8Float, compareInt8Double}; @@ -295,9 +297,9 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { case TSDB_DATA_TYPE_NCHAR: { if (optr == OP_TYPE_MATCH) { - comparFn = 19; + comparFn = 28; } else if (optr == OP_TYPE_NMATCH) { - comparFn = 20; + comparFn = 29; } else if (optr == OP_TYPE_LIKE) { comparFn = 9; } else if (optr == OP_TYPE_NOT_LIKE) { @@ -336,7 +338,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr) { return gDataCompare[filterGetCompFuncIdx(type, optr)]; } __compar_fn_t filterGetCompFuncEx(int32_t lType, int32_t rType, int32_t optr) { - if (TSDB_DATA_TYPE_NULL == rType) { + if (TSDB_DATA_TYPE_NULL == rType || TSDB_DATA_TYPE_JSON == rType) { return NULL; } @@ -703,7 +705,7 @@ int32_t filterAddRangeImpl(void *h, SFilterRange *ra, int32_t optr) { int32_t filterAddRange(void *h, SFilterRange *ra, int32_t optr) { SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; int64_t tmp = 0; - + if (FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { SIMPLE_COPY_VALUES(&ra->s, getDataMin(ctx->type, &tmp)); // FILTER_CLR_FLAG(ra->sflag, RA_NULL); @@ -721,7 +723,7 @@ int32_t filterAddRangeCtx(void *dst, void *src, int32_t optr) { SFilterRangeCtx *dctx = (SFilterRangeCtx *)dst; SFilterRangeCtx *sctx = (SFilterRangeCtx *)src; - assert(optr == LOGIC_COND_TYPE_OR); + ASSERT(optr == LOGIC_COND_TYPE_OR); if (sctx->rs == NULL) { return TSDB_CODE_SUCCESS; @@ -776,7 +778,10 @@ int32_t filterFinishRange(void *h) { while (r && r->next) { int64_t tmp = 1; - operateVal(&tmp, &r->ra.e, &tmp, OP_TYPE_ADD, ctx->type); + int32_t code = operateVal(&tmp, &r->ra.e, &tmp, OP_TYPE_ADD, ctx->type); + if (code != 0) { + return TSDB_CODE_APP_ERROR; + } if (ctx->pCompareFunc(&tmp, &r->next->ra.s) == 0) { rn = r->next; SIMPLE_COPY_VALUES((char *)&r->next->ra.s, (char *)&r->ra.s); @@ -1120,7 +1125,7 @@ int32_t filterAddUnitImpl(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, if (u->right.type == FLD_TYPE_VALUE) { SFilterField *val = FILTER_UNIT_RIGHT_FIELD(info, u); - assert(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE)); + ASSERT(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE)); } else { int32_t paramNum = scalarGetOperatorParamNum(optr); if (1 != paramNum) { @@ -1130,7 +1135,7 @@ int32_t filterAddUnitImpl(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, } SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, u); - assert(FILTER_GET_FLAG(col->flag, FLD_TYPE_COLUMN)); + ASSERT(FILTER_GET_FLAG(col->flag, FLD_TYPE_COLUMN)); info->units[info->unitNum].compare.type = FILTER_GET_COL_FIELD_TYPE(col); info->units[info->unitNum].compare.precision = FILTER_GET_COL_FIELD_PRECISION(col); @@ -1290,29 +1295,29 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if (optr == LOGIC_COND_TYPE_AND) { if (ctx->isnull) { - assert(ctx->notnull == false && ctx->isrange == false); + ASSERT(ctx->notnull == false && ctx->isrange == false); filterAddUnit(dst, OP_TYPE_IS_NULL, &left, NULL, &uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } if (ctx->notnull) { - assert(ctx->isnull == false && ctx->isrange == false); + ASSERT(ctx->isnull == false && ctx->isrange == false); filterAddUnit(dst, OP_TYPE_IS_NOT_NULL, &left, NULL, &uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } if (!ctx->isrange) { - assert(ctx->isnull || ctx->notnull); + ASSERT(ctx->isnull || ctx->notnull); return TSDB_CODE_SUCCESS; } - assert(ctx->rs && ctx->rs->next == NULL); + ASSERT(ctx->rs && ctx->rs->next == NULL); SFilterRange *ra = &ctx->rs->ra; - assert(!((FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)))); + ASSERT(!((FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)))); if ((!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL))) { __compar_fn_t func = getComparFunc(type, 0); @@ -1366,7 +1371,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan SFilterGroup ng = {0}; g = &ng; - assert(ctx->isnull || ctx->notnull || ctx->isrange); + ASSERT(ctx->isnull || ctx->notnull || ctx->isrange); if (ctx->isnull) { filterAddUnit(dst, OP_TYPE_IS_NULL, &left, NULL, &uidx); @@ -1375,7 +1380,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan } if (ctx->notnull) { - assert(!ctx->isrange); + ASSERT(!ctx->isrange); memset(g, 0, sizeof(*g)); filterAddUnit(dst, OP_TYPE_IS_NOT_NULL, &left, NULL, &uidx); @@ -1384,7 +1389,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan } if (!ctx->isrange) { - assert(ctx->isnull || ctx->notnull); + ASSERT(ctx->isnull || ctx->notnull); g->unitNum = 0; return TSDB_CODE_SUCCESS; } @@ -1442,7 +1447,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan filterAddUnitToGroup(g, uidx); } - assert(g->unitNum > 0); + ASSERT(g->unitNum > 0); taosArrayPush(res, g); @@ -1898,7 +1903,7 @@ void filterFreeInfo(SFilterInfo *info) { } int32_t filterHandleValueExtInfo(SFilterUnit *unit, char extInfo) { - assert(extInfo > 0 || extInfo < 0); + ASSERT(extInfo > 0 || extInfo < 0); uint8_t optr = FILTER_UNIT_OPTR(unit); switch (optr) { @@ -1914,7 +1919,8 @@ int32_t filterHandleValueExtInfo(SFilterUnit *unit, char extInfo) { unit->compare.optr = FILTER_DUMMY_EMPTY_OPTR; break; default: - assert(0); + fltError("unsupported operator type"); + return TSDB_CODE_APP_ERROR; } return TSDB_CODE_SUCCESS; @@ -1924,13 +1930,13 @@ int32_t fltInitValFieldData(SFilterInfo *info) { for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit *unit = &info->units[i]; if (unit->right.type != FLD_TYPE_VALUE) { - assert(unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR || scalarGetOperatorParamNum(unit->compare.optr) == 1); + ASSERT(unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR || scalarGetOperatorParamNum(unit->compare.optr) == 1); continue; } SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); - assert(FILTER_GET_FLAG(right->flag, FLD_TYPE_VALUE)); + ASSERT(FILTER_GET_FLAG(right->flag, FLD_TYPE_VALUE)); uint32_t type = FILTER_UNIT_DATA_TYPE(unit); int8_t precision = FILTER_UNIT_DATA_PRECISION(unit); @@ -1938,7 +1944,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { SValueNode *var = (SValueNode *)fi->desc; if (var == NULL) { - assert(fi->data != NULL); + ASSERT(fi->data != NULL); continue; } @@ -2066,7 +2072,8 @@ bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right) } default: - assert(false); + fltError("unsupported operator type"); + return false; } return true; @@ -2099,7 +2106,7 @@ int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit *u, SFilterRangeCtx *c FILTER_SET_FLAG(ra.sflag, RANGE_FLG_NULL); break; case OP_TYPE_NOT_EQUAL: - assert(type == TSDB_DATA_TYPE_BOOL); + ASSERT(type == TSDB_DATA_TYPE_BOOL); if (GET_INT8_VAL(val)) { SIMPLE_COPY_VALUES(&ra.s, &tmp); SIMPLE_COPY_VALUES(&ra.e, &tmp); @@ -2114,7 +2121,8 @@ int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit *u, SFilterRangeCtx *c SIMPLE_COPY_VALUES(&ra.e, val); break; default: - assert(0); + fltError("unsupported operator type"); + return TSDB_CODE_APP_ERROR; } filterAddRange(ctx, &ra, optr); @@ -2366,8 +2374,8 @@ int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32 filterReuseRangeCtx(*ctx, type, 0); } - assert(gRes2->colInfo[cidx].type == RANGE_TYPE_MR_CTX); - assert(gRes1->colInfo[cidx].type == RANGE_TYPE_MR_CTX); + ASSERT(gRes2->colInfo[cidx].type == RANGE_TYPE_MR_CTX); + ASSERT(gRes1->colInfo[cidx].type == RANGE_TYPE_MR_CTX); filterCopyRangeCtx(*ctx, gRes2->colInfo[cidx].info); filterSourceRangeFromCtx(*ctx, gRes1->colInfo[cidx].info, optr, empty, all); @@ -2403,7 +2411,7 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx **gRes1, SFilter continue; } - assert(idx1 == idx2); + ASSERT(idx1 == idx2); ++merNum; @@ -2453,15 +2461,15 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx **gRes1, SFilter } } - assert(merNum > 0); + ASSERT(merNum > 0); SFilterColInfo *colInfo = NULL; - assert(merNum == equal1 || merNum == equal2); + ASSERT(merNum == equal1 || merNum == equal2); filterFreeGroupCtx(*gRes2); *gRes2 = NULL; - assert(colCtxs && taosArrayGetSize(colCtxs) > 0); + ASSERT(colCtxs && taosArrayGetSize(colCtxs) > 0); int32_t ctxSize = (int32_t)taosArrayGetSize(colCtxs); SFilterColCtx *pctx = NULL; @@ -2518,7 +2526,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t *gR if (pColNum > 0) { for (int32_t m = 0; m <= pEnd; ++m) { for (int32_t n = cStart; n <= cEnd; ++n) { - assert(m < n); + ASSERT(m < n); filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all); FLT_CHK_JMP(all); @@ -2539,7 +2547,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t *gR for (int32_t m = cStart; m < cEnd; ++m) { for (int32_t n = m + 1; n <= cEnd; ++n) { - assert(m < n); + ASSERT(m < n); filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all); FLT_CHK_JMP(all); @@ -2634,7 +2642,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gResNum for (uint32_t m = 0; m < res->colNum; ++m) { colInfo = &res->colInfo[res->colIdx[m]]; if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { - assert(colInfo->type == RANGE_TYPE_UNIT); + ASSERT(colInfo->type == RANGE_TYPE_UNIT); int32_t usize = (int32_t)taosArrayGetSize((SArray *)colInfo->info); for (int32_t n = 0; n < usize; ++n) { @@ -2647,7 +2655,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gResNum continue; } - assert(colInfo->type == RANGE_TYPE_MR_CTX); + ASSERT(colInfo->type == RANGE_TYPE_MR_CTX); filterAddGroupUnitFromCtx(info, &oinfo, colInfo->info, res->colIdx[m], &ng, optr, group); } @@ -2688,7 +2696,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx **gRes, int32_ continue; } - assert(idxNum[i] == gResNum); + ASSERT(idxNum[i] == gResNum); if (idxs == NULL) { idxs = taosMemoryCalloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxs)); @@ -2712,7 +2720,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx **gRes, int32_ continue; } - assert(res->colIdx[n] == idxs[m]); + ASSERT(res->colIdx[n] == idxs[m]); SFilterColInfo *colInfo = &res->colInfo[res->colIdx[n]]; if (info->colRange[m] == NULL) { @@ -2721,7 +2729,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx **gRes, int32_ info->colRange[m]->colId = FILTER_GET_COL_FIELD_ID(fi); } - assert(colInfo->type == RANGE_TYPE_MR_CTX); + ASSERT(colInfo->type == RANGE_TYPE_MR_CTX); bool all = false; filterSourceRangeFromCtx(info->colRange[m], colInfo->info, LOGIC_COND_TYPE_OR, NULL, &all); @@ -2969,7 +2977,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 unitIdx = pGroupIdx; --info->blkGroupNum; - assert(empty || all); + ASSERT(empty || all); if (empty) { FILTER_SET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY); @@ -3075,7 +3083,7 @@ int32_t filterExecuteBasedOnStatis(SFilterInfo *info, int32_t numOfRows, SColumn goto _return; } - assert(info->unitNum > 1); + ASSERT(info->unitNum > 1); *all = filterExecuteBasedOnStatisImpl(info, numOfRows, p, statis, numOfCols); goto _return; @@ -3173,6 +3181,7 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, SColumnInfoData *pRe void *colData = colDataGetData(pData, i); if (colData == NULL || colDataIsNull_s(pData, i)) { all = false; + p[i] = 0; continue; } @@ -4085,7 +4094,7 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, SColumnInfoData **p, SC SArray *pList = taosArrayInit(1, POINTER_BYTES); taosArrayPush(pList, &pSrc); - int32_t code = scalarCalculate(info->sclCtx.node, pList, &output); + code = scalarCalculate(info->sclCtx.node, pList, &output); taosArrayDestroy(pList); FLT_ERR_RET(code); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index debc210f0f18d833b760426ad31f09d91e0aebb4..c1e958b055f3442f8338c3dcdcae68bd318f0091 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -327,7 +327,10 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t case QUERY_NODE_VALUE: { SValueNode *valueNode = (SValueNode *)node; - ASSERT(param->columnData == NULL); + if (param->columnData != NULL) { + sclError("columnData should be NULL"); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } param->numOfRows = 1; int32_t code = sclCreateColumnInfoData(&valueNode->node.resType, 1, param); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 1de8a35308b4cf35982ee9e028b67ba1f7db9960..411b9b7ab504176f77c691cce1b83452ec2267c0 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -361,7 +361,6 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; - ASSERT(pOutputData->info.type == TSDB_DATA_TYPE_BIGINT); int64_t *out = (int64_t *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { @@ -1729,37 +1728,31 @@ bool getTimePseudoFuncEnv(SFunctionNode *UNUSED_PARAM(pFunc), SFuncExecEnv *pEnv } int32_t qStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - ASSERT(inputNum == 1); colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 0)); return TSDB_CODE_SUCCESS; } int32_t qEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - ASSERT(inputNum == 1); colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 1)); return TSDB_CODE_SUCCESS; } int32_t winDurFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - ASSERT(inputNum == 1); colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 2)); return TSDB_CODE_SUCCESS; } int32_t winStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - ASSERT(inputNum == 1); colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 3)); return TSDB_CODE_SUCCESS; } int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - ASSERT(inputNum == 1); colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 4)); return TSDB_CODE_SUCCESS; } int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - ASSERT(inputNum == 1); char* p = colDataGetVarData(pInput->columnData, 0); colDataAppendNItems(pOutput->columnData, pOutput->numOfRows, p, pInput->numOfRows); @@ -2598,7 +2591,7 @@ static bool checkStateOp(int8_t op, SColumnInfoData *pCol, int32_t index, SScala break; } default: { - ASSERT(0); + return false; } } return false; @@ -2771,7 +2764,9 @@ static bool getHistogramBinDesc(SHistoFuncBin **bins, int32_t *binNum, char *bin intervals[0] = -INFINITY; intervals[numOfBins - 1] = INFINITY; // in case of desc bin orders, -inf/inf should be swapped - ASSERT(numOfBins >= 4); + if (numOfBins < 4) { + return false; + } if (intervals[1] > intervals[numOfBins - 2]) { TSWAP(intervals[0], intervals[numOfBins - 1]); } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index a1995bdf500bedaa906c488fa5769146b41a8741..e9f345215e5c7eb5792a7141484ca244be560c75 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -37,6 +37,11 @@ #define IS_HELPER_NULL(col, i) colDataIsNull_s(col, i) || IS_JSON_NULL(col->info.type, colDataGetVarData(col, i)) +bool noConvertBeforeCompare(int32_t leftType, int32_t rightType, int32_t optr) { + return IS_NUMERIC_TYPE(leftType) && IS_NUMERIC_TYPE(rightType) && + (optr >= OP_TYPE_GREATER_THAN && optr <= OP_TYPE_NOT_EQUAL); +} + void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType) { switch (outType) { case TSDB_DATA_TYPE_BOOL: { @@ -338,6 +343,7 @@ static FORCE_INLINE void varToBool(char *buf, SScalarParam *pOut, int32_t rowInd colDataAppendInt8(pOut->columnData, rowIndex, (int8_t *)&v); } +// todo remove this malloc static FORCE_INLINE void varToNchar(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) { int32_t len = 0; int32_t inputLen = varDataLen(buf); @@ -383,22 +389,24 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { func = varToUnsigned; } else if (IS_FLOAT_TYPE(pCtx->outType)) { func = varToFloat; - } else if (pCtx->outType == TSDB_DATA_TYPE_BINARY) { // nchar -> binary - ASSERT(pCtx->inType == TSDB_DATA_TYPE_NCHAR); + } else if (pCtx->outType == TSDB_DATA_TYPE_VARCHAR && + pCtx->inType == TSDB_DATA_TYPE_NCHAR) { // nchar -> binary func = ncharToVar; vton = true; - } else if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { // binary -> nchar - ASSERT(pCtx->inType == TSDB_DATA_TYPE_VARCHAR); + } else if (pCtx->outType == TSDB_DATA_TYPE_NCHAR && + pCtx->inType == TSDB_DATA_TYPE_VARCHAR) { // binary -> nchar func = varToNchar; vton = true; } else if (TSDB_DATA_TYPE_TIMESTAMP == pCtx->outType) { func = varToTimestamp; } else { - sclError("invalid convert outType:%d", pCtx->outType); + sclError("invalid convert outType:%d, inType:%d", pCtx->outType, pCtx->inType); return TSDB_CODE_APP_ERROR; } pCtx->pOut->numOfRows = pCtx->pIn->numOfRows; + char* tmp = NULL; + for (int32_t i = pCtx->startIndex; i <= pCtx->endIndex; ++i) { if (IS_HELPER_NULL(pCtx->pIn->columnData, i)) { colDataAppendNULL(pCtx->pOut->columnData, i); @@ -408,12 +416,10 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { char *data = colDataGetVarData(pCtx->pIn->columnData, i); int32_t convertType = pCtx->inType; if (pCtx->inType == TSDB_DATA_TYPE_JSON) { - if (*data == TSDB_DATA_TYPE_NULL) { - ASSERT(0); - } else if (*data == TSDB_DATA_TYPE_NCHAR) { + if (*data == TSDB_DATA_TYPE_NCHAR) { data += CHAR_BYTES; convertType = TSDB_DATA_TYPE_NCHAR; - } else if (tTagIsJson(data)) { + } else if (tTagIsJson(data) || *data == TSDB_DATA_TYPE_NULL) { terrno = TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR; return terrno; } else { @@ -421,12 +427,16 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { continue; } } + int32_t bufSize = pCtx->pIn->columnData->info.bytes; - char *tmp = taosMemoryMalloc(varDataTLen(data)); - if (!tmp) { - sclError("out of memory in vectorConvertFromVarData"); - return TSDB_CODE_OUT_OF_MEMORY; + if (tmp == NULL) { + tmp = taosMemoryMalloc(bufSize); + if (tmp == NULL) { + sclError("out of memory in vectorConvertFromVarData"); + return TSDB_CODE_OUT_OF_MEMORY; + } } + if (vton) { memcpy(tmp, data, varDataTLen(data)); } else { @@ -434,7 +444,12 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { memcpy(tmp, varDataVal(data), varDataLen(data)); tmp[varDataLen(data)] = 0; } else if (TSDB_DATA_TYPE_NCHAR == convertType) { - ASSERT(varDataLen(data) <= bufSize); + // we need to convert it to native char string, and then perform the string to numeric data + if (varDataLen(data) > bufSize) { + sclError("castConvert convert buffer size too small"); + taosMemoryFreeClear(tmp); + return TSDB_CODE_APP_ERROR; + } int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp); if (len < 0) { @@ -448,9 +463,11 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { } (*func)(tmp, pCtx->pOut, i, overflow); - taosMemoryFreeClear(tmp); } + if (tmp != NULL) { + taosMemoryFreeClear(tmp); + } return TSDB_CODE_SUCCESS; } @@ -542,27 +559,17 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t *fp = filterGetCompFunc(type, optr); if (IS_NUMERIC_TYPE(type)) { - if (typeLeft == TSDB_DATA_TYPE_NCHAR) { - ASSERT(0); - // convertNcharToDouble(*pLeftData, pLeftOut); - // *pLeftData = pLeftOut; - } else if (typeLeft == TSDB_DATA_TYPE_BINARY) { - ASSERT(0); - // convertBinaryToDouble(*pLeftData, pLeftOut); - // *pLeftData = pLeftOut; + if (typeLeft == TSDB_DATA_TYPE_NCHAR || + typeLeft == TSDB_DATA_TYPE_VARCHAR) { + return false; } else if (typeLeft != type) { convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type); *pLeftData = pLeftOut; } - if (typeRight == TSDB_DATA_TYPE_NCHAR) { - ASSERT(0); - // convertNcharToDouble(*pRightData, pRightOut); - // *pRightData = pRightOut; - } else if (typeRight == TSDB_DATA_TYPE_BINARY) { - ASSERT(0); - // convertBinaryToDouble(*pRightData, pRightOut); - // *pRightData = pRightOut; + if (typeRight == TSDB_DATA_TYPE_NCHAR || + typeRight == TSDB_DATA_TYPE_VARCHAR) { + return false; } else if (typeRight != type) { convertNumberToNumber(*pRightData, pRightOut, typeRight, type); *pRightData = pRightOut; @@ -577,7 +584,7 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t *freeRight = true; } } else { - ASSERT(0); + return false; } return true; @@ -668,7 +675,10 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, } if (overflow) { - ASSERT(1 == pIn->numOfRows); + if (1 != pIn->numOfRows) { + sclError("invalid numOfRows %d", pIn->numOfRows); + return TSDB_CODE_APP_ERROR; + } pOut->numOfRows = 0; @@ -901,9 +911,11 @@ int32_t vectorGetConvertType(int32_t type1, int32_t type2) { int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, int32_t startIndex, int32_t numOfRows) { - SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; output->numOfRows = input->numOfRows; + SDataType t = {.type = type}; + t.bytes = IS_VAR_DATA_TYPE(t.type)? input->columnData->info.bytes:tDataTypes[type].bytes; + int32_t code = sclCreateColumnInfoData(&t, input->numOfRows, output); if (code != TSDB_CODE_SUCCESS) { return TSDB_CODE_OUT_OF_MEMORY; @@ -925,25 +937,43 @@ int32_t vectorConvertCols(SScalarParam *pLeft, SScalarParam *pRight, SScalarPara return TSDB_CODE_SUCCESS; } + int8_t type = 0; + int32_t code = 0; + SScalarParam *param1 = NULL, *paramOut1 = NULL; SScalarParam *param2 = NULL, *paramOut2 = NULL; - int32_t code = 0; - if (leftType < rightType) { + // always convert least data + if (IS_VAR_DATA_TYPE(leftType) && IS_VAR_DATA_TYPE(rightType) && (pLeft->numOfRows != pRight->numOfRows) && + leftType != TSDB_DATA_TYPE_JSON && rightType != TSDB_DATA_TYPE_JSON) { param1 = pLeft; param2 = pRight; paramOut1 = pLeftOut; paramOut2 = pRightOut; + + if (pLeft->numOfRows > pRight->numOfRows) { + type = leftType; + } else { + type = rightType; + } } else { - param1 = pRight; - param2 = pLeft; - paramOut1 = pRightOut; - paramOut2 = pLeftOut; - } + // we only define half value in the convert-matrix, so make sure param1 always less equal than param2 + if (leftType < rightType) { + param1 = pLeft; + param2 = pRight; + paramOut1 = pLeftOut; + paramOut2 = pRightOut; + } else { + param1 = pRight; + param2 = pLeft; + paramOut1 = pRightOut; + paramOut2 = pLeftOut; + } - int8_t type = vectorGetConvertType(GET_PARAM_TYPE(param1), GET_PARAM_TYPE(param2)); - if (0 == type) { - return TSDB_CODE_SUCCESS; + type = vectorGetConvertType(GET_PARAM_TYPE(param1), GET_PARAM_TYPE(param2)); + if (0 == type) { + return TSDB_CODE_SUCCESS; + } } if (type != GET_PARAM_TYPE(param1)) { @@ -1683,23 +1713,13 @@ void vectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam * SScalarParam *param1 = NULL; SScalarParam *param2 = NULL; - if (SCL_NO_NEED_CONVERT_COMPARISION(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), optr)) { + if (noConvertBeforeCompare(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), optr)) { param1 = pLeft; param2 = pRight; } else { vectorConvertCols(pLeft, pRight, &pLeftOut, &pRightOut, startIndex, numOfRows); - - if (pLeftOut.columnData != NULL) { - param1 = &pLeftOut; - } else { - param1 = pLeft; - } - - if (pRightOut.columnData != NULL) { - param2 = &pRightOut; - } else { - param2 = pRight; - } + param1 = (pLeftOut.columnData != NULL) ? &pLeftOut : pLeft; + param2 = (pRightOut.columnData != NULL) ? &pRightOut : pRight; } doVectorCompare(param1, param2, pOut, startIndex, numOfRows, _ord, optr); @@ -1913,7 +1933,6 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { case OP_TYPE_JSON_CONTAINS: return vectorJsonContains; default: - ASSERT(0); return NULL; } } diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index f5a40c9a87f25c6c9e3e5d39ab3db5ff55d8a959..97002ed9bfaa5e1edca3654a910237422ffdcb9a 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -344,7 +344,7 @@ TEST(constantTest, int_or_binary) { ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BIGINT); - ASSERT_EQ(v->datum.b, scltLeftV | scltRightV); + ASSERT_EQ(v->datum.i, scltLeftV | scltRightV); nodesDestroyNode(res); } @@ -1101,7 +1101,8 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do opType == OP_TYPE_LIKE || opType == OP_TYPE_NOT_LIKE || opType == OP_TYPE_MATCH || opType == OP_TYPE_NMATCH) { printf("op:%s,3result:%d,except:%f\n", operatorTypeStr(opType), *((bool *)colDataGetData(column, 0)), exceptValue); - ASSERT_EQ(*((bool *)colDataGetData(column, 0)), exceptValue); + assert(*(bool *)colDataGetData(column, 0) == exceptValue); +// ASSERT_EQ((int) *((bool *)colDataGetData(column, 0)), (int)exceptValue); } taosArrayDestroyEx(blockList, scltFreeDataBlock); @@ -1426,7 +1427,7 @@ TEST(columnTest, json_column_logic_op) { printf("--------------------json string-- 6.6hello {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n"); key = "k9"; - bool eRes8[len + len1] = {false, false, false, false, false, false, false, true, true, false, true, false, true}; + bool eRes8[len + len1] = {false, false, false, false, false, false, false, true, true, false, true, true, true}; for (int i = 0; i < len; i++) { makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes8[i], op[i], false); } @@ -1437,6 +1438,9 @@ TEST(columnTest, json_column_logic_op) { for (int i = len; i < len + len1; i++) { void *rightData = prepareNchar(inputNchar[i - len]); + if (i == 11) { + printf("abc\n"); + } makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes8[i], op[i], false); taosMemoryFree(rightData); } diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 48df7e36a30737324342bb06bdbbfced8b73a5cd..2a5eeecd36f03f26519768c822343b8cd7323b22 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -375,7 +375,7 @@ extern SSchedulerMgmt schMgmt; #define SCH_JOB_NEED_WAIT(_job) (!SCH_IS_QUERY_JOB(_job)) #define SCH_JOB_NEED_DROP(_job) (SCH_IS_QUERY_JOB(_job)) #define SCH_IS_EXPLAIN_JOB(_job) (EXPLAIN_MODE_ANALYZE == (_job)->attr.explainMode) -#define SCH_NETWORK_ERR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) +#define SCH_NETWORK_ERR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || (_code) == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED) #define SCH_MERGE_TASK_NETWORK_ERR(_task, _code, _len) \ (SCH_NETWORK_ERR(_code) && (((_len) > 0) || (!SCH_IS_DATA_BIND_TASK(_task)) || (_task)->redirectCtx.inRedirect)) #define SCH_REDIRECT_MSGTYPE(_msgType) \ @@ -478,34 +478,34 @@ extern SSchedulerMgmt schMgmt; #define SCH_LOCK(type, _lock) \ do { \ if (SCH_READ == (type)) { \ - assert(atomic_load_32(_lock) >= 0); \ + ASSERTS(atomic_load_32(_lock) >= 0, "invalid lock value before read lock"); \ SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRLockLatch(_lock); \ SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32(_lock) > 0); \ + ASSERTS(atomic_load_32(_lock) > 0, "invalid lock value after read lock"); \ } else { \ - assert(atomic_load_32(_lock) >= 0); \ + ASSERTS(atomic_load_32(_lock) >= 0, "invalid lock value before write lock"); \ SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWLockLatch(_lock); \ SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32(_lock) == TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32(_lock) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value after write lock"); \ } \ } while (0) #define SCH_UNLOCK(type, _lock) \ do { \ if (SCH_READ == (type)) { \ - assert(atomic_load_32((_lock)) > 0); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value before read unlock"); \ SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRUnLockLatch(_lock); \ SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after read unlock"); \ } else { \ - assert(atomic_load_32((_lock)) & TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32((_lock)) & TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value before write unlock"); \ SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWUnLockLatch(_lock); \ SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after write unlock"); \ } \ } while (0) diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index d422f0e88f104749dc92804b46d38aca6f89b40b..6a8f81f8c78a0146396371d79f4c8ed95e73b1fb 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -668,6 +668,7 @@ void schFreeJobImpl(void *job) { taosMemoryFreeClear(pJob->userRes.execRes); taosMemoryFreeClear(pJob->fetchRes); taosMemoryFreeClear(pJob->sql); + tsem_destroy(&pJob->rspSem); taosMemoryFree(pJob); int32_t jobNum = atomic_sub_fetch_32(&schMgmt.jobNum, 1); @@ -748,7 +749,10 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - tsem_init(&pJob->rspSem, 0, 0); + if (tsem_init(&pJob->rspSem, 0, 0)) { + SCH_JOB_ELOG("tsem_init failed, errno:%d", errno); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } pJob->refId = taosAddRef(schMgmt.jobRef, pJob); if (pJob->refId < 0) { diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index b6de9383d7a6e59ac7453227ead12fb4ba095edc..5ceb8228b03bdd0b4811b7834b24fafe8b5e6a30 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -261,46 +261,51 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa if (msg) { SDecoder coder = {0}; - SSubmitRsp *rsp = taosMemoryMalloc(sizeof(*rsp)); + SSubmitRsp2 *rsp = taosMemoryMalloc(sizeof(*rsp)); tDecoderInit(&coder, msg, msgSize); - code = tDecodeSSubmitRsp(&coder, rsp); + code = tDecodeSSubmitRsp2(&coder, rsp); + tDecoderClear(&coder); if (code) { - SCH_TASK_ELOG("decode submitRsp failed, code:%d", code); - tFreeSSubmitRsp(rsp); + SCH_TASK_ELOG("tDecodeSSubmitRsp2 failed, code:%d", code); + tDestroySSubmitRsp2(rsp, TSDB_MSG_FLG_DECODE); + taosMemoryFree(rsp); SCH_ERR_JRET(code); } - if (rsp->nBlocks > 0) { - for (int32_t i = 0; i < rsp->nBlocks; ++i) { - SSubmitBlkRsp *blk = rsp->pBlocks + i; - if (TSDB_CODE_SUCCESS != blk->code) { - code = blk->code; - tFreeSSubmitRsp(rsp); - SCH_ERR_JRET(code); + atomic_add_fetch_64(&pJob->resNumOfRows, rsp->affectedRows); + + int32_t createTbRspNum = taosArrayGetSize(rsp->aCreateTbRsp); + SCH_TASK_DLOG("submit succeed, affectedRows:%d, createTbRspNum:%d", rsp->affectedRows, createTbRspNum); + + if (rsp->aCreateTbRsp && taosArrayGetSize(rsp->aCreateTbRsp) > 0) { + SCH_LOCK(SCH_WRITE, &pJob->resLock); + if (pJob->execRes.res) { + SSubmitRsp2 *sum = pJob->execRes.res; + sum->affectedRows += rsp->affectedRows; + if (sum->aCreateTbRsp) { + taosArrayAddAll(sum->aCreateTbRsp, rsp->aCreateTbRsp); + taosArrayDestroy(rsp->aCreateTbRsp); + } else { + TSWAP(sum->aCreateTbRsp, rsp->aCreateTbRsp); } + taosMemoryFree(rsp); + } else { + pJob->execRes.res = rsp; + pJob->execRes.msgType = TDMT_VND_SUBMIT; } - } - - atomic_add_fetch_64(&pJob->resNumOfRows, rsp->affectedRows); - SCH_TASK_DLOG("submit succeed, affectedRows:%d, blocks:%d", rsp->affectedRows, rsp->nBlocks); - - SCH_LOCK(SCH_WRITE, &pJob->resLock); - if (pJob->execRes.res) { - SSubmitRsp *sum = pJob->execRes.res; - sum->affectedRows += rsp->affectedRows; - sum->nBlocks += rsp->nBlocks; - if (rsp->nBlocks > 0 && rsp->pBlocks) { - sum->pBlocks = taosMemoryRealloc(sum->pBlocks, sum->nBlocks * sizeof(*sum->pBlocks)); - memcpy(sum->pBlocks + sum->nBlocks - rsp->nBlocks, rsp->pBlocks, rsp->nBlocks * sizeof(*sum->pBlocks)); + pJob->execRes.numOfBytes += pTask->msgLen; + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + } else { + SCH_LOCK(SCH_WRITE, &pJob->resLock); + pJob->execRes.numOfBytes += pTask->msgLen; + if (NULL == pJob->execRes.res) { + TSWAP(pJob->execRes.res, rsp); + pJob->execRes.msgType = TDMT_VND_SUBMIT; } - taosMemoryFree(rsp->pBlocks); + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + tDestroySSubmitRsp2(rsp, TSDB_MSG_FLG_DECODE); taosMemoryFree(rsp); - } else { - pJob->execRes.res = rsp; - pJob->execRes.msgType = TDMT_VND_SUBMIT; } - pJob->execRes.numOfBytes += pTask->msgLen; - SCH_UNLOCK(SCH_WRITE, &pJob->resLock); } taosMemoryFreeClear(msg); diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 6cc684dddf88c6d635f3299e4bd6f81ed811fb9e..3f2feb9d286a1cad87c45fe24bc0b5132b873a7f 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -66,12 +66,13 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock return 0; } -SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq) { - SStreamDataSubmit* pDataSubmit = (SStreamDataSubmit*)taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM, 0); +SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit) { + SStreamDataSubmit2* pDataSubmit = (SStreamDataSubmit2*)taosAllocateQitem(sizeof(SStreamDataSubmit2), DEF_QITEM, 0); + if (pDataSubmit == NULL) return NULL; pDataSubmit->dataRef = (int32_t*)taosMemoryMalloc(sizeof(int32_t)); if (pDataSubmit->dataRef == NULL) goto FAIL; - pDataSubmit->data = pReq; + pDataSubmit->submit = submit; *pDataSubmit->dataRef = 1; pDataSubmit->type = STREAM_INPUT__DATA_SUBMIT; return pDataSubmit; @@ -80,47 +81,49 @@ FAIL: return NULL; } -SStreamMergedSubmit* streamMergedSubmitNew() { - SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)taosAllocateQitem(sizeof(SStreamMergedSubmit), DEF_QITEM, 0); +SStreamMergedSubmit2* streamMergedSubmitNew() { + SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)taosAllocateQitem(sizeof(SStreamMergedSubmit2), DEF_QITEM, 0); + if (pMerged == NULL) return NULL; - pMerged->reqs = taosArrayInit(0, sizeof(void*)); + pMerged->submits = taosArrayInit(0, sizeof(SPackedData)); pMerged->dataRefs = taosArrayInit(0, sizeof(void*)); - if (pMerged->dataRefs == NULL || pMerged->reqs == NULL) goto FAIL; + if (pMerged->dataRefs == NULL || pMerged->submits == NULL) goto FAIL; pMerged->type = STREAM_INPUT__MERGED_SUBMIT; return pMerged; FAIL: - if (pMerged->reqs) taosArrayDestroy(pMerged->reqs); + if (pMerged->submits) taosArrayDestroy(pMerged->submits); if (pMerged->dataRefs) taosArrayDestroy(pMerged->dataRefs); taosFreeQitem(pMerged); return NULL; } -int32_t streamMergeSubmit(SStreamMergedSubmit* pMerged, SStreamDataSubmit* pSubmit) { +int32_t streamMergeSubmit(SStreamMergedSubmit2* pMerged, SStreamDataSubmit2* pSubmit) { taosArrayPush(pMerged->dataRefs, &pSubmit->dataRef); - taosArrayPush(pMerged->reqs, &pSubmit->data); + taosArrayPush(pMerged->submits, &pSubmit->submit); pMerged->ver = pSubmit->ver; return 0; } -static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit* pDataSubmit) { +static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit2* pDataSubmit) { atomic_add_fetch_32(pDataSubmit->dataRef, 1); } -SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit) { - SStreamDataSubmit* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM, 0); +SStreamDataSubmit2* streamSubmitRefClone(SStreamDataSubmit2* pSubmit) { + SStreamDataSubmit2* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit2), DEF_QITEM, 0); + if (pSubmitClone == NULL) { return NULL; } streamDataSubmitRefInc(pSubmit); - memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit)); + memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit2)); return pSubmitClone; } -void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit) { +void streamDataSubmitRefDec(SStreamDataSubmit2* pDataSubmit) { int32_t ref = atomic_sub_fetch_32(pDataSubmit->dataRef, 1); ASSERT(ref >= 0); if (ref == 0) { - taosMemoryFree(pDataSubmit->data); + taosMemoryFree(pDataSubmit->submit.msgStr); taosMemoryFree(pDataSubmit->dataRef); } } @@ -135,16 +138,16 @@ SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* taosFreeQitem(elem); return dst; } else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && elem->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)dst; - SStreamDataSubmit* pBlockSrc = (SStreamDataSubmit*)elem; + SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)dst; + SStreamDataSubmit2* pBlockSrc = (SStreamDataSubmit2*)elem; streamMergeSubmit(pMerged, pBlockSrc); taosFreeQitem(elem); return dst; } else if (dst->type == STREAM_INPUT__DATA_SUBMIT && elem->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamMergedSubmit* pMerged = streamMergedSubmitNew(); + SStreamMergedSubmit2* pMerged = streamMergedSubmitNew(); ASSERT(pMerged); - streamMergeSubmit(pMerged, (SStreamDataSubmit*)dst); - streamMergeSubmit(pMerged, (SStreamDataSubmit*)elem); + streamMergeSubmit(pMerged, (SStreamDataSubmit2*)dst); + streamMergeSubmit(pMerged, (SStreamDataSubmit2*)elem); taosFreeQitem(dst); taosFreeQitem(elem); return (SStreamQueueItem*)pMerged; @@ -162,22 +165,22 @@ void streamFreeQitem(SStreamQueueItem* data) { taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)blockDataFreeRes); taosFreeQitem(data); } else if (type == STREAM_INPUT__DATA_SUBMIT) { - streamDataSubmitRefDec((SStreamDataSubmit*)data); + streamDataSubmitRefDec((SStreamDataSubmit2*)data); taosFreeQitem(data); } else if (type == STREAM_INPUT__MERGED_SUBMIT) { - SStreamMergedSubmit* pMerge = (SStreamMergedSubmit*)data; - int32_t sz = taosArrayGetSize(pMerge->reqs); + SStreamMergedSubmit2* pMerge = (SStreamMergedSubmit2*)data; + int32_t sz = taosArrayGetSize(pMerge->submits); for (int32_t i = 0; i < sz; i++) { int32_t* pRef = taosArrayGetP(pMerge->dataRefs, i); int32_t ref = atomic_sub_fetch_32(pRef, 1); ASSERT(ref >= 0); if (ref == 0) { - void* dataStr = taosArrayGetP(pMerge->reqs, i); - taosMemoryFree(dataStr); + SPackedData* pSubmit = (SPackedData*)taosArrayGet(pMerge->submits, i); + taosMemoryFree(pSubmit->msgStr); taosMemoryFree(pRef); } } - taosArrayDestroy(pMerge->reqs); + taosArrayDestroy(pMerge->submits); taosArrayDestroy(pMerge->dataRefs); taosFreeQitem(pMerge); } else if (type == STREAM_INPUT__REF_DATA_BLOCK) { diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 4e0b0630bc1f08969ebec2cf4a2ff674cb1824ff..f2b1db19e8eb3a15f9edf04d1bbbac3a6c750e0c 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -487,8 +487,6 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat taosMemoryFree(pReqs); } return code; - } else { - ASSERT(0); } return 0; } @@ -514,7 +512,6 @@ int32_t streamDispatch(SStreamTask* pTask) { int32_t code = 0; if (streamDispatchAllBlocks(pTask, pBlock) < 0) { - ASSERT(0); code = -1; streamQueueProcessFail(pTask->outputQueue); atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 415fa41e56f84ef0b25865ebd7ce7a44dbe92727..8f63cbbd99dcc6f75dcdc4554ba3d7bf959f481c 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -26,17 +26,18 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); - const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)data; - qDebug("task %d %p set submit input %p %p %d 1", pTask->taskId, pTask, pSubmit, pSubmit->data, *pSubmit->dataRef); - qSetMultiStreamInput(exec, pSubmit->data, 1, STREAM_INPUT__DATA_SUBMIT); + const SStreamDataSubmit2* pSubmit = (const SStreamDataSubmit2*)data; + qDebug("task %d %p set submit input %p %p %d %" PRId64, pTask->taskId, pTask, pSubmit, pSubmit->submit.msgStr, + pSubmit->submit.msgLen, pSubmit->submit.ver); + qSetMultiStreamInput(exec, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { const SStreamDataBlock* pBlock = (const SStreamDataBlock*)data; SArray* blocks = pBlock->blocks; qDebug("task %d %p set ssdata input", pTask->taskId, pTask); qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { - const SStreamMergedSubmit* pMerged = (const SStreamMergedSubmit*)data; - SArray* blocks = pMerged->reqs; + const SStreamMergedSubmit2* pMerged = (const SStreamMergedSubmit2*)data; + SArray* blocks = pMerged->submits; qDebug("task %d %p set submit input (merged), batch num: %d", pTask->taskId, pTask, (int32_t)blocks->size); qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__MERGED_SUBMIT); } else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) { @@ -48,6 +49,10 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* // exec while (1) { + if (pTask->taskStatus == TASK_STATUS__DROPPING) { + return 0; + } + SSDataBlock* output = NULL; uint64_t ts = 0; if ((code = qExecTask(exec, &output, &ts)) < 0) { @@ -106,10 +111,14 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { int32_t batchCnt = 0; while (1) { + if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + return 0; + } + SSDataBlock* output = NULL; uint64_t ts = 0; if (qExecTask(exec, &output, &ts) < 0) { - ASSERT(0); + return -1; } if (output == NULL) { if (qStreamRecoverScanFinished(exec)) { @@ -254,11 +263,11 @@ int32_t streamExecForAll(SStreamTask* pTask) { qRes->blocks = pRes; if (((SStreamQueueItem*)input)->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)input; + SStreamDataSubmit2* pSubmit = (SStreamDataSubmit2*)input; qRes->childId = pTask->selfChildId; qRes->sourceVer = pSubmit->ver; } else if (((SStreamQueueItem*)input)->type == STREAM_INPUT__MERGED_SUBMIT) { - SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)input; + SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)input; qRes->childId = pTask->selfChildId; qRes->sourceVer = pMerged->ver; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index afad78c5e57724809500905e35be136c9354885f..6f77769decdf2649b66e5cac0c8bbf823cce85e1 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -69,8 +69,7 @@ _err: } void streamMetaClose(SStreamMeta* pMeta) { - tdbCommit(pMeta->db, pMeta->txn); - tdbPostCommit(pMeta->db, pMeta->txn); + tdbAbort(pMeta->db, pMeta->txn); tdbTbClose(pMeta->pTaskDb); tdbTbClose(pMeta->pCheckpointDb); tdbClose(pMeta->db); @@ -88,6 +87,7 @@ void streamMetaClose(SStreamMeta* pMeta) { /*streamMetaReleaseTask(pMeta, pTask);*/ } taosHashCleanup(pMeta->pTasks); + taosHashCleanup(pMeta->pRecoverStatus); taosMemoryFree(pMeta->path); taosMemoryFree(pMeta); } @@ -294,6 +294,7 @@ int32_t streamLoadTasks(SStreamMeta* pMeta) { tdbTbcClose(pCur); return -1; } + pTask->taskStatus = TASK_STATUS__NORMAL; } tdbFree(pKey); diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 6889a870d19b2ec01f2f77ab57d8e70866599053..061b211ddfa7045e0121b01d28dc77a581a401d2 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -143,6 +143,7 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* ASSERT(left >= 0); if (left == 0) { taosArrayDestroy(pTask->checkReqIds); + pTask->checkReqIds = NULL; streamTaskLaunchRecover(pTask, version); } } else if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { @@ -199,7 +200,6 @@ int32_t streamBuildSourceRecover2Req(SStreamTask* pTask, SStreamRecoverStep2Req* int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver) { void* exec = pTask->exec.executor; if (qStreamSourceRecoverStep2(exec, ver) < 0) { - ASSERT(0); } return streamScanExec(pTask, 100); } diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index cf388da92ca8bec662fb5ddfbd6ada79510bfcd2..e26c8a5c5b128d6087c7c687a17cd3c5a2fa0b3e 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -663,8 +663,7 @@ int32_t streamStateSessionClear(SStreamState* pState) { void* buf = NULL; int32_t size = 0; int32_t code = streamStateSessionGetKVByCur(pCur, &delKey, &buf, &size); - if (code == 0) { - ASSERT(size > 0); + if (code == 0 && size > 0) { memset(buf, 0, size); streamStateSessionPut(pState, &delKey, buf, size); } else { diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index f7252ed8a036152b2ae9fb46861fdb8ac6754d2d..e9aba0bc3941d501ba25a5ea28aaf02e11d42cbd 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -175,6 +175,8 @@ void tFreeSStreamTask(SStreamTask* pTask) { } if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { taosArrayDestroy(pTask->shuffleDispatcher.dbInfo.pVgroupInfos); + taosArrayDestroy(pTask->checkReqIds); + pTask->checkReqIds = NULL; } if (pTask->pState) streamStateClose(pTask->pState); diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 1ce4a35dff6441a9ec289d1cb94eb24743122762..be12c72d004711e6c126792b5a7cfe4053ad1f18 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -85,9 +85,7 @@ static int64_t adjustWatermark(int64_t adjInterval, int64_t originInt, int64_t w watermark = TMAX(originInt / adjInterval, 1) * adjInterval; } else if (watermark > MAX_NUM_SCALABLE_BF * adjInterval) { watermark = MAX_NUM_SCALABLE_BF * adjInterval; - }/* else if (watermark < MIN_NUM_SCALABLE_BF * adjInterval) { - watermark = MIN_NUM_SCALABLE_BF * adjInterval; - }*/ // Todo(liuyao) save window info to tdb + } return watermark; } diff --git a/source/libs/sync/inc/syncIndexMgr.h b/source/libs/sync/inc/syncIndexMgr.h index 79b4fa0fbf06be6d1c16e783ebc59638fff1a5b5..4e6ab284f8cf9b10a06f5880abd8f14e7c13de81 100644 --- a/source/libs/sync/inc/syncIndexMgr.h +++ b/source/libs/sync/inc/syncIndexMgr.h @@ -25,29 +25,27 @@ extern "C" { // SIndexMgr ----------------------------- typedef struct SSyncIndexMgr { SRaftId (*replicas)[TSDB_MAX_REPLICA]; - SyncIndex index[TSDB_MAX_REPLICA]; - SyncTerm privateTerm[TSDB_MAX_REPLICA]; // for advanced function - - int64_t startTimeArr[TSDB_MAX_REPLICA]; - int64_t recvTimeArr[TSDB_MAX_REPLICA]; - + SyncIndex index[TSDB_MAX_REPLICA]; + SyncTerm privateTerm[TSDB_MAX_REPLICA]; // for advanced function + int64_t startTimeArr[TSDB_MAX_REPLICA]; + int64_t recvTimeArr[TSDB_MAX_REPLICA]; int32_t replicaNum; - SSyncNode *pSyncNode; + SSyncNode *pNode; } SSyncIndexMgr; -SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode); -void syncIndexMgrUpdate(SSyncIndexMgr *pSyncIndexMgr, SSyncNode *pSyncNode); -void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr); -void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr); -void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncIndex index); -SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId); - -void syncIndexMgrSetStartTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, int64_t startTime); -int64_t syncIndexMgrGetStartTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId); -void syncIndexMgrSetRecvTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, int64_t recvTime); -int64_t syncIndexMgrGetRecvTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId); -void syncIndexMgrSetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncTerm term); -SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId); +SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pNode); +void syncIndexMgrUpdate(SSyncIndexMgr *pIndexMgr, SSyncNode *pNode); +void syncIndexMgrDestroy(SSyncIndexMgr *pIndexMgr); +void syncIndexMgrClear(SSyncIndexMgr *pIndexMgr); +void syncIndexMgrSetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, SyncIndex index); +SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId); + +void syncIndexMgrSetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, int64_t startTime); +int64_t syncIndexMgrGetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId); +void syncIndexMgrSetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, int64_t recvTime); +int64_t syncIndexMgrGetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId); +void syncIndexMgrSetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, SyncTerm term); +SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 6af60af43d0af5c526054e1f99494d6068e817cf..7e08e195c1d255783da14967a26a4386b31e0b6b 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -32,11 +32,9 @@ typedef struct SyncRequestVoteReply SyncRequestVoteReply; typedef struct SyncAppendEntries SyncAppendEntries; typedef struct SyncAppendEntriesReply SyncAppendEntriesReply; typedef struct SSyncEnv SSyncEnv; -typedef struct SRaftStore SRaftStore; typedef struct SVotesGranted SVotesGranted; typedef struct SVotesRespond SVotesRespond; typedef struct SSyncIndexMgr SSyncIndexMgr; -typedef struct SRaftCfg SRaftCfg; typedef struct SSyncRespMgr SSyncRespMgr; typedef struct SSyncSnapshotSender SSyncSnapshotSender; typedef struct SSyncSnapshotReceiver SSyncSnapshotReceiver; @@ -53,11 +51,28 @@ typedef struct SyncPreSnapshot SyncPreSnapshot; typedef struct SSyncLogBuffer SSyncLogBuffer; typedef struct SSyncLogReplMgr SSyncLogReplMgr; +#define MAX_CONFIG_INDEX_COUNT 256 + +typedef struct SRaftCfg { + SSyncCfg cfg; + int32_t batchSize; + int8_t isStandBy; + int8_t snapshotStrategy; + SyncIndex lastConfigIndex; + int32_t configIndexCount; + SyncIndex configIndexArr[MAX_CONFIG_INDEX_COUNT]; +} SRaftCfg; + typedef struct SRaftId { SyncNodeId addr; SyncGroupId vgId; } SRaftId; +typedef struct SRaftStore { + SyncTerm currentTerm; + SRaftId voteFor; +} SRaftStore; + typedef struct SSyncHbTimerData { int64_t syncNodeRid; SSyncTimer* pTimer; @@ -93,15 +108,15 @@ typedef struct SPeerState { typedef struct SSyncNode { // init by SSyncInfo SyncGroupId vgId; - SRaftCfg* pRaftCfg; + SRaftCfg raftCfg; char path[TSDB_FILENAME_LEN]; char raftStorePath[TSDB_FILENAME_LEN * 2]; char configPath[TSDB_FILENAME_LEN * 2]; // sync io SSyncLogBuffer* pLogBuf; - SWal* pWal; - const SMsgCb* msgcb; + SWal* pWal; + const SMsgCb* msgcb; int32_t (*syncSendMSg)(const SEpSet* pEpSet, SRpcMsg* pMsg); int32_t (*syncEqMsg)(const SMsgCb* msgcb, SRpcMsg* pMsg); int32_t (*syncEqCtrlMsg)(const SMsgCb* msgcb, SRpcMsg* pMsg); @@ -112,6 +127,7 @@ typedef struct SSyncNode { int32_t peersNum; SNodeInfo peersNodeInfo[TSDB_MAX_REPLICA]; + SEpSet peersEpset[TSDB_MAX_REPLICA]; SRaftId peersId[TSDB_MAX_REPLICA]; int32_t replicaNum; @@ -126,8 +142,8 @@ typedef struct SSyncNode { int64_t rid; // tla+ server vars - ESyncState state; - SRaftStore* pRaftStore; + ESyncState state; + SRaftStore raftStore; // tla+ candidate vars SVotesGranted* pVotesGranted; @@ -215,7 +231,8 @@ int32_t syncNodeStart(SSyncNode* pSyncNode); int32_t syncNodeStartStandBy(SSyncNode* pSyncNode); void syncNodeClose(SSyncNode* pSyncNode); void syncNodePreClose(SSyncNode* pSyncNode); -int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_t *seq); +void syncNodePostClose(SSyncNode* pSyncNode); +int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_t* seq); int32_t syncNodeRestore(SSyncNode* pSyncNode); void syncHbTimerDataFree(SSyncHbTimerData* pData); @@ -238,14 +255,13 @@ int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode); int32_t syncNodeStartElectTimer(SSyncNode* pSyncNode, int32_t ms); int32_t syncNodeStopElectTimer(SSyncNode* pSyncNode); int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms); -int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode); +void syncNodeResetElectTimer(SSyncNode* pSyncNode); int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode); int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode); int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode); // utils -------------- int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); -int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); SyncIndex syncMinMatchIndex(SSyncNode* pSyncNode); int32_t syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHandle** h); bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode); diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 3bd94dbab5811fc071d7444ab864d3087f785d53..c3566c7c820b97c4484f15ce59239d423db9d480 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -247,8 +247,8 @@ typedef struct SyncLocalCmd { SRaftId destId; int32_t cmd; - SyncTerm sdNewTerm; // step down new term - SyncIndex fcIndex; // follower commit index + SyncTerm currentTerm; // step down new term + SyncIndex commitIndex; // follower commit index } SyncLocalCmd; int32_t syncBuildTimeout(SRpcMsg* pMsg, ESyncTimeoutType ttype, uint64_t logicClock, int32_t ms, SSyncNode* pNode); @@ -258,8 +258,8 @@ int32_t syncBuildRequestVote(SRpcMsg* pMsg, int32_t vgId); int32_t syncBuildRequestVoteReply(SRpcMsg* pMsg, int32_t vgId); int32_t syncBuildAppendEntries(SRpcMsg* pMsg, int32_t dataLen, int32_t vgId); int32_t syncBuildAppendEntriesReply(SRpcMsg* pMsg, int32_t vgId); -int32_t syncBuildAppendEntriesFromRaftLog(SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevLogTerm, - SRpcMsg* pRpcMsg); +int32_t syncBuildAppendEntriesFromRaftEntry(SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevLogTerm, + SRpcMsg* pRpcMsg); int32_t syncBuildHeartbeat(SRpcMsg* pMsg, int32_t vgId); int32_t syncBuildHeartbeatReply(SRpcMsg* pMsg, int32_t vgId); int32_t syncBuildPreSnapshot(SRpcMsg* pMsg, int32_t vgId); diff --git a/source/libs/sync/inc/syncPipeline.h b/source/libs/sync/inc/syncPipeline.h index 8c7edf85ffa9db4e3ab833c9171f1f388a87fdf4..504a9f0bd77158a2e73ed1b3d1da317cbf1ddcd6 100644 --- a/source/libs/sync/inc/syncPipeline.h +++ b/source/libs/sync/inc/syncPipeline.h @@ -61,7 +61,7 @@ typedef struct SSyncLogBuffer { // SSyncLogRepMgr SSyncLogReplMgr* syncLogReplMgrCreate(); void syncLogReplMgrDestroy(SSyncLogReplMgr* pMgr); -int32_t syncLogReplMgrReset(SSyncLogReplMgr* pMgr); +void syncLogReplMgrReset(SSyncLogReplMgr* pMgr); int32_t syncNodeLogReplMgrInit(SSyncNode* pNode); void syncNodeLogReplMgrDestroy(SSyncNode* pNode); @@ -78,14 +78,14 @@ static FORCE_INLINE int32_t syncLogGetNextRetryBackoff(SSyncLogReplMgr* pMgr) { SyncTerm syncLogReplMgrGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); int32_t syncLogReplMgrReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode); -int32_t syncLogBufferReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, - SRaftId* pDestId, bool* pBarrier); -int32_t syncLogReplMgrReplicateAttemptedOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode); -int32_t syncLogReplMgrReplicateProbeOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); +int32_t syncLogReplMgrReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, + SRaftId* pDestId, bool* pBarrier); +int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode); +int32_t syncLogReplMgrReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); int32_t syncLogReplMgrProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); -int32_t syncLogReplMgrProcessReplyInRecoveryMode(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); -int32_t syncLogReplMgrProcessReplyInNormalMode(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); +int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); +int32_t syncLogReplMgrProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); int32_t syncLogReplMgrProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg); int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode); @@ -98,6 +98,7 @@ int32_t syncLogBufferReInit(SSyncLogBuffer* pBuf, SSyncNode* pNode); // access int64_t syncLogBufferGetEndIndex(SSyncLogBuffer* pBuf); +SyncTerm syncLogBufferGetLastMatchTerm(SSyncLogBuffer* pBuf); int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEntry* pEntry); int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevTerm); int64_t syncLogBufferProceed(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncTerm* pMatchTerm); @@ -109,6 +110,8 @@ SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode, int32_t syncLogBufferValidate(SSyncLogBuffer* pBuf); int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex toIndex); +int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry, + int32_t applyCode); #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncRaftCfg.h b/source/libs/sync/inc/syncRaftCfg.h index 823983e732c88baaf8ef17ad398902ccf71346d8..4f03a60fbca27487511c0989d50d7d3518873feb 100644 --- a/source/libs/sync/inc/syncRaftCfg.h +++ b/source/libs/sync/inc/syncRaftCfg.h @@ -22,64 +22,9 @@ extern "C" { #include "syncInt.h" -#define CONFIG_FILE_LEN 2048 -#define MAX_CONFIG_INDEX_COUNT 256 - -typedef struct SRaftCfgIndex { - TdFilePtr pFile; - char path[TSDB_FILENAME_LEN * 2]; - - SyncIndex configIndexArr[MAX_CONFIG_INDEX_COUNT]; - int32_t configIndexCount; -} SRaftCfgIndex; - -SRaftCfgIndex *raftCfgIndexOpen(const char *path); -int32_t raftCfgIndexClose(SRaftCfgIndex *pRaftCfgIndex); -int32_t raftCfgIndexPersist(SRaftCfgIndex *pRaftCfgIndex); -int32_t raftCfgIndexAddConfigIndex(SRaftCfgIndex *pRaftCfgIndex, SyncIndex configIndex); - -cJSON *raftCfgIndex2Json(SRaftCfgIndex *pRaftCfgIndex); -char *raftCfgIndex2Str(SRaftCfgIndex *pRaftCfgIndex); -int32_t raftCfgIndexFromJson(const cJSON *pRoot, SRaftCfgIndex *pRaftCfgIndex); -int32_t raftCfgIndexFromStr(const char *s, SRaftCfgIndex *pRaftCfgIndex); -int32_t raftCfgIndexCreateFile(const char *path); - -typedef struct SRaftCfg { - SSyncCfg cfg; - TdFilePtr pFile; - char path[TSDB_FILENAME_LEN * 2]; - int8_t isStandBy; - int32_t batchSize; - int8_t snapshotStrategy; - SyncIndex lastConfigIndex; - - SyncIndex configIndexArr[MAX_CONFIG_INDEX_COUNT]; - int32_t configIndexCount; - -} SRaftCfg; - -SRaftCfg *raftCfgOpen(const char *path); -int32_t raftCfgClose(SRaftCfg *pRaftCfg); -int32_t raftCfgPersist(SRaftCfg *pRaftCfg); -int32_t raftCfgAddConfigIndex(SRaftCfg *pRaftCfg, SyncIndex configIndex); - -void syncCfg2SimpleStr(const SSyncCfg *pCfg, char *str, int32_t bufLen); -cJSON *syncCfg2Json(SSyncCfg *pSyncCfg); -int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg); - -cJSON *raftCfg2Json(SRaftCfg *pRaftCfg); -char *raftCfg2Str(SRaftCfg *pRaftCfg); -int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg); -int32_t raftCfgFromStr(const char *s, SRaftCfg *pRaftCfg); - -typedef struct SRaftCfgMeta { - int8_t isStandBy; - int32_t batchSize; - int8_t snapshotStrategy; - SyncIndex lastConfigIndex; -} SRaftCfgMeta; - -int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path); +int32_t syncWriteCfgFile(SSyncNode *pNode); +int32_t syncReadCfgFile(SSyncNode *pNode); +int32_t syncAddCfgIndex(SSyncNode *pNode, SyncIndex cfgIndex); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index bb6405f6b2844476273c3ecf428e159125b0688c..21a8fc64a811a832bc3893223b4c1940a6638f54 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -24,27 +24,16 @@ extern "C" { #define RAFT_STORE_BLOCK_SIZE 512 #define RAFT_STORE_PATH_LEN (TSDB_FILENAME_LEN * 2) +#define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0}) -#define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0}) - -typedef struct SRaftStore { - SyncTerm currentTerm; - SRaftId voteFor; - TdFilePtr pFile; - char path[RAFT_STORE_PATH_LEN]; -} SRaftStore; - -SRaftStore *raftStoreOpen(const char *path); -int32_t raftStoreClose(SRaftStore *pRaftStore); -int32_t raftStorePersist(SRaftStore *pRaftStore); -int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len); -int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len); - -bool raftStoreHasVoted(SRaftStore *pRaftStore); -void raftStoreVote(SRaftStore *pRaftStore, SRaftId *pRaftId); -void raftStoreClearVote(SRaftStore *pRaftStore); -void raftStoreNextTerm(SRaftStore *pRaftStore); -void raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term); +int32_t raftStoreReadFile(SSyncNode *pNode); +int32_t raftStoreWriteFile(SSyncNode *pNode); + +bool raftStoreHasVoted(SSyncNode *pNode); +void raftStoreVote(SSyncNode *pNode, SRaftId *pRaftId); +void raftStoreClearVote(SSyncNode *pNode); +void raftStoreNextTerm(SSyncNode *pNode); +void raftStoreSetTerm(SSyncNode *pNode, SyncTerm term); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 2b6e14a457b1b6a649bd7686bb7238157598314e..5277e7818fd8a152551eef51c2fdb5e858d724a5 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -56,8 +56,7 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI void snapshotSenderDestroy(SSyncSnapshotSender *pSender); bool snapshotSenderIsStart(SSyncSnapshotSender *pSender); int32_t snapshotSenderStart(SSyncSnapshotSender *pSender); -int32_t snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish); -int32_t snapshotSend(SSyncSnapshotSender *pSender); +void snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish); int32_t snapshotReSend(SSyncSnapshotSender *pSender); typedef struct SSyncSnapshotReceiver { @@ -79,10 +78,9 @@ typedef struct SSyncSnapshotReceiver { SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId); void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver); -int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg); -int32_t snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); +void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg); +void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); -void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver); // on message int32_t syncNodeOnSnapshot(SSyncNode *ths, const SRpcMsg *pMsg); diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index 7d08585656584efacc08c7faf89b3cb1f8610473..39c679a2ad4677cea4db8432b7603b951b2e21c7 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -62,22 +62,19 @@ extern "C" { // clang-format on -uint64_t syncUtilAddr2U64(const char* host, uint16_t port); -void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port); -void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet); -void syncUtilRaftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet); -bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId); -bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2); -bool syncUtilEmptyId(const SRaftId* pId); +#define CID(pRaftId) (int32_t)(((pRaftId)->addr) >> 32) +#define DID(pRaftId) (int32_t)((pRaftId)->addr) +#define SYNC_ADDR(pInfo) (int64_t)(((pInfo)->clusterId << 32) | (pInfo)->nodeId) + +void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet); +bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId); +bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2); +bool syncUtilEmptyId(const SRaftId* pId); int32_t syncUtilElectRandomMS(int32_t min, int32_t max); int32_t syncUtilQuorum(int32_t replicaNum); -cJSON* syncUtilRaftId2Json(const SRaftId* p); const char* syncStr(ESyncState state); -char* syncUtilPrintBin(char* ptr, uint32_t len); -char* syncUtilPrintBin2(char* ptr, uint32_t len); void syncUtilMsgHtoN(void* msg); -void syncUtilMsgNtoH(void* msg); bool syncUtilUserPreCommit(tmsg_t msgType); bool syncUtilUserRollback(tmsg_t msgType); diff --git a/source/libs/sync/inc/syncVoteMgr.h b/source/libs/sync/inc/syncVoteMgr.h index 066a4dd76faafaaf822cd7f770f50acaef1d0637..6e63a2b3963d944497a92655bed92b8800d3ec27 100644 --- a/source/libs/sync/inc/syncVoteMgr.h +++ b/source/libs/sync/inc/syncVoteMgr.h @@ -30,12 +30,12 @@ typedef struct SVotesGranted { SyncTerm term; int32_t quorum; bool toLeader; - SSyncNode *pSyncNode; + SSyncNode *pNode; } SVotesGranted; -SVotesGranted *voteGrantedCreate(SSyncNode *pSyncNode); +SVotesGranted *voteGrantedCreate(SSyncNode *pNode); void voteGrantedDestroy(SVotesGranted *pVotesGranted); -void voteGrantedUpdate(SVotesGranted *pVotesGranted, SSyncNode *pSyncNode); +void voteGrantedUpdate(SVotesGranted *pVotesGranted, SSyncNode *pNode); bool voteGrantedMajority(SVotesGranted *pVotesGranted); void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg); void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term); @@ -45,12 +45,12 @@ typedef struct SVotesRespond { bool isRespond[TSDB_MAX_REPLICA]; int32_t replicaNum; SyncTerm term; - SSyncNode *pSyncNode; + SSyncNode *pNode; } SVotesRespond; -SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode); +SVotesRespond *votesRespondCreate(SSyncNode *pNode); void votesRespondDestory(SVotesRespond *pVotesRespond); -void votesRespondUpdate(SVotesRespond *pVotesRespond, SSyncNode *pSyncNode); +void votesRespondUpdate(SVotesRespond *pVotesRespond, SSyncNode *pNode); bool votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId); void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *pMsg); void votesRespondReset(SVotesRespond *pVotesRespond, SyncTerm term); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 1dc6905b88cbf93dc31a4b9dc73f27567868876b..e77a8d4be333e1267aafb94c7da71e5a9651b385 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -90,6 +90,7 @@ // int32_t syncNodeFollowerCommit(SSyncNode* ths, SyncIndex newCommitIndex) { + ASSERT(false && "deprecated"); if (ths->state != TAOS_SYNC_STATE_FOLLOWER) { sNTrace(ths, "can not do follower commit"); return -1; @@ -127,7 +128,7 @@ int32_t syncNodeFollowerCommit(SSyncNode* ths, SyncIndex newCommitIndex) { return 0; } -SSyncRaftEntry* syncLogAppendEntriesToRaftEntry(const SyncAppendEntries* pMsg) { +SSyncRaftEntry* syncBuildRaftEntryFromAppendEntries(const SyncAppendEntries* pMsg) { SSyncRaftEntry* pEntry = taosMemoryMalloc(pMsg->dataLen); if (pEntry == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -158,17 +159,17 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { // prepare response msg pReply->srcId = ths->myRaftId; pReply->destId = pMsg->srcId; - pReply->term = ths->pRaftStore->currentTerm; + pReply->term = ths->raftStore.currentTerm; pReply->success = false; pReply->matchIndex = SYNC_INDEX_INVALID; pReply->lastSendIndex = pMsg->prevLogIndex + 1; pReply->startTime = ths->startTime; - if (pMsg->term < ths->pRaftStore->currentTerm) { + if (pMsg->term < ths->raftStore.currentTerm) { goto _SEND_RESPONSE; } - if (pMsg->term > ths->pRaftStore->currentTerm) { + if (pMsg->term > ths->raftStore.currentTerm) { pReply->term = pMsg->term; } @@ -181,7 +182,7 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { goto _IGNORE; } - SSyncRaftEntry* pEntry = syncLogAppendEntriesToRaftEntry(pMsg); + SSyncRaftEntry* pEntry = syncBuildRaftEntryFromAppendEntries(pMsg); if (pEntry == NULL) { sError("vgId:%d, failed to get raft entry from append entries since %s", ths->vgId, terrstr()); @@ -206,12 +207,13 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { accepted = true; _SEND_RESPONSE: + pEntry = NULL; pReply->matchIndex = syncLogBufferProceed(ths->pLogBuf, ths, &pReply->lastMatchTerm); bool matched = (pReply->matchIndex >= pReply->lastSendIndex); if (accepted && matched) { pReply->success = true; // update commit index only after matching - (void)syncNodeUpdateCommitIndex(ths, pMsg->commitIndex); + (void)syncNodeUpdateCommitIndex(ths, TMIN(pMsg->commitIndex, pReply->lastSendIndex)); } // ack, i.e. send response @@ -251,19 +253,19 @@ int32_t syncNodeOnAppendEntriesOld(SSyncNode* ths, const SRpcMsg* pRpcMsg) { SyncAppendEntriesReply* pReply = rpcRsp.pCont; pReply->srcId = ths->myRaftId; pReply->destId = pMsg->srcId; - pReply->term = ths->pRaftStore->currentTerm; + pReply->term = ths->raftStore.currentTerm; pReply->success = false; // pReply->matchIndex = ths->pLogStore->syncLogLastIndex(ths->pLogStore); pReply->matchIndex = SYNC_INDEX_INVALID; pReply->lastSendIndex = pMsg->prevLogIndex + 1; pReply->startTime = ths->startTime; - if (pMsg->term < ths->pRaftStore->currentTerm) { + if (pMsg->term < ths->raftStore.currentTerm) { syncLogRecvAppendEntries(ths, pMsg, "reject, small term"); goto _SEND_RESPONSE; } - if (pMsg->term > ths->pRaftStore->currentTerm) { + if (pMsg->term > ths->raftStore.currentTerm) { pReply->term = pMsg->term; } diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index b83be2bebb41556d68ecd1596bb3065aacc20633..8157a5a14f9c5275036d14f8917aebf3dfe05a26 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -50,19 +50,19 @@ int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { } // drop stale response - if (pMsg->term < ths->pRaftStore->currentTerm) { + if (pMsg->term < ths->raftStore.currentTerm) { syncLogRecvAppendEntriesReply(ths, pMsg, "drop stale response"); return 0; } if (ths->state == TAOS_SYNC_STATE_LEADER) { - if (pMsg->term > ths->pRaftStore->currentTerm) { + if (pMsg->term > ths->raftStore.currentTerm) { syncLogRecvAppendEntriesReply(ths, pMsg, "error term"); syncNodeStepDown(ths, pMsg->term); return -1; } - ASSERT(pMsg->term == ths->pRaftStore->currentTerm); + ASSERT(pMsg->term == ths->raftStore.currentTerm); sTrace("vgId:%d, received append entries reply. srcId:0x%016" PRIx64 ", term:%" PRId64 ", matchIndex:%" PRId64 "", pMsg->vgId, pMsg->srcId.addr, pMsg->term, pMsg->matchIndex); @@ -100,19 +100,19 @@ int32_t syncNodeOnAppendEntriesReplyOld(SSyncNode* ths, SyncAppendEntriesReply* } // drop stale response - if (pMsg->term < ths->pRaftStore->currentTerm) { + if (pMsg->term < ths->raftStore.currentTerm) { syncLogRecvAppendEntriesReply(ths, pMsg, "drop stale response"); return 0; } if (ths->state == TAOS_SYNC_STATE_LEADER) { - if (pMsg->term > ths->pRaftStore->currentTerm) { + if (pMsg->term > ths->raftStore.currentTerm) { syncLogRecvAppendEntriesReply(ths, pMsg, "error term"); syncNodeStepDown(ths, pMsg->term); return -1; } - ASSERT(pMsg->term == ths->pRaftStore->currentTerm); + ASSERT(pMsg->term == ths->raftStore.currentTerm); if (pMsg->success) { SyncIndex oldMatchIndex = syncIndexMgrGetIndex(ths->pMatchIndex, &(pMsg->srcId)); diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 5fdcbeb91c119bfebb21805b12fc1faa15eb2cab..286cf4daf52a0ef206ff4d2bf285b673dd1a7b1a 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -44,6 +44,7 @@ // /\ UNCHANGED <> // void syncOneReplicaAdvance(SSyncNode* pSyncNode) { + ASSERT(false && "deprecated"); if (pSyncNode == NULL) { sError("pSyncNode is NULL"); return; @@ -132,7 +133,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { } } // cannot commit, even if quorum agree. need check term! - if (pEntry->term <= pSyncNode->pRaftStore->currentTerm) { + if (pEntry->term <= pSyncNode->raftStore.currentTerm) { // update commit index newCommitIndex = index; @@ -328,7 +329,7 @@ int64_t syncNodeCheckCommitIndex(SSyncNode* ths, SyncIndex indexLikely) { SyncIndex commitIndex = indexLikely; syncNodeUpdateCommitIndex(ths, commitIndex); sTrace("vgId:%d, agreed upon. role:%d, term:%" PRId64 ", index: %" PRId64 "", ths->vgId, ths->state, - ths->pRaftStore->currentTerm, commitIndex); + ths->raftStore.currentTerm, commitIndex); } return ths->commitIndex; } diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 8d548114fbb851e24035edd5acb11fe371390dcf..cd3ffc33e34fef0047f9e92f319a5fadaf271419 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -48,7 +48,7 @@ static int32_t syncNodeRequestVotePeers(SSyncNode* pNode) { SyncRequestVote* pMsg = rpcMsg.pCont; pMsg->srcId = pNode->myRaftId; pMsg->destId = pNode->peersId[i]; - pMsg->term = pNode->pRaftStore->currentTerm; + pMsg->term = pNode->raftStore.currentTerm; ret = syncNodeGetLastIndexTerm(pNode, &pMsg->lastLogIndex, &pMsg->lastLogTerm); ASSERT(ret == 0); @@ -75,10 +75,10 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) { } // start election - raftStoreNextTerm(pSyncNode->pRaftStore); - raftStoreClearVote(pSyncNode->pRaftStore); - voteGrantedReset(pSyncNode->pVotesGranted, pSyncNode->pRaftStore->currentTerm); - votesRespondReset(pSyncNode->pVotesRespond, pSyncNode->pRaftStore->currentTerm); + raftStoreNextTerm(pSyncNode); + raftStoreClearVote(pSyncNode); + voteGrantedReset(pSyncNode->pVotesGranted, pSyncNode->raftStore.currentTerm); + votesRespondReset(pSyncNode->pVotesRespond, pSyncNode->raftStore.currentTerm); syncNodeVoteForSelf(pSyncNode); if (voteGrantedMajority(pSyncNode->pVotesGranted)) { @@ -94,7 +94,7 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) { voteGrantedUpdate(pSyncNode->pVotesGranted, pSyncNode); votesRespondUpdate(pSyncNode->pVotesRespond, pSyncNode); - pSyncNode->quorum = syncUtilQuorum(pSyncNode->pRaftCfg->cfg.replicaNum); + pSyncNode->quorum = syncUtilQuorum(pSyncNode->raftCfg.cfg.replicaNum); syncNodeCandidate2Leader(pSyncNode); pSyncNode->pVotesGranted->toLeader = true; diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index 28ae8711a8cd728a3850f016d32151e72332927f..1fa67cfa4d7a9b5831219cae699f474403ff243a 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -114,8 +114,8 @@ void syncHbTimerDataRemove(int64_t rid) { taosRemoveRef(gHbDataRefId, rid); } SSyncHbTimerData *syncHbTimerDataAcquire(int64_t rid) { SSyncHbTimerData *pData = taosAcquireRef(gHbDataRefId, rid); - if (pData == NULL) { - sError("failed to acquire hb-timer-data from refId:%" PRId64, rid); + if (pData == NULL && rid > 0) { + sInfo("failed to acquire hb-timer-data from refId:%" PRId64, rid); terrno = TSDB_CODE_SYN_INTERNAL_ERROR; } diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index 7933258e53f477795726d0ee5dcc286a35ccaa43..7ecb9d77825b496ea290d4bbca6f3c873edc716a 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -17,173 +17,152 @@ #include "syncIndexMgr.h" #include "syncUtil.h" -SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode) { - SSyncIndexMgr *pSyncIndexMgr = taosMemoryCalloc(1, sizeof(SSyncIndexMgr)); - if (pSyncIndexMgr == NULL) { +SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pNode) { + SSyncIndexMgr *pIndexMgr = taosMemoryCalloc(1, sizeof(SSyncIndexMgr)); + if (pIndexMgr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pSyncIndexMgr->replicas = &(pSyncNode->replicasId); - pSyncIndexMgr->replicaNum = pSyncNode->replicaNum; - pSyncIndexMgr->pSyncNode = pSyncNode; - syncIndexMgrClear(pSyncIndexMgr); + pIndexMgr->replicas = &pNode->replicasId; + pIndexMgr->replicaNum = pNode->replicaNum; + pIndexMgr->pNode = pNode; + syncIndexMgrClear(pIndexMgr); - return pSyncIndexMgr; + return pIndexMgr; } -void syncIndexMgrUpdate(SSyncIndexMgr *pSyncIndexMgr, SSyncNode *pSyncNode) { - pSyncIndexMgr->replicas = &(pSyncNode->replicasId); - pSyncIndexMgr->replicaNum = pSyncNode->replicaNum; - pSyncIndexMgr->pSyncNode = pSyncNode; - syncIndexMgrClear(pSyncIndexMgr); +void syncIndexMgrUpdate(SSyncIndexMgr *pIndexMgr, SSyncNode *pNode) { + pIndexMgr->replicas = &pNode->replicasId; + pIndexMgr->replicaNum = pNode->replicaNum; + pIndexMgr->pNode = pNode; + syncIndexMgrClear(pIndexMgr); } -void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr) { - if (pSyncIndexMgr != NULL) { - taosMemoryFree(pSyncIndexMgr); +void syncIndexMgrDestroy(SSyncIndexMgr *pIndexMgr) { + if (pIndexMgr != NULL) { + taosMemoryFree(pIndexMgr); } } -void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr) { - memset(pSyncIndexMgr->index, 0, sizeof(pSyncIndexMgr->index)); - memset(pSyncIndexMgr->privateTerm, 0, sizeof(pSyncIndexMgr->privateTerm)); +void syncIndexMgrClear(SSyncIndexMgr *pIndexMgr) { + memset(pIndexMgr->index, 0, sizeof(pIndexMgr->index)); + memset(pIndexMgr->privateTerm, 0, sizeof(pIndexMgr->privateTerm)); - // int64_t timeNow = taosGetMonotonicMs(); int64_t timeNow = taosGetTimestampMs(); - for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { - pSyncIndexMgr->startTimeArr[i] = 0; - pSyncIndexMgr->recvTimeArr[i] = timeNow; + for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + pIndexMgr->startTimeArr[i] = 0; + pIndexMgr->recvTimeArr[i] = timeNow; } - - /* - for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { - pSyncIndexMgr->index[i] = 0; - } - */ } -void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncIndex index) { - for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { - if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) { - (pSyncIndexMgr->index)[i] = index; +void syncIndexMgrSetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, SyncIndex index) { + for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { + (pIndexMgr->index)[i] = index; return; } } - // maybe config change - // ASSERT(0); - - char host[128]; - uint16_t port; - syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port); - sError("vgId:%d, index mgr set for %s:%d, index:%" PRId64 " error", pSyncIndexMgr->pSyncNode->vgId, host, port, - index); + sError("vgId:%d, indexmgr set index:%" PRId64 " for dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, index, + DID(pRaftId), CID(pRaftId)); } -SSyncLogReplMgr *syncNodeGetLogReplMgr(SSyncNode *pNode, SRaftId *pDestId) { +SSyncLogReplMgr *syncNodeGetLogReplMgr(SSyncNode *pNode, SRaftId *pRaftId) { for (int i = 0; i < pNode->replicaNum; i++) { - if (syncUtilSameId(&(pNode->replicasId[i]), pDestId)) { + if (syncUtilSameId(&pNode->replicasId[i], pRaftId)) { return pNode->logReplMgrs[i]; } } + + sError("vgId:%d, indexmgr get replmgr from dnode:%d cluster:%d failed", pNode->vgId, DID(pRaftId), CID(pRaftId)); return NULL; } -SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) { - if (pSyncIndexMgr == NULL) { - return SYNC_INDEX_INVALID; - } - - for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { - if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) { - SyncIndex idx = (pSyncIndexMgr->index)[i]; +SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) { + for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { + SyncIndex idx = (pIndexMgr->index)[i]; return idx; } } + sError("vgId:%d, indexmgr get index from dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, DID(pRaftId), + CID(pRaftId)); return SYNC_INDEX_INVALID; } -void syncIndexMgrSetStartTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, int64_t startTime) { - for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { - if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) { - (pSyncIndexMgr->startTimeArr)[i] = startTime; +void syncIndexMgrSetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, int64_t startTime) { + for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { + (pIndexMgr->startTimeArr)[i] = startTime; return; } } - // maybe config change - // ASSERT(0); - char host[128]; - uint16_t port; - syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port); - sError("vgId:%d, index mgr set for %s:%d, start-time:%" PRId64 " error", pSyncIndexMgr->pSyncNode->vgId, host, port, - startTime); + sError("vgId:%d, indexmgr set start-time:%" PRId64 " for dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, + startTime, DID(pRaftId), CID(pRaftId)); } -int64_t syncIndexMgrGetStartTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) { - for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { - if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) { - int64_t startTime = (pSyncIndexMgr->startTimeArr)[i]; +int64_t syncIndexMgrGetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) { + for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { + int64_t startTime = (pIndexMgr->startTimeArr)[i]; return startTime; } } - ASSERT(0); + + sError("vgId:%d, indexmgr get start-time from dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, DID(pRaftId), + CID(pRaftId)); return -1; } -void syncIndexMgrSetRecvTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, int64_t recvTime) { - for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { - if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) { - (pSyncIndexMgr->recvTimeArr)[i] = recvTime; +void syncIndexMgrSetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, int64_t recvTime) { + for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { + (pIndexMgr->recvTimeArr)[i] = recvTime; return; } } - // maybe config change - // ASSERT(0); - char host[128]; - uint16_t port; - syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port); - sError("vgId:%d, index mgr set for %s:%d, recv-time:%" PRId64 " error", pSyncIndexMgr->pSyncNode->vgId, host, port, - recvTime); + sError("vgId:%d, indexmgr set recv-time:%" PRId64 " for dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, recvTime, + DID(pRaftId), CID(pRaftId)); } -int64_t syncIndexMgrGetRecvTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) { - for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { - if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) { - int64_t recvTime = (pSyncIndexMgr->recvTimeArr)[i]; +int64_t syncIndexMgrGetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) { + for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { + int64_t recvTime = (pIndexMgr->recvTimeArr)[i]; return recvTime; } } + sError("vgId:%d, indexmgr get recv-time from dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, DID(pRaftId), + CID(pRaftId)); return -1; } -void syncIndexMgrSetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncTerm term) { - for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { - if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) { - (pSyncIndexMgr->privateTerm)[i] = term; +void syncIndexMgrSetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, SyncTerm term) { + for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { + (pIndexMgr->privateTerm)[i] = term; return; } } - // maybe config change - // ASSERT(0); - char host[128]; - uint16_t port; - syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port); - sError("vgId:%d, index mgr set for %s:%d, term:%" PRIu64 " error", pSyncIndexMgr->pSyncNode->vgId, host, port, term); + sError("vgId:%d, indexmgr set term:%" PRId64 " for dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, term, + DID(pRaftId), CID(pRaftId)); } -SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) { - for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { - if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) { - SyncTerm term = (pSyncIndexMgr->privateTerm)[i]; +SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) { + for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { + SyncTerm term = (pIndexMgr->privateTerm)[i]; return term; } } - ASSERT(0); + + sError("vgId:%d, indexmgr get term from dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, DID(pRaftId), + CID(pRaftId)); return -1; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index f1aa9312c627cb48a0d12610c7c42cd2f865b12a..02f9795cad8a308a602fb314b8d87535e8f22cd3 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -124,6 +124,14 @@ void syncPreStop(int64_t rid) { } } +void syncPostStop(int64_t rid) { + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode != NULL) { + syncNodePostClose(pSyncNode); + syncNodeRelease(pSyncNode); + } +} + static bool syncNodeCheckNewConfig(SSyncNode* pSyncNode, const SSyncCfg* pCfg) { if (!syncNodeInConfig(pSyncNode, pCfg)) return false; return abs(pCfg->replicaNum - pSyncNode->replicaNum) <= 1; @@ -200,12 +208,15 @@ int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg) { code = syncNodeOnLocalCmd(pSyncNode, pMsg); break; default: - sError("vgId:%d, failed to process msg:%p since invalid type:%s", pSyncNode->vgId, pMsg, - TMSG_INFO(pMsg->msgType)); + terrno = TSDB_CODE_MSG_NOT_PROCESSED; code = -1; } syncNodeRelease(pSyncNode); + if (code != 0) { + sDebug("vgId:%d, failed to process sync msg:%p type:%s since 0x%x", pSyncNode->vgId, pMsg, TMSG_INFO(pMsg->msgType), + terrno); + } return code; } @@ -228,8 +239,7 @@ int32_t syncSendTimeoutRsp(int64_t rid, int64_t seq) { syncNodeRelease(pNode); if (ret == 1) { - sInfo("send timeout response, seq:%" PRId64 " handle:%p ahandle:%p", seq, rpcMsg.info.handle, - rpcMsg.info.ahandle); + sInfo("send timeout response, seq:%" PRId64 " handle:%p ahandle:%p", seq, rpcMsg.info.handle, rpcMsg.info.ahandle); rpcSendResponse(&rpcMsg); return 0; } else { @@ -304,15 +314,10 @@ int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) { for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { int64_t matchIndex = syncIndexMgrGetIndex(pSyncNode->pMatchIndex, &(pSyncNode->peersId[i])); if (lastApplyIndex > matchIndex) { - do { - char host[64]; - uint16_t port; - syncUtilU642Addr(pSyncNode->peersId[i].addr, host, sizeof(host), &port); - sNTrace(pSyncNode, - "new-snapshot-index:%" PRId64 " is greater than match-index:%" PRId64 - " of %s:%d, do not delete wal", - lastApplyIndex, matchIndex, host, port); - } while (0); + sNTrace(pSyncNode, + "new-snapshot-index:%" PRId64 " is greater than match-index:%" PRId64 + " of dnode:%d, do not delete wal", + lastApplyIndex, matchIndex, DID(&pSyncNode->peersId[i])); syncNodeRelease(pSyncNode); return 0; @@ -463,7 +468,7 @@ bool syncNodeIsReadyForRead(SSyncNode* pSyncNode) { } if (code == 0 && pEntry != NULL) { - if (pEntry->originalRpcType == TDMT_SYNC_NOOP && pEntry->term == pSyncNode->pRaftStore->currentTerm) { + if (pEntry->originalRpcType == TDMT_SYNC_NOOP && pEntry->term == pSyncNode->raftStore.currentTerm) { ready = true; } @@ -554,7 +559,7 @@ int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeader) { (void)syncBuildLeaderTransfer(&rpcMsg, pSyncNode->vgId); SyncLeaderTransfer* pMsg = rpcMsg.pCont; - pMsg->newLeaderId.addr = syncUtilAddr2U64(newLeader.nodeFqdn, newLeader.nodePort); + pMsg->newLeaderId.addr = SYNC_ADDR(&newLeader); pMsg->newLeaderId.vgId = pSyncNode->vgId; pMsg->newNodeInfo = newLeader; @@ -620,9 +625,9 @@ int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta) { return -1; } ASSERT(rid == pSyncNode->rid); - sMeta->lastConfigIndex = pSyncNode->pRaftCfg->lastConfigIndex; + sMeta->lastConfigIndex = pSyncNode->raftCfg.lastConfigIndex; - sTrace("vgId:%d, get snapshot meta, lastConfigIndex:%" PRId64, pSyncNode->vgId, pSyncNode->pRaftCfg->lastConfigIndex); + sTrace("vgId:%d, get snapshot meta, lastConfigIndex:%" PRId64, pSyncNode->vgId, pSyncNode->raftCfg.lastConfigIndex); syncNodeRelease(pSyncNode); return 0; @@ -635,13 +640,13 @@ int32_t syncGetSnapshotMetaByIndex(int64_t rid, SyncIndex snapshotIndex, struct } ASSERT(rid == pSyncNode->rid); - ASSERT(pSyncNode->pRaftCfg->configIndexCount >= 1); - SyncIndex lastIndex = (pSyncNode->pRaftCfg->configIndexArr)[0]; + ASSERT(pSyncNode->raftCfg.configIndexCount >= 1); + SyncIndex lastIndex = (pSyncNode->raftCfg.configIndexArr)[0]; - for (int32_t i = 0; i < pSyncNode->pRaftCfg->configIndexCount; ++i) { - if ((pSyncNode->pRaftCfg->configIndexArr)[i] > lastIndex && - (pSyncNode->pRaftCfg->configIndexArr)[i] <= snapshotIndex) { - lastIndex = (pSyncNode->pRaftCfg->configIndexArr)[i]; + for (int32_t i = 0; i < pSyncNode->raftCfg.configIndexCount; ++i) { + if ((pSyncNode->raftCfg.configIndexArr)[i] > lastIndex && + (pSyncNode->raftCfg.configIndexArr)[i] <= snapshotIndex) { + lastIndex = (pSyncNode->raftCfg.configIndexArr)[i]; } } sMeta->lastConfigIndex = lastIndex; @@ -654,13 +659,13 @@ int32_t syncGetSnapshotMetaByIndex(int64_t rid, SyncIndex snapshotIndex, struct #endif SyncIndex syncNodeGetSnapshotConfigIndex(SSyncNode* pSyncNode, SyncIndex snapshotLastApplyIndex) { - ASSERT(pSyncNode->pRaftCfg->configIndexCount >= 1); - SyncIndex lastIndex = (pSyncNode->pRaftCfg->configIndexArr)[0]; + ASSERT(pSyncNode->raftCfg.configIndexCount >= 1); + SyncIndex lastIndex = (pSyncNode->raftCfg.configIndexArr)[0]; - for (int32_t i = 0; i < pSyncNode->pRaftCfg->configIndexCount; ++i) { - if ((pSyncNode->pRaftCfg->configIndexArr)[i] > lastIndex && - (pSyncNode->pRaftCfg->configIndexArr)[i] <= snapshotLastApplyIndex) { - lastIndex = (pSyncNode->pRaftCfg->configIndexArr)[i]; + for (int32_t i = 0; i < pSyncNode->raftCfg.configIndexCount; ++i) { + if ((pSyncNode->raftCfg.configIndexArr)[i] > lastIndex && + (pSyncNode->raftCfg.configIndexArr)[i] <= snapshotLastApplyIndex) { + lastIndex = (pSyncNode->raftCfg.configIndexArr)[i]; } } sTrace("vgId:%d, sync get last config index, index:%" PRId64 " lcindex:%" PRId64, pSyncNode->vgId, @@ -675,15 +680,15 @@ void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet) { SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) return; - for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) { SEp* pEp = &pEpSet->eps[i]; - tstrncpy(pEp->fqdn, pSyncNode->pRaftCfg->cfg.nodeInfo[i].nodeFqdn, TSDB_FQDN_LEN); - pEp->port = (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodePort; + tstrncpy(pEp->fqdn, pSyncNode->raftCfg.cfg.nodeInfo[i].nodeFqdn, TSDB_FQDN_LEN); + pEp->port = (pSyncNode->raftCfg.cfg.nodeInfo)[i].nodePort; pEpSet->numOfEps++; sDebug("vgId:%d, sync get retry epset, index:%d %s:%d", pSyncNode->vgId, i, pEp->fqdn, pEp->port); } if (pEpSet->numOfEps > 0) { - pEpSet->inUse = (pSyncNode->pRaftCfg->cfg.myIndex + 1) % pEpSet->numOfEps; + pEpSet->inUse = (pSyncNode->raftCfg.cfg.myIndex + 1) % pEpSet->numOfEps; } sInfo("vgId:%d, sync get retry epset numOfEps:%d inUse:%d", pSyncNode->vgId, pEpSet->numOfEps, pEpSet->inUse); @@ -731,7 +736,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_ int32_t code = syncNodeOnClientRequest(pSyncNode, pMsg, &retIndex); if (code == 0) { pMsg->info.conn.applyIndex = retIndex; - pMsg->info.conn.applyTerm = pSyncNode->pRaftStore->currentTerm; + pMsg->info.conn.applyTerm = pSyncNode->raftStore.currentTerm; sTrace("vgId:%d, propose optimized msg, index:%" PRId64 " type:%s", pSyncNode->vgId, retIndex, TMSG_INFO(pMsg->msgType)); return 1; @@ -847,60 +852,57 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { } } + memcpy(pSyncNode->path, pSyncInfo->path, sizeof(pSyncNode->path)); + snprintf(pSyncNode->raftStorePath, sizeof(pSyncNode->raftStorePath), "%s%sraft_store.json", pSyncInfo->path, + TD_DIRSEP); snprintf(pSyncNode->configPath, sizeof(pSyncNode->configPath), "%s%sraft_config.json", pSyncInfo->path, TD_DIRSEP); + if (!taosCheckExistFile(pSyncNode->configPath)) { // create a new raft config file - SRaftCfgMeta meta = {0}; - meta.isStandBy = pSyncInfo->isStandBy; - meta.snapshotStrategy = pSyncInfo->snapshotStrategy; - meta.lastConfigIndex = SYNC_INDEX_INVALID; - meta.batchSize = pSyncInfo->batchSize; - if (raftCfgCreateFile(&pSyncInfo->syncCfg, meta, pSyncNode->configPath) != 0) { - sError("vgId:%d, failed to create raft cfg file at %s", pSyncNode->vgId, pSyncNode->configPath); + sInfo("vgId:%d, create a new raft config file", pSyncNode->vgId); + pSyncNode->raftCfg.isStandBy = pSyncInfo->isStandBy; + pSyncNode->raftCfg.snapshotStrategy = pSyncInfo->snapshotStrategy; + pSyncNode->raftCfg.lastConfigIndex = SYNC_INDEX_INVALID; + pSyncNode->raftCfg.batchSize = pSyncInfo->batchSize; + pSyncNode->raftCfg.cfg = pSyncInfo->syncCfg; + pSyncNode->raftCfg.configIndexCount = 1; + pSyncNode->raftCfg.configIndexArr[0] = -1; + + if (syncWriteCfgFile(pSyncNode) != 0) { + sError("vgId:%d, failed to create sync cfg file", pSyncNode->vgId); goto _error; } - if (pSyncInfo->syncCfg.replicaNum == 0) { - sInfo("vgId:%d, sync config not input", pSyncNode->vgId); - pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg; - } } else { // update syncCfg by raft_config.json - pSyncNode->pRaftCfg = raftCfgOpen(pSyncNode->configPath); - if (pSyncNode->pRaftCfg == NULL) { - sError("vgId:%d, failed to open raft cfg file at %s", pSyncNode->vgId, pSyncNode->configPath); + if (syncReadCfgFile(pSyncNode) != 0) { + sError("vgId:%d, failed to read sync cfg file", pSyncNode->vgId); goto _error; } - if (pSyncInfo->syncCfg.replicaNum > 0 && syncIsConfigChanged(&pSyncNode->pRaftCfg->cfg, &pSyncInfo->syncCfg)) { + if (pSyncInfo->syncCfg.replicaNum > 0 && syncIsConfigChanged(&pSyncNode->raftCfg.cfg, &pSyncInfo->syncCfg)) { sInfo("vgId:%d, use sync config from input options and write to cfg file", pSyncNode->vgId); - pSyncNode->pRaftCfg->cfg = pSyncInfo->syncCfg; - if (raftCfgPersist(pSyncNode->pRaftCfg) != 0) { - sError("vgId:%d, failed to persist raft cfg file at %s", pSyncNode->vgId, pSyncNode->configPath); + pSyncNode->raftCfg.cfg = pSyncInfo->syncCfg; + if (syncWriteCfgFile(pSyncNode) != 0) { + sError("vgId:%d, failed to write sync cfg file", pSyncNode->vgId); goto _error; } } else { - sInfo("vgId:%d, use sync config from raft cfg file", pSyncNode->vgId); - pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg; + sInfo("vgId:%d, use sync config from sync cfg file", pSyncNode->vgId); + pSyncInfo->syncCfg = pSyncNode->raftCfg.cfg; } - - raftCfgClose(pSyncNode->pRaftCfg); - pSyncNode->pRaftCfg = NULL; } // init by SSyncInfo pSyncNode->vgId = pSyncInfo->vgId; - SSyncCfg* pCfg = &pSyncInfo->syncCfg; + SSyncCfg* pCfg = &pSyncNode->raftCfg.cfg; sInfo("vgId:%d, start to open sync node, replica:%d selfIndex:%d", pSyncNode->vgId, pCfg->replicaNum, pCfg->myIndex); for (int32_t i = 0; i < pCfg->replicaNum; ++i) { SNodeInfo* pNode = &pCfg->nodeInfo[i]; - sInfo("vgId:%d, index:%d ep:%s:%u", pSyncNode->vgId, i, pNode->nodeFqdn, pNode->nodePort); + tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + sInfo("vgId:%d, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, pSyncNode->vgId, i, pNode->nodeFqdn, pNode->nodePort, + pNode->nodeId, pNode->clusterId); } - memcpy(pSyncNode->path, pSyncInfo->path, sizeof(pSyncNode->path)); - snprintf(pSyncNode->raftStorePath, sizeof(pSyncNode->raftStorePath), "%s%sraft_store.json", pSyncInfo->path, - TD_DIRSEP); - snprintf(pSyncNode->configPath, sizeof(pSyncNode->configPath), "%s%sraft_config.json", pSyncInfo->path, TD_DIRSEP); - pSyncNode->pWal = pSyncInfo->pWal; pSyncNode->msgcb = pSyncInfo->msgcb; pSyncNode->syncSendMSg = pSyncInfo->syncSendMSg; @@ -914,26 +916,20 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { goto _error; } - // init raft config - pSyncNode->pRaftCfg = raftCfgOpen(pSyncNode->configPath); - if (pSyncNode->pRaftCfg == NULL) { - sError("vgId:%d, failed to open raft cfg file at %s", pSyncNode->vgId, pSyncNode->configPath); - goto _error; - } - // init internal - pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex]; + pSyncNode->myNodeInfo = pSyncNode->raftCfg.cfg.nodeInfo[pSyncNode->raftCfg.cfg.myIndex]; if (!syncUtilNodeInfo2RaftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId)) { sError("vgId:%d, failed to determine my raft member id", pSyncNode->vgId); goto _error; } // init peersNum, peers, peersId - pSyncNode->peersNum = pSyncNode->pRaftCfg->cfg.replicaNum - 1; + pSyncNode->peersNum = pSyncNode->raftCfg.cfg.replicaNum - 1; int32_t j = 0; - for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { - if (i != pSyncNode->pRaftCfg->cfg.myIndex) { - pSyncNode->peersNodeInfo[j] = pSyncNode->pRaftCfg->cfg.nodeInfo[i]; + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) { + if (i != pSyncNode->raftCfg.cfg.myIndex) { + pSyncNode->peersNodeInfo[j] = pSyncNode->raftCfg.cfg.nodeInfo[i]; + syncUtilNodeInfo2EpSet(&pSyncNode->peersNodeInfo[j], &pSyncNode->peersEpset[j]); j++; } } @@ -945,9 +941,9 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { } // init replicaNum, replicasId - pSyncNode->replicaNum = pSyncNode->pRaftCfg->cfg.replicaNum; - for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { - if (!syncUtilNodeInfo2RaftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i])) { + pSyncNode->replicaNum = pSyncNode->raftCfg.cfg.replicaNum; + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) { + if (!syncUtilNodeInfo2RaftId(&pSyncNode->raftCfg.cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i])) { sError("vgId:%d, failed to determine raft member id, replica:%d", pSyncNode->vgId, i); goto _error; } @@ -956,7 +952,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { // init raft algorithm pSyncNode->pFsm = pSyncInfo->pFsm; pSyncInfo->pFsm = NULL; - pSyncNode->quorum = syncUtilQuorum(pSyncNode->pRaftCfg->cfg.replicaNum); + pSyncNode->quorum = syncUtilQuorum(pSyncNode->raftCfg.cfg.replicaNum); pSyncNode->leaderCache = EMPTY_RAFT_ID; // init life cycle outside @@ -987,8 +983,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { // init TLA+ server vars pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; - pSyncNode->pRaftStore = raftStoreOpen(pSyncNode->raftStorePath); - if (pSyncNode->pRaftStore == NULL) { + if (raftStoreReadFile(pSyncNode) != 0) { sError("vgId:%d, failed to open raft store at path %s", pSyncNode->vgId, pSyncNode->raftStorePath); goto _error; } @@ -1034,6 +1029,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { } } pSyncNode->commitIndex = commitIndex; + sInfo("vgId:%d, sync node commitIndex initialized as %" PRId64, pSyncNode->vgId, pSyncNode->commitIndex); if (syncNodeLogStoreRestoreOnNeed(pSyncNode) < 0) { goto _error; @@ -1084,13 +1080,17 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { // snapshot senders for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { SSyncSnapshotSender* pSender = snapshotSenderCreate(pSyncNode, i); - // ASSERT(pSender != NULL); - (pSyncNode->senders)[i] = pSender; - sSDebug(pSender, "snapshot sender create new while open, data:%p", pSender); + if (pSender == NULL) return NULL; + + pSyncNode->senders[i] = pSender; + sSDebug(pSender, "snapshot sender create while open sync node, data:%p", pSender); } // snapshot receivers pSyncNode->pNewNodeReceiver = snapshotReceiverCreate(pSyncNode, EMPTY_RAFT_ID); + if (pSyncNode->pNewNodeReceiver == NULL) return NULL; + sRDebug(pSyncNode->pNewNodeReceiver, "snapshot receiver create while open sync node, data:%p", + pSyncNode->pNewNodeReceiver); // is config changing pSyncNode->changing = false; @@ -1131,10 +1131,8 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->hbrSlowNum = 0; pSyncNode->tmrRoutineNum = 0; - sNInfo(pSyncNode, "sync open, node:%p", pSyncNode); - sTrace("vgId:%d, tsElectInterval:%d, tsHeartbeatInterval:%d, tsHeartbeatTimeout:%d", pSyncNode->vgId, tsElectInterval, - tsHeartbeatInterval, tsHeartbeatTimeout); - + sNInfo(pSyncNode, "sync open, node:%p electInterval:%d heartbeatInterval:%d heartbeatTimeout:%d", pSyncNode, + tsElectInterval, tsHeartbeatInterval, tsHeartbeatTimeout); return pSyncNode; _error: @@ -1172,9 +1170,10 @@ int32_t syncNodeRestore(SSyncNode* pSyncNode) { } ASSERT(endIndex == lastVer + 1); - commitIndex = TMAX(pSyncNode->commitIndex, commitIndex); + pSyncNode->commitIndex = TMAX(pSyncNode->commitIndex, commitIndex); + sInfo("vgId:%d, restore sync until commitIndex:%" PRId64, pSyncNode->vgId, pSyncNode->commitIndex); - if (syncLogBufferCommit(pSyncNode->pLogBuf, pSyncNode, commitIndex) < 0) { + if (syncLogBufferCommit(pSyncNode->pLogBuf, pSyncNode, pSyncNode->commitIndex) < 0) { return -1; } @@ -1184,7 +1183,7 @@ int32_t syncNodeRestore(SSyncNode* pSyncNode) { int32_t syncNodeStart(SSyncNode* pSyncNode) { // start raft if (pSyncNode->replicaNum == 1) { - raftStoreNextTerm(pSyncNode->pRaftStore); + raftStoreNextTerm(pSyncNode); syncNodeBecomeLeader(pSyncNode, "one replica start"); // Raft 3.6.2 Committing entries from previous terms @@ -1202,7 +1201,7 @@ int32_t syncNodeStart(SSyncNode* pSyncNode) { void syncNodeStartOld(SSyncNode* pSyncNode) { // start raft if (pSyncNode->replicaNum == 1) { - raftStoreNextTerm(pSyncNode->pRaftStore); + raftStoreNextTerm(pSyncNode); syncNodeBecomeLeader(pSyncNode, "one replica start"); // Raft 3.6.2 Committing entries from previous terms @@ -1246,20 +1245,40 @@ void syncNodePreClose(SSyncNode* pSyncNode) { } } +#if 0 if (pSyncNode->pNewNodeReceiver != NULL) { if (snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) { - snapshotReceiverForceStop(pSyncNode->pNewNodeReceiver); + snapshotReceiverStop(pSyncNode->pNewNodeReceiver); } + sDebug("vgId:%d, snapshot receiver destroy while preclose sync node, data:%p", pSyncNode->vgId, + pSyncNode->pNewNodeReceiver); snapshotReceiverDestroy(pSyncNode->pNewNodeReceiver); pSyncNode->pNewNodeReceiver = NULL; } +#endif // stop elect timer syncNodeStopElectTimer(pSyncNode); // stop heartbeat timer syncNodeStopHeartbeatTimer(pSyncNode); + + // clean rsp + syncRespCleanRsp(pSyncNode->pSyncRespMgr); +} + +void syncNodePostClose(SSyncNode* pSyncNode) { + if (pSyncNode->pNewNodeReceiver != NULL) { + if (snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) { + snapshotReceiverStop(pSyncNode->pNewNodeReceiver); + } + + sDebug("vgId:%d, snapshot receiver destroy while preclose sync node, data:%p", pSyncNode->vgId, + pSyncNode->pNewNodeReceiver); + snapshotReceiverDestroy(pSyncNode->pNewNodeReceiver); + pSyncNode->pNewNodeReceiver = NULL; + } } void syncHbTimerDataFree(SSyncHbTimerData* pData) { taosMemoryFree(pData); } @@ -1268,10 +1287,6 @@ void syncNodeClose(SSyncNode* pSyncNode) { if (pSyncNode == NULL) return; sNInfo(pSyncNode, "sync close, node:%p", pSyncNode); - int32_t ret = raftStoreClose(pSyncNode->pRaftStore); - ASSERT(ret == 0); - pSyncNode->pRaftStore = NULL; - syncNodeLogReplMgrDestroy(pSyncNode); syncRespMgrDestroy(pSyncNode->pSyncRespMgr); pSyncNode->pSyncRespMgr = NULL; @@ -1287,31 +1302,30 @@ void syncNodeClose(SSyncNode* pSyncNode) { pSyncNode->pLogStore = NULL; syncLogBufferDestroy(pSyncNode->pLogBuf); pSyncNode->pLogBuf = NULL; - raftCfgClose(pSyncNode->pRaftCfg); - pSyncNode->pRaftCfg = NULL; syncNodeStopPingTimer(pSyncNode); syncNodeStopElectTimer(pSyncNode); syncNodeStopHeartbeatTimer(pSyncNode); for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { - if ((pSyncNode->senders)[i] != NULL) { - sSTrace((pSyncNode->senders)[i], "snapshot sender destroy while close, data:%p", (pSyncNode->senders)[i]); + if (pSyncNode->senders[i] != NULL) { + sDebug("vgId:%d, snapshot sender destroy while close, data:%p", pSyncNode->vgId, pSyncNode->senders[i]); - if (snapshotSenderIsStart((pSyncNode->senders)[i])) { - snapshotSenderStop((pSyncNode->senders)[i], false); + if (snapshotSenderIsStart(pSyncNode->senders[i])) { + snapshotSenderStop(pSyncNode->senders[i], false); } - snapshotSenderDestroy((pSyncNode->senders)[i]); - (pSyncNode->senders)[i] = NULL; + snapshotSenderDestroy(pSyncNode->senders[i]); + pSyncNode->senders[i] = NULL; } } if (pSyncNode->pNewNodeReceiver != NULL) { if (snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) { - snapshotReceiverForceStop(pSyncNode->pNewNodeReceiver); + snapshotReceiverStop(pSyncNode->pNewNodeReceiver); } + sDebug("vgId:%d, snapshot receiver destroy while close, data:%p", pSyncNode->vgId, pSyncNode->pNewNodeReceiver); snapshotReceiverDestroy(pSyncNode->pNewNodeReceiver); pSyncNode->pNewNodeReceiver = NULL; } @@ -1323,7 +1337,7 @@ void syncNodeClose(SSyncNode* pSyncNode) { taosMemoryFree(pSyncNode); } -ESyncStrategy syncNodeStrategy(SSyncNode* pSyncNode) { return pSyncNode->pRaftCfg->snapshotStrategy; } +ESyncStrategy syncNodeStrategy(SSyncNode* pSyncNode) { return pSyncNode->raftCfg.snapshotStrategy; } // timer control -------------- int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) { @@ -1382,20 +1396,19 @@ int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms) { return ret; } -int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode) { - int32_t ret = 0; +void syncNodeResetElectTimer(SSyncNode* pSyncNode) { int32_t electMS; - if (pSyncNode->pRaftCfg->isStandBy) { + if (pSyncNode->raftCfg.isStandBy) { electMS = TIMER_MAX_MS; } else { electMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine); } - ret = syncNodeRestartElectTimer(pSyncNode, electMS); + + (void)syncNodeRestartElectTimer(pSyncNode, electMS); sNTrace(pSyncNode, "reset elect timer, min:%d, max:%d, ms:%d", pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine, electMS); - return ret; } static int32_t syncNodeDoStartHeartbeatTimer(SSyncNode* pSyncNode) { @@ -1455,58 +1468,51 @@ int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode) { return 0; } -// utils -------------- -int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { - SEpSet epSet; - syncUtilRaftId2EpSet(destRaftId, &epSet); - if (pSyncNode->syncSendMSg != NULL) { - // htonl - syncUtilMsgHtoN(pMsg->pCont); - - pMsg->info.noResp = 1; - pSyncNode->syncSendMSg(&epSet, pMsg); - } else { - sError("vgId:%d, sync send msg by id error, fp-send-msg is null", pSyncNode->vgId); - rpcFreeCont(pMsg->pCont); - return -1; +int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pNode, SRpcMsg* pMsg) { + SEpSet* epSet = NULL; + for (int32_t i = 0; i < pNode->peersNum; ++i) { + if (destRaftId->addr == pNode->peersId[i].addr) { + epSet = &pNode->peersEpset[i]; + break; + } } - return 0; -} - -int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg) { - SEpSet epSet; - syncUtilNodeInfo2EpSet(nodeInfo, &epSet); - if (pSyncNode->syncSendMSg != NULL) { - // htonl + int32_t code = -1; + if (pNode->syncSendMSg != NULL && epSet != NULL) { syncUtilMsgHtoN(pMsg->pCont); - pMsg->info.noResp = 1; - pSyncNode->syncSendMSg(&epSet, pMsg); - } else { - sError("vgId:%d, sync send msg by info error, fp-send-msg is null", pSyncNode->vgId); + code = pNode->syncSendMSg(epSet, pMsg); } - return 0; + + if (code < 0) { + sError("vgId:%d, sync send msg by id error, epset:%p dnode:%d addr:%" PRId64 " err:0x%x", pNode->vgId, epSet, + DID(destRaftId), destRaftId->addr, terrno); + rpcFreeCont(pMsg->pCont); + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + } + + return code; } -inline bool syncNodeInConfig(SSyncNode* pSyncNode, const SSyncCfg* config) { +inline bool syncNodeInConfig(SSyncNode* pNode, const SSyncCfg* pCfg) { bool b1 = false; bool b2 = false; - for (int32_t i = 0; i < config->replicaNum; ++i) { - if (strcmp((config->nodeInfo)[i].nodeFqdn, pSyncNode->myNodeInfo.nodeFqdn) == 0 && - (config->nodeInfo)[i].nodePort == pSyncNode->myNodeInfo.nodePort) { + for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + if (strcmp(pCfg->nodeInfo[i].nodeFqdn, pNode->myNodeInfo.nodeFqdn) == 0 && + pCfg->nodeInfo[i].nodePort == pNode->myNodeInfo.nodePort) { b1 = true; break; } } - for (int32_t i = 0; i < config->replicaNum; ++i) { - SRaftId raftId; - raftId.addr = syncUtilAddr2U64((config->nodeInfo)[i].nodeFqdn, (config->nodeInfo)[i].nodePort); - raftId.vgId = pSyncNode->vgId; + for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + SRaftId raftId = { + .addr = SYNC_ADDR(&pCfg->nodeInfo[i]), + .vgId = pNode->vgId, + }; - if (syncUtilSameId(&raftId, &(pSyncNode->myRaftId))) { + if (syncUtilSameId(&raftId, &pNode->myRaftId)) { b2 = true; break; } @@ -1530,14 +1536,14 @@ static bool syncIsConfigChanged(const SSyncCfg* pOldCfg, const SSyncCfg* pNewCfg } void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex lastConfigChangeIndex) { - SSyncCfg oldConfig = pSyncNode->pRaftCfg->cfg; + SSyncCfg oldConfig = pSyncNode->raftCfg.cfg; if (!syncIsConfigChanged(&oldConfig, pNewConfig)) { sInfo("vgId:1, sync not reconfig since not changed"); return; } - pSyncNode->pRaftCfg->cfg = *pNewConfig; - pSyncNode->pRaftCfg->lastConfigIndex = lastConfigChangeIndex; + pSyncNode->raftCfg.cfg = *pNewConfig; + pSyncNode->raftCfg.lastConfigIndex = lastConfigChangeIndex; pSyncNode->configChangeNum++; @@ -1560,21 +1566,18 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } // log begin config change - char oldCfgStr[1024] = {0}; - char newCfgStr[1024] = {0}; - syncCfg2SimpleStr(&oldConfig, oldCfgStr, sizeof(oldCfgStr)); - syncCfg2SimpleStr(pNewConfig, oldCfgStr, sizeof(oldCfgStr)); - sNInfo(pSyncNode, "begin do config change, from %s to %s", oldCfgStr, oldCfgStr); + sNInfo(pSyncNode, "begin do config change, from %d to %d", pSyncNode->vgId, oldConfig.replicaNum, + pNewConfig->replicaNum); if (IamInNew) { - pSyncNode->pRaftCfg->isStandBy = 0; // change isStandBy to normal + pSyncNode->raftCfg.isStandBy = 0; // change isStandBy to normal } if (isDrop) { - pSyncNode->pRaftCfg->isStandBy = 1; // set standby + pSyncNode->raftCfg.isStandBy = 1; // set standby } // add last config index - raftCfgAddConfigIndex(pSyncNode->pRaftCfg, lastConfigChangeIndex); + syncAddCfgIndex(pSyncNode, lastConfigChangeIndex); if (IamInNew) { //----------------------------------------- @@ -1586,20 +1589,21 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde memcpy(oldReplicasId, pSyncNode->replicasId, sizeof(oldReplicasId)); SSyncSnapshotSender* oldSenders[TSDB_MAX_REPLICA]; for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { - oldSenders[i] = (pSyncNode->senders)[i]; + oldSenders[i] = pSyncNode->senders[i]; sSTrace(oldSenders[i], "snapshot sender save old"); } // init internal - pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex]; + pSyncNode->myNodeInfo = pSyncNode->raftCfg.cfg.nodeInfo[pSyncNode->raftCfg.cfg.myIndex]; syncUtilNodeInfo2RaftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId); // init peersNum, peers, peersId - pSyncNode->peersNum = pSyncNode->pRaftCfg->cfg.replicaNum - 1; + pSyncNode->peersNum = pSyncNode->raftCfg.cfg.replicaNum - 1; int32_t j = 0; - for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { - if (i != pSyncNode->pRaftCfg->cfg.myIndex) { - pSyncNode->peersNodeInfo[j] = pSyncNode->pRaftCfg->cfg.nodeInfo[i]; + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) { + if (i != pSyncNode->raftCfg.cfg.myIndex) { + pSyncNode->peersNodeInfo[j] = pSyncNode->raftCfg.cfg.nodeInfo[i]; + syncUtilNodeInfo2EpSet(&pSyncNode->peersNodeInfo[j], &pSyncNode->peersEpset[j]); j++; } } @@ -1608,13 +1612,13 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } // init replicaNum, replicasId - pSyncNode->replicaNum = pSyncNode->pRaftCfg->cfg.replicaNum; - for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { - syncUtilNodeInfo2RaftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]); + pSyncNode->replicaNum = pSyncNode->raftCfg.cfg.replicaNum; + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) { + syncUtilNodeInfo2RaftId(&pSyncNode->raftCfg.cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]); } // update quorum first - pSyncNode->quorum = syncUtilQuorum(pSyncNode->pRaftCfg->cfg.replicaNum); + pSyncNode->quorum = syncUtilQuorum(pSyncNode->raftCfg.cfg.replicaNum); syncIndexMgrUpdate(pSyncNode->pNextIndex, pSyncNode); syncIndexMgrUpdate(pSyncNode->pMatchIndex, pSyncNode); @@ -1625,7 +1629,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // clear new for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { - (pSyncNode->senders)[i] = NULL; + pSyncNode->senders[i] = NULL; } // reset new @@ -1634,22 +1638,19 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde bool reset = false; for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) { if (syncUtilSameId(&(pSyncNode->replicasId)[i], &oldReplicasId[j]) && oldSenders[j] != NULL) { - char host[128]; - uint16_t port; - syncUtilU642Addr((pSyncNode->replicasId)[i].addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "snapshot sender reset for: %" PRId64 ", newIndex:%d, %s:%d, %p", - (pSyncNode->replicasId)[i].addr, i, host, port, oldSenders[j]); + sNTrace(pSyncNode, "snapshot sender reset for:%" PRId64 ", newIndex:%d, dnode:%d, %p", + (pSyncNode->replicasId)[i].addr, i, DID(&pSyncNode->replicasId[i]), oldSenders[j]); - (pSyncNode->senders)[i] = oldSenders[j]; + pSyncNode->senders[i] = oldSenders[j]; oldSenders[j] = NULL; reset = true; // reset replicaIndex - int32_t oldreplicaIndex = (pSyncNode->senders)[i]->replicaIndex; - (pSyncNode->senders)[i]->replicaIndex = i; + int32_t oldreplicaIndex = pSyncNode->senders[i]->replicaIndex; + pSyncNode->senders[i]->replicaIndex = i; - sNTrace(pSyncNode, "snapshot sender udpate replicaIndex from %d to %d, %s:%d, %p, reset:%d", oldreplicaIndex, - i, host, port, (pSyncNode->senders)[i], reset); + sNTrace(pSyncNode, "snapshot sender udpate replicaIndex from %d to %d, dnode:%d, %p, reset:%d", + oldreplicaIndex, i, DID(&pSyncNode->replicasId[i]), pSyncNode->senders[i], reset); break; } @@ -1658,88 +1659,88 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // create new for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { - if ((pSyncNode->senders)[i] == NULL) { - (pSyncNode->senders)[i] = snapshotSenderCreate(pSyncNode, i); - sSTrace((pSyncNode->senders)[i], "snapshot sender create new while reconfig, data:%p", (pSyncNode->senders)[i]); + if (pSyncNode->senders[i] == NULL) { + pSyncNode->senders[i] = snapshotSenderCreate(pSyncNode, i); + if (pSyncNode->senders[i] == NULL) { + // will be created later while send snapshot + sSError(pSyncNode->senders[i], "snapshot sender create failed while reconfig"); + } else { + sSDebug(pSyncNode->senders[i], "snapshot sender create while reconfig, data:%p", pSyncNode->senders[i]); + } } else { - sSTrace((pSyncNode->senders)[i], "snapshot sender already exist, data:%p", (pSyncNode->senders)[i]); + sSDebug(pSyncNode->senders[i], "snapshot sender already exist, data:%p", pSyncNode->senders[i]); } } // free old for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { if (oldSenders[i] != NULL) { - sNTrace(pSyncNode, "snapshot sender destroy old, data:%p replica-index:%d", oldSenders[i], i); + sSDebug(oldSenders[i], "snapshot sender destroy old, data:%p replica-index:%d", oldSenders[i], i); snapshotSenderDestroy(oldSenders[i]); oldSenders[i] = NULL; } } // persist cfg - raftCfgPersist(pSyncNode->pRaftCfg); - - char tmpbuf[1024] = {0}; - snprintf(tmpbuf, sizeof(tmpbuf), "config change from %d to %d, index:%" PRId64 ", %s --> %s", - oldConfig.replicaNum, pNewConfig->replicaNum, lastConfigChangeIndex, oldCfgStr, newCfgStr); + syncWriteCfgFile(pSyncNode); // change isStandBy to normal (election timeout) if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { - syncNodeBecomeLeader(pSyncNode, tmpbuf); + syncNodeBecomeLeader(pSyncNode, ""); // Raft 3.6.2 Committing entries from previous terms syncNodeAppendNoop(pSyncNode); // syncMaybeAdvanceCommitIndex(pSyncNode); } else { - syncNodeBecomeFollower(pSyncNode, tmpbuf); + syncNodeBecomeFollower(pSyncNode, ""); } } else { // persist cfg - raftCfgPersist(pSyncNode->pRaftCfg); - sNInfo(pSyncNode, "do not config change from %d to %d, index:%" PRId64 ", %s --> %s", oldConfig.replicaNum, - pNewConfig->replicaNum, lastConfigChangeIndex, oldCfgStr, newCfgStr); + syncWriteCfgFile(pSyncNode); + sNInfo(pSyncNode, "do not config change from %d to %d", oldConfig.replicaNum, pNewConfig->replicaNum); } _END: // log end config change - sNInfo(pSyncNode, "end do config change, from %s to %s", oldCfgStr, newCfgStr); + sNInfo(pSyncNode, "end do config change, from %d to %d", oldConfig.replicaNum, pNewConfig->replicaNum); } // raft state change -------------- void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { - if (term > pSyncNode->pRaftStore->currentTerm) { - raftStoreSetTerm(pSyncNode->pRaftStore, term); + if (term > pSyncNode->raftStore.currentTerm) { + raftStoreSetTerm(pSyncNode, term); char tmpBuf[64]; snprintf(tmpBuf, sizeof(tmpBuf), "update term to %" PRId64, term); syncNodeBecomeFollower(pSyncNode, tmpBuf); - raftStoreClearVote(pSyncNode->pRaftStore); + raftStoreClearVote(pSyncNode); } } void syncNodeUpdateTermWithoutStepDown(SSyncNode* pSyncNode, SyncTerm term) { - if (term > pSyncNode->pRaftStore->currentTerm) { - raftStoreSetTerm(pSyncNode->pRaftStore, term); + if (term > pSyncNode->raftStore.currentTerm) { + raftStoreSetTerm(pSyncNode, term); } } void syncNodeStepDown(SSyncNode* pSyncNode, SyncTerm newTerm) { - if (pSyncNode->pRaftStore->currentTerm > newTerm) { + if (pSyncNode->raftStore.currentTerm > newTerm) { sNTrace(pSyncNode, "step down, ignore, new-term:%" PRId64 ", current-term:%" PRId64, newTerm, - pSyncNode->pRaftStore->currentTerm); + pSyncNode->raftStore.currentTerm); return; } do { sNTrace(pSyncNode, "step down, new-term:%" PRId64 ", current-term:%" PRId64, newTerm, - pSyncNode->pRaftStore->currentTerm); + pSyncNode->raftStore.currentTerm); } while (0); - if (pSyncNode->pRaftStore->currentTerm < newTerm) { - raftStoreSetTerm(pSyncNode->pRaftStore, newTerm); + if (pSyncNode->raftStore.currentTerm < newTerm) { + raftStoreSetTerm(pSyncNode, newTerm); char tmpBuf[64]; snprintf(tmpBuf, sizeof(tmpBuf), "step down, update term to %" PRId64, newTerm); syncNodeBecomeFollower(pSyncNode, tmpBuf); - raftStoreClearVote(pSyncNode->pRaftStore); + raftStoreClearVote(pSyncNode); } else { if (pSyncNode->state != TAOS_SYNC_STATE_FOLLOWER) { @@ -1844,8 +1845,8 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { SSyncSnapshotSender* pMySender = syncNodeGetSnapshotSender(pSyncNode, &(pSyncNode->myRaftId)); if (pMySender != NULL) { for (int32_t i = 0; i < pSyncNode->pMatchIndex->replicaNum; ++i) { - if ((pSyncNode->senders)[i]->privateTerm > pMySender->privateTerm) { - pMySender->privateTerm = (pSyncNode->senders)[i]->privateTerm; + if (pSyncNode->senders[i]->privateTerm > pMySender->privateTerm) { + pMySender->privateTerm = pSyncNode->senders[i]->privateTerm; } } (pMySender->privateTerm) += 100; @@ -1855,7 +1856,7 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { // close receiver if (pSyncNode != NULL && pSyncNode->pNewNodeReceiver != NULL && snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) { - snapshotReceiverForceStop(pSyncNode->pNewNodeReceiver); + snapshotReceiverStop(pSyncNode->pNewNodeReceiver); } // stop elect timer @@ -1897,7 +1898,7 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); ASSERT(lastIndex >= 0); sInfo("vgId:%d, become leader. term: %" PRId64 ", commit index: %" PRId64 ", last index: %" PRId64 "", - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, lastIndex); + pSyncNode->vgId, pSyncNode->raftStore.currentTerm, pSyncNode->commitIndex, lastIndex); } void syncNodeCandidate2LeaderOld(SSyncNode* pSyncNode) { @@ -1930,7 +1931,7 @@ void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { pSyncNode->state = TAOS_SYNC_STATE_CANDIDATE; SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); sInfo("vgId:%d, become candidate from follower. term: %" PRId64 ", commit index: %" PRId64 ", last index: %" PRId64, - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, lastIndex); + pSyncNode->vgId, pSyncNode->raftStore.currentTerm, pSyncNode->commitIndex, lastIndex); sNTrace(pSyncNode, "follower to candidate"); } @@ -1940,7 +1941,7 @@ void syncNodeLeader2Follower(SSyncNode* pSyncNode) { syncNodeBecomeFollower(pSyncNode, "leader to follower"); SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); sInfo("vgId:%d, become follower from leader. term: %" PRId64 ", commit index: %" PRId64 ", last index: %" PRId64, - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, lastIndex); + pSyncNode->vgId, pSyncNode->raftStore.currentTerm, pSyncNode->commitIndex, lastIndex); sNTrace(pSyncNode, "leader to follower"); } @@ -1950,7 +1951,7 @@ void syncNodeCandidate2Follower(SSyncNode* pSyncNode) { syncNodeBecomeFollower(pSyncNode, "candidate to follower"); SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); sInfo("vgId:%d, become follower from candidate. term: %" PRId64 ", commit index: %" PRId64 ", last index: %" PRId64, - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, lastIndex); + pSyncNode->vgId, pSyncNode->raftStore.currentTerm, pSyncNode->commitIndex, lastIndex); sNTrace(pSyncNode, "candidate to follower"); } @@ -1958,15 +1959,15 @@ void syncNodeCandidate2Follower(SSyncNode* pSyncNode) { // just called by syncNodeVoteForSelf // need assert void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId) { - ASSERT(term == pSyncNode->pRaftStore->currentTerm); - ASSERT(!raftStoreHasVoted(pSyncNode->pRaftStore)); + ASSERT(term == pSyncNode->raftStore.currentTerm); + ASSERT(!raftStoreHasVoted(pSyncNode)); - raftStoreVote(pSyncNode->pRaftStore, pRaftId); + raftStoreVote(pSyncNode, pRaftId); } // simulate get vote from outside void syncNodeVoteForSelf(SSyncNode* pSyncNode) { - syncNodeVoteForTerm(pSyncNode, pSyncNode->pRaftStore->currentTerm, &pSyncNode->myRaftId); + syncNodeVoteForTerm(pSyncNode, pSyncNode->raftStore.currentTerm, &pSyncNode->myRaftId); SRpcMsg rpcMsg = {0}; int32_t ret = syncBuildRequestVoteReply(&rpcMsg, pSyncNode->vgId); @@ -1975,7 +1976,7 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode) { SyncRequestVoteReply* pMsg = rpcMsg.pCont; pMsg->srcId = pSyncNode->myRaftId; pMsg->destId = pSyncNode->myRaftId; - pMsg->term = pSyncNode->pRaftStore->currentTerm; + pMsg->term = pSyncNode->raftStore.currentTerm; pMsg->voteGranted = true; voteGrantedVote(pSyncNode->pVotesGranted, pMsg); @@ -2265,13 +2266,6 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { return; } - if (pSyncNode->pRaftStore == NULL) { - syncNodeRelease(pSyncNode); - syncHbTimerDataRelease(pData); - sError("vgId:%d, hb timer raft store already stop", pSyncNode->vgId); - return; - } - // sTrace("vgId:%d, eq peer hb timer", pSyncNode->vgId); if (pSyncNode->replicaNum > 1) { @@ -2295,7 +2289,7 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { SyncHeartbeat* pSyncMsg = rpcMsg.pCont; pSyncMsg->srcId = pSyncNode->myRaftId; pSyncMsg->destId = pData->destId; - pSyncMsg->term = pSyncNode->pRaftStore->currentTerm; + pSyncMsg->term = pSyncNode->raftStore.currentTerm; pSyncMsg->commitIndex = pSyncNode->commitIndex; pSyncMsg->minMatchIndex = syncMinMatchIndex(pSyncNode); pSyncMsg->privateTerm = 0; @@ -2341,7 +2335,7 @@ static int32_t syncNodeEqNoop(SSyncNode* pNode) { } SyncIndex index = pNode->pLogStore->syncLogWriteIndex(pNode->pLogStore); - SyncTerm term = pNode->pRaftStore->currentTerm; + SyncTerm term = pNode->raftStore.currentTerm; SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, pNode->vgId); if (pEntry == NULL) return -1; @@ -2376,9 +2370,19 @@ int32_t syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHand } int32_t syncNodeAppend(SSyncNode* ths, SSyncRaftEntry* pEntry) { + if (pEntry->dataLen < sizeof(SMsgHead)) { + sError("vgId:%d, cannot append an invalid client request with no msg head. type:%s, dataLen:%d", ths->vgId, + TMSG_INFO(pEntry->originalRpcType), pEntry->dataLen); + syncEntryDestroy(pEntry); + return -1; + } + // append to log buffer if (syncLogBufferAppend(ths->pLogBuf, ths, pEntry) < 0) { - sError("vgId:%d, failed to enqueue sync log buffer. index:%" PRId64 "", ths->vgId, pEntry->index); + sError("vgId:%d, failed to enqueue sync log buffer, index:%" PRId64, ths->vgId, pEntry->index); + terrno = TSDB_CODE_SYN_BUFFER_FULL; + (void)syncLogFsmExecute(ths, ths->pFsm, ths->state, ths->raftStore.currentTerm, pEntry, TSDB_CODE_SYN_BUFFER_FULL); + syncEntryDestroy(pEntry); return -1; } @@ -2450,7 +2454,7 @@ bool syncNodeSnapshotRecving(SSyncNode* pSyncNode) { static int32_t syncNodeAppendNoop(SSyncNode* ths) { SyncIndex index = syncLogBufferGetEndIndex(ths->pLogBuf); - SyncTerm term = ths->pRaftStore->currentTerm; + SyncTerm term = ths->raftStore.currentTerm; SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, ths->vgId); if (pEntry == NULL) { @@ -2466,7 +2470,7 @@ static int32_t syncNodeAppendNoopOld(SSyncNode* ths) { int32_t ret = 0; SyncIndex index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore); - SyncTerm term = ths->pRaftStore->currentTerm; + SyncTerm term = ths->raftStore.currentTerm; SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, ths->vgId); ASSERT(pEntry != NULL); @@ -2508,12 +2512,12 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { SyncHeartbeatReply* pMsgReply = rpcMsg.pCont; pMsgReply->destId = pMsg->srcId; pMsgReply->srcId = ths->myRaftId; - pMsgReply->term = ths->pRaftStore->currentTerm; + pMsgReply->term = ths->raftStore.currentTerm; pMsgReply->privateTerm = 8864; // magic number pMsgReply->startTime = ths->startTime; pMsgReply->timeStamp = tsMs; - if (pMsg->term == ths->pRaftStore->currentTerm && ths->state != TAOS_SYNC_STATE_LEADER) { + if (pMsg->term == ths->raftStore.currentTerm && ths->state != TAOS_SYNC_STATE_LEADER) { syncIndexMgrSetRecvTime(ths->pNextIndex, &(pMsg->srcId), tsMs); syncNodeResetElectTimer(ths); @@ -2526,8 +2530,9 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { SyncLocalCmd* pSyncMsg = rpcMsgLocalCmd.pCont; pSyncMsg->cmd = SYNC_LOCAL_CMD_FOLLOWER_CMT; - pSyncMsg->fcIndex = pMsg->commitIndex; - SyncIndex fcIndex = pSyncMsg->fcIndex; + pSyncMsg->commitIndex = pMsg->commitIndex; + pSyncMsg->currentTerm = pMsg->term; + SyncIndex fcIndex = pSyncMsg->commitIndex; if (ths->syncEqMsg != NULL && ths->msgcb != NULL) { int32_t code = ths->syncEqMsg(ths->msgcb, &rpcMsgLocalCmd); @@ -2541,14 +2546,15 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { } } - if (pMsg->term >= ths->pRaftStore->currentTerm && ths->state != TAOS_SYNC_STATE_FOLLOWER) { + if (pMsg->term >= ths->raftStore.currentTerm && ths->state != TAOS_SYNC_STATE_FOLLOWER) { // syncNodeStepDown(ths, pMsg->term); SRpcMsg rpcMsgLocalCmd = {0}; (void)syncBuildLocalCmd(&rpcMsgLocalCmd, ths->vgId); SyncLocalCmd* pSyncMsg = rpcMsgLocalCmd.pCont; pSyncMsg->cmd = SYNC_LOCAL_CMD_STEP_DOWN; - pSyncMsg->sdNewTerm = pMsg->term; + pSyncMsg->currentTerm = pMsg->term; + pSyncMsg->commitIndex = pMsg->commitIndex; if (ths->syncEqMsg != NULL && ths->msgcb != NULL) { int32_t code = ths->syncEqMsg(ths->msgcb, &rpcMsgLocalCmd); @@ -2556,7 +2562,7 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { sError("vgId:%d, sync enqueue step-down msg error, code:%d", ths->vgId, code); rpcFreeCont(rpcMsgLocalCmd.pCont); } else { - sTrace("vgId:%d, sync enqueue step-down msg, new-term: %" PRId64, ths->vgId, pSyncMsg->sdNewTerm); + sTrace("vgId:%d, sync enqueue step-down msg, new-term: %" PRId64, ths->vgId, pSyncMsg->currentTerm); } } } @@ -2614,10 +2620,13 @@ int32_t syncNodeOnLocalCmd(SSyncNode* ths, const SRpcMsg* pRpcMsg) { syncLogRecvLocalCmd(ths, pMsg, ""); if (pMsg->cmd == SYNC_LOCAL_CMD_STEP_DOWN) { - syncNodeStepDown(ths, pMsg->sdNewTerm); + syncNodeStepDown(ths, pMsg->currentTerm); } else if (pMsg->cmd == SYNC_LOCAL_CMD_FOLLOWER_CMT) { - (void)syncNodeUpdateCommitIndex(ths, pMsg->fcIndex); + SyncTerm matchTerm = syncLogBufferGetLastMatchTerm(ths->pLogBuf); + if (pMsg->currentTerm == matchTerm) { + (void)syncNodeUpdateCommitIndex(ths, pMsg->commitIndex); + } if (syncLogBufferCommit(ths->pLogBuf, ths, ths->commitIndex) < 0) { sError("vgId:%d, failed to commit raft log since %s. commit index: %" PRId64 "", ths->vgId, terrstr(), ths->commitIndex); @@ -2630,14 +2639,15 @@ int32_t syncNodeOnLocalCmd(SSyncNode* ths, const SRpcMsg* pRpcMsg) { } int32_t syncNodeOnLocalCmdOld(SSyncNode* ths, const SRpcMsg* pRpcMsg) { + ASSERT(false && "deprecated"); SyncLocalCmd* pMsg = pRpcMsg->pCont; syncLogRecvLocalCmd(ths, pMsg, ""); if (pMsg->cmd == SYNC_LOCAL_CMD_STEP_DOWN) { - syncNodeStepDown(ths, pMsg->sdNewTerm); + syncNodeStepDown(ths, pMsg->currentTerm); } else if (pMsg->cmd == SYNC_LOCAL_CMD_FOLLOWER_CMT) { - syncNodeFollowerCommit(ths, pMsg->fcIndex); + syncNodeFollowerCommit(ths, pMsg->commitIndex); } else { sError("error local cmd"); @@ -2663,7 +2673,7 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIn int32_t code = 0; SyncIndex index = syncLogBufferGetEndIndex(ths->pLogBuf); - SyncTerm term = ths->pRaftStore->currentTerm; + SyncTerm term = ths->raftStore.currentTerm; SSyncRaftEntry* pEntry = NULL; if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { pEntry = syncEntryBuildFromClientRequest(pMsg->pCont, term, index); @@ -2671,19 +2681,23 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIn pEntry = syncEntryBuildFromRpcMsg(pMsg, term, index); } + if (pEntry == NULL) { + sError("vgId:%d, failed to process client request since %s.", ths->vgId, terrstr()); + return -1; + } + if (ths->state == TAOS_SYNC_STATE_LEADER) { if (pRetIndex) { (*pRetIndex) = index; } int32_t code = syncNodeAppend(ths, pEntry); - if (code < 0 && ths->vgId != 1 && vnodeIsMsgBlock(pEntry->originalRpcType)) { - ASSERTS(false, "failed to append blocking msg"); - } return code; + } else { + syncEntryDestroy(pEntry); + pEntry = NULL; + return -1; } - - return -1; } int32_t syncNodeOnClientRequestOld(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIndex) { @@ -2693,7 +2707,7 @@ int32_t syncNodeOnClientRequestOld(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRe int32_t code = 0; SyncIndex index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore); - SyncTerm term = ths->pRaftStore->currentTerm; + SyncTerm term = ths->raftStore.currentTerm; SSyncRaftEntry* pEntry; if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { @@ -2727,7 +2741,7 @@ int32_t syncNodeOnClientRequestOld(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRe .state = ths->state, .seqNum = pEntry->seqNum, .term = pEntry->term, - .currentTerm = ths->pRaftStore->currentTerm, + .currentTerm = ths->raftStore.currentTerm, .flag = 0, }; ths->pFsm->FpCommitCb(ths->pFsm, pMsg, &cbMeta); @@ -2805,7 +2819,7 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p return 0; } - if (pEntry->term < ths->pRaftStore->currentTerm) { + if (pEntry->term < ths->raftStore.currentTerm) { sNTrace(ths, "little term:%" PRId64 ", can not do leader transfer", pEntry->term); return 0; } @@ -2843,7 +2857,7 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p if (ths->pFsm->FpLeaderTransferCb != NULL) { SFsmCbMeta cbMeta = { .code = 0, - .currentTerm = ths->pRaftStore->currentTerm, + .currentTerm = ths->raftStore.currentTerm, .flag = 0, .index = pEntry->index, .lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, pEntry->index), @@ -2862,9 +2876,10 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p int32_t syncNodeUpdateNewConfigIndex(SSyncNode* ths, SSyncCfg* pNewCfg) { for (int32_t i = 0; i < pNewCfg->replicaNum; ++i) { - SRaftId raftId; - raftId.addr = syncUtilAddr2U64((pNewCfg->nodeInfo)[i].nodeFqdn, (pNewCfg->nodeInfo)[i].nodePort); - raftId.vgId = ths->vgId; + SRaftId raftId = { + .addr = SYNC_ADDR(&pNewCfg->nodeInfo[i]), + .vgId = ths->vgId, + }; if (syncUtilSameId(&(ths->myRaftId), &raftId)) { pNewCfg->myIndex = i; @@ -2958,7 +2973,7 @@ int32_t syncNodeDoCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endInde .state = ths->state, .seqNum = pEntry->seqNum, .term = pEntry->term, - .currentTerm = ths->pRaftStore->currentTerm, + .currentTerm = ths->raftStore.currentTerm, .flag = flag, }; diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 467b4e2219690ea447bec7a8fb9876f02a9858d4..7d534c671e6c80453f8a3b47fffebcf8028e062e 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -154,8 +154,8 @@ int32_t syncBuildAppendEntriesReply(SRpcMsg* pMsg, int32_t vgId) { return 0; } -int32_t syncBuildAppendEntriesFromRaftLog(SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevLogTerm, - SRpcMsg* pRpcMsg) { +int32_t syncBuildAppendEntriesFromRaftEntry(SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevLogTerm, + SRpcMsg* pRpcMsg) { uint32_t dataLen = pEntry->bytes; uint32_t bytes = sizeof(SyncAppendEntries) + dataLen; pRpcMsg->contLen = bytes; @@ -176,7 +176,7 @@ int32_t syncBuildAppendEntriesFromRaftLog(SSyncNode* pNode, SSyncRaftEntry* pEnt pMsg->prevLogTerm = prevLogTerm; pMsg->vgId = pNode->vgId; pMsg->srcId = pNode->myRaftId; - pMsg->term = pNode->pRaftStore->currentTerm; + pMsg->term = pNode->raftStore.currentTerm; pMsg->commitIndex = pNode->commitIndex; pMsg->privateTerm = 0; return 0; diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index f438856ace8091fad89f7e75c754d1c50e6089c4..b61fc2e90dc8cadd8265346a0bbe38e7a533238c 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -26,6 +26,15 @@ #include "syncSnapshot.h" #include "syncUtil.h" +static bool syncIsMsgBlock(tmsg_t type) { + return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || + (type == TDMT_VND_UPDATE_TAG_VAL) || (type == TDMT_VND_ALTER_CONFIRM); +} + +FORCE_INLINE static int64_t syncGetRetryMaxWaitMs() { + return SYNC_LOG_REPL_RETRY_WAIT_MS * (1 << SYNC_MAX_RETRY_BACKOFF); +} + int64_t syncLogBufferGetEndIndex(SSyncLogBuffer* pBuf) { taosThreadMutexLock(&pBuf->mutex); int64_t index = pBuf->endIndex; @@ -40,7 +49,7 @@ int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt if (index - pBuf->startIndex >= pBuf->size) { sError("vgId:%d, failed to append due to sync log buffer full. index:%" PRId64 "", pNode->vgId, index); - goto _out; + goto _err; } ASSERT(index == pBuf->endIndex); @@ -61,9 +70,8 @@ int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt taosThreadMutexUnlock(&pBuf->mutex); return 0; -_out: +_err: syncLogBufferValidate(pBuf); - syncEntryDestroy(pEntry); taosThreadMutexUnlock(&pBuf->mutex); return -1; } @@ -112,7 +120,7 @@ SyncTerm syncLogReplMgrGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, S return prevLogTerm; } - sError("vgId:%d, failed to get log term since %s. index: %" PRId64 "", pNode->vgId, terrstr(), prevIndex); + sInfo("vgId:%d, failed to get log term since %s. index:%" PRId64, pNode->vgId, terrstr(), prevIndex); terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; return -1; } @@ -260,20 +268,27 @@ int32_t syncLogBufferReInit(SSyncLogBuffer* pBuf, SSyncNode* pNode) { return ret; } -FORCE_INLINE SyncTerm syncLogBufferGetLastMatchTerm(SSyncLogBuffer* pBuf) { +FORCE_INLINE SyncTerm syncLogBufferGetLastMatchTermWithoutLock(SSyncLogBuffer* pBuf) { SyncIndex index = pBuf->matchIndex; SSyncRaftEntry* pEntry = pBuf->entries[(index + pBuf->size) % pBuf->size].pItem; ASSERT(pEntry != NULL); return pEntry->term; } +SyncTerm syncLogBufferGetLastMatchTerm(SSyncLogBuffer* pBuf) { + taosThreadMutexLock(&pBuf->mutex); + SyncTerm term = syncLogBufferGetLastMatchTermWithoutLock(pBuf); + taosThreadMutexUnlock(&pBuf->mutex); + return term; +} + int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevTerm) { taosThreadMutexLock(&pBuf->mutex); syncLogBufferValidate(pBuf); - int32_t ret = -1; - SyncIndex index = pEntry->index; - SyncIndex prevIndex = pEntry->index - 1; - SyncTerm lastMatchTerm = syncLogBufferGetLastMatchTerm(pBuf); + int32_t ret = -1; + SyncIndex index = pEntry->index; + SyncIndex prevIndex = pEntry->index - 1; + SyncTerm lastMatchTerm = syncLogBufferGetLastMatchTermWithoutLock(pBuf); SSyncRaftEntry* pExist = NULL; bool inBuf = true; @@ -325,6 +340,8 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt } // update + ASSERT(pBuf->startIndex < index); + ASSERT(index - pBuf->startIndex < pBuf->size); ASSERT(pBuf->entries[index % pBuf->size].pItem == NULL); SSyncLogBufEntry tmp = {.pItem = pEntry, .prevLogIndex = prevIndex, .prevLogTerm = prevTerm}; pEntry = NULL; @@ -441,26 +458,30 @@ _out: return matchIndex; } -int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry) { - ASSERTS(pFsm->FpCommitCb != NULL, "No commit cb registered for the FSM"); - +int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry, + int32_t applyCode) { if ((pNode->replicaNum == 1) && pNode->restoreFinish && pNode->vgId != 1) { return 0; } - if (pNode->vgId != 1 && vnodeIsMsgBlock(pEntry->originalRpcType)) { - sTrace("vgId:%d, blocking msg ready to execute. index:%" PRId64 ", term: %" PRId64 ", type: %s", pNode->vgId, - pEntry->index, pEntry->term, TMSG_INFO(pEntry->originalRpcType)); + if (pNode->vgId != 1 && syncIsMsgBlock(pEntry->originalRpcType)) { + sTrace("vgId:%d, blocking msg ready to execute, index:%" PRId64 ", term:%" PRId64 ", type:%s code:0x%x", + pNode->vgId, pEntry->index, pEntry->term, TMSG_INFO(pEntry->originalRpcType), applyCode); } - SRpcMsg rpcMsg = {0}; + if (pEntry->originalRpcType == TDMT_VND_COMMIT) { + sInfo("vgId:%d, fsm execute vnode commit. index: %" PRId64 ", term: %" PRId64 "", pNode->vgId, pEntry->index, + pEntry->term); + } + + SRpcMsg rpcMsg = {.code = applyCode}; syncEntry2OriginalRpc(pEntry, &rpcMsg); SFsmCbMeta cbMeta = {0}; cbMeta.index = pEntry->index; cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(pNode, pEntry->index); cbMeta.isWeak = pEntry->isWeak; - cbMeta.code = 0; + cbMeta.code = applyCode; cbMeta.state = role; cbMeta.seqNum = pEntry->seqNum; cbMeta.term = pEntry->term; @@ -469,7 +490,6 @@ int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, Syn (void)syncRespMgrGetAndDel(pNode->pSyncRespMgr, cbMeta.seqNum, &rpcMsg.info); int32_t code = pFsm->FpCommitCb(pFsm, &rpcMsg, &cbMeta); - ASSERT(rpcMsg.pCont == NULL); return code; } @@ -489,7 +509,7 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm SSyncLogStore* pLogStore = pNode->pLogStore; SSyncFSM* pFsm = pNode->pFsm; ESyncState role = pNode->state; - SyncTerm term = pNode->pRaftStore->currentTerm; + SyncTerm term = pNode->raftStore.currentTerm; SyncGroupId vgId = pNode->vgId; int32_t ret = -1; int64_t upperIndex = TMIN(commitIndex, pBuf->matchIndex); @@ -520,7 +540,7 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm pEntry->term, TMSG_INFO(pEntry->originalRpcType)); } - if (syncLogFsmExecute(pNode, pFsm, role, term, pEntry) != 0) { + if (syncLogFsmExecute(pNode, pFsm, role, term, pEntry, 0) != 0) { sError("vgId:%d, failed to execute sync log entry. index:%" PRId64 ", term:%" PRId64 ", role: %d, current term: %" PRId64, vgId, pEntry->index, pEntry->term, role, term); @@ -550,7 +570,8 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm ret = 0; _out: // mark as restored if needed - if (!pNode->restoreFinish && pBuf->commitIndex >= pNode->commitIndex) { + if (!pNode->restoreFinish && pBuf->commitIndex >= pNode->commitIndex && pEntry != NULL && + pNode->raftStore.currentTerm <= pEntry->term) { pNode->pFsm->FpRestoreFinishCb(pNode->pFsm); pNode->restoreFinish = true; sInfo("vgId:%d, restore finished. log buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, @@ -566,7 +587,9 @@ _out: return ret; } -int32_t syncLogReplMgrReset(SSyncLogReplMgr* pMgr) { +void syncLogReplMgrReset(SSyncLogReplMgr* pMgr) { + if (pMgr == NULL) return; + ASSERT(pMgr->startIndex >= 0); for (SyncIndex index = pMgr->startIndex; index < pMgr->endIndex; index++) { memset(&pMgr->states[index % pMgr->size], 0, sizeof(pMgr->states[0])); @@ -576,7 +599,6 @@ int32_t syncLogReplMgrReset(SSyncLogReplMgr* pMgr) { pMgr->endIndex = 0; pMgr->restored = false; pMgr->retryBackoff = 0; - return 0; } int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { @@ -592,9 +614,9 @@ int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { return -1; } - int32_t ret = -1; - bool retried = false; - int64_t retryWaitMs = syncLogGetRetryBackoffTimeMs(pMgr); + int32_t ret = -1; + bool retried = false; + int64_t retryWaitMs = syncLogGetRetryBackoffTimeMs(pMgr); int64_t nowMs = taosGetMonoTimestampMs(); int count = 0; int64_t firstIndex = -1; @@ -610,11 +632,17 @@ int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { } if (pMgr->states[pos].acked) { + if (pMgr->matchIndex < index && pMgr->states[pos].timeMs + (syncGetRetryMaxWaitMs() << 3) < nowMs) { + syncLogReplMgrReset(pMgr); + sWarn("vgId:%d, reset sync log repl mgr since stagnation. index: %" PRId64 ", peer: %" PRIx64, pNode->vgId, + index, pDestId->addr); + goto _out; + } continue; } bool barrier = false; - if (syncLogBufferReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { + if (syncLogReplMgrReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { sError("vgId:%d, failed to replicate sync log entry since %s. index: %" PRId64 ", dest: %" PRIx64 "", pNode->vgId, terrstr(), index, pDestId->addr); goto _out; @@ -636,31 +664,29 @@ int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { _out: if (retried) { pMgr->retryBackoff = syncLogGetNextRetryBackoff(pMgr); - sInfo("vgId:%d, resent %d sync log entries. dest: %" PRIx64 ", indexes: %" PRId64 " ..., terms: ... %" PRId64 - ", retryWaitMs: %" PRId64 ", repl mgr: [%" PRId64 " %" PRId64 ", %" PRId64 ")", + SSyncLogBuffer* pBuf = pNode->pLogBuf; + sInfo("vgId:%d, resend %d sync log entries. dest: %" PRIx64 ", indexes: %" PRId64 " ..., terms: ... %" PRId64 + ", retryWaitMs: %" PRId64 ", mgr: [%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 + " %" PRId64 ", %" PRId64 ")", pNode->vgId, count, pDestId->addr, firstIndex, term, retryWaitMs, pMgr->startIndex, pMgr->matchIndex, - pMgr->endIndex); + pMgr->endIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); } return ret; } -int32_t syncLogReplMgrProcessReplyInRecoveryMode(SSyncLogReplMgr* pMgr, SSyncNode* pNode, - SyncAppendEntriesReply* pMsg) { +int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { SSyncLogBuffer* pBuf = pNode->pLogBuf; SRaftId destId = pMsg->srcId; ASSERT(pMgr->restored == false); - char host[64]; - uint16_t port; - syncUtilU642Addr(destId.addr, host, sizeof(host), &port); if (pMgr->endIndex == 0) { ASSERT(pMgr->startIndex == 0); ASSERT(pMgr->matchIndex == 0); if (pMsg->matchIndex < 0) { pMgr->restored = true; - sInfo("vgId:%d, sync log repl mgr restored. peer: %s:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64 + sInfo("vgId:%d, sync log repl mgr restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", - pNode->vgId, host, port, destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, + pNode->vgId, DID(&destId), destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); return 0; } @@ -675,21 +701,21 @@ int32_t syncLogReplMgrProcessReplyInRecoveryMode(SSyncLogReplMgr* pMgr, SSyncNod if (pMsg->success && pMsg->matchIndex == pMsg->lastSendIndex) { pMgr->matchIndex = pMsg->matchIndex; pMgr->restored = true; - sInfo("vgId:%d, sync log repl mgr restored. peer: %s:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64 + sInfo("vgId:%d, sync log repl mgr restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", - pNode->vgId, host, port, destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, + pNode->vgId, DID(&destId), destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); return 0; } if (pMsg->success == false && pMsg->matchIndex >= pMsg->lastSendIndex) { - sWarn("vgId:%d, failed to rollback match index. peer: %s:%d, match index: %" PRId64 ", last sent: %" PRId64, - pNode->vgId, host, port, pMsg->matchIndex, pMsg->lastSendIndex); + sWarn("vgId:%d, failed to rollback match index. peer: dnode:%d, match index: %" PRId64 ", last sent: %" PRId64, + pNode->vgId, DID(&destId), pMsg->matchIndex, pMsg->lastSendIndex); if (syncNodeStartSnapshot(pNode, &destId) < 0) { - sError("vgId:%d, failed to start snapshot for peer %s:%d", pNode->vgId, host, port); + sError("vgId:%d, failed to start snapshot for peer dnode:%d", pNode->vgId, DID(&destId)); return -1; } - sInfo("vgId:%d, snapshot replication to peer %s:%d", pNode->vgId, host, port); + sInfo("vgId:%d, snapshot replication to peer dnode:%d", pNode->vgId, DID(&destId)); return 0; } } @@ -704,10 +730,10 @@ int32_t syncLogReplMgrProcessReplyInRecoveryMode(SSyncLogReplMgr* pMgr, SSyncNod if (term < 0 || (term != pMsg->lastMatchTerm && (index + 1 == firstVer || index == firstVer))) { ASSERT(term >= 0 || terrno == TSDB_CODE_WAL_LOG_NOT_EXIST); if (syncNodeStartSnapshot(pNode, &destId) < 0) { - sError("vgId:%d, failed to start snapshot for peer %s:%d", pNode->vgId, host, port); + sError("vgId:%d, failed to start snapshot for peer dnode:%d", pNode->vgId, DID(&destId)); return -1; } - sInfo("vgId:%d, snapshot replication to peer %s:%d", pNode->vgId, host, port); + sInfo("vgId:%d, snapshot replication to peer dnode:%d", pNode->vgId, DID(&destId)); return 0; } @@ -723,7 +749,7 @@ int32_t syncLogReplMgrProcessReplyInRecoveryMode(SSyncLogReplMgr* pMgr, SSyncNod // attempt to replicate the raft log at index (void)syncLogReplMgrReset(pMgr); - return syncLogReplMgrReplicateProbeOnce(pMgr, pNode, index); + return syncLogReplMgrReplicateProbe(pMgr, pNode, index); } int32_t syncLogReplMgrProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg) { @@ -751,9 +777,9 @@ int32_t syncLogReplMgrProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, Sync } if (pMgr->restored) { - (void)syncLogReplMgrProcessReplyInNormalMode(pMgr, pNode, pMsg); + (void)syncLogReplMgrProcessReplyAsNormal(pMgr, pNode, pMsg); } else { - (void)syncLogReplMgrProcessReplyInRecoveryMode(pMgr, pNode, pMsg); + (void)syncLogReplMgrProcessReplyAsRecovery(pMgr, pNode, pMsg); } taosThreadMutexUnlock(&pBuf->mutex); return 0; @@ -761,17 +787,17 @@ int32_t syncLogReplMgrProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, Sync int32_t syncLogReplMgrReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { if (pMgr->restored) { - (void)syncLogReplMgrReplicateAttemptedOnce(pMgr, pNode); + (void)syncLogReplMgrReplicateAttempt(pMgr, pNode); } else { - (void)syncLogReplMgrReplicateProbeOnce(pMgr, pNode, pNode->pLogBuf->matchIndex); + (void)syncLogReplMgrReplicateProbe(pMgr, pNode, pNode->pLogBuf->matchIndex); } return 0; } -int32_t syncLogReplMgrReplicateProbeOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { +int32_t syncLogReplMgrReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { ASSERT(!pMgr->restored); ASSERT(pMgr->startIndex >= 0); - int64_t retryMaxWaitMs = SYNC_LOG_REPL_RETRY_WAIT_MS * (1 << SYNC_MAX_RETRY_BACKOFF); + int64_t retryMaxWaitMs = syncGetRetryMaxWaitMs(); int64_t nowMs = taosGetMonoTimestampMs(); if (pMgr->endIndex > pMgr->startIndex && @@ -780,10 +806,10 @@ int32_t syncLogReplMgrReplicateProbeOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode } (void)syncLogReplMgrReset(pMgr); - SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; - bool barrier = false; - SyncTerm term = -1; - if (syncLogBufferReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { + SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; + bool barrier = false; + SyncTerm term = -1; + if (syncLogReplMgrReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { sError("vgId:%d, failed to replicate log entry since %s. index: %" PRId64 ", dest: 0x%016" PRIx64 "", pNode->vgId, terrstr(), index, pDestId->addr); return -1; @@ -799,25 +825,26 @@ int32_t syncLogReplMgrReplicateProbeOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode pMgr->endIndex = index + 1; SSyncLogBuffer* pBuf = pNode->pLogBuf; - sTrace("vgId:%d, attempted to probe the %d'th peer with msg of index:%" PRId64 " term: %" PRId64 - ". pMgr(rs:%d): [%" PRId64 " %" PRId64 ", %" PRId64 "), pBuf: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 - ")", - pNode->vgId, pMgr->peerId, index, term, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, - pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); + sInfo("vgId:%d, probe peer:%" PRIx64 " with msg of index:%" PRId64 " term: %" PRId64 ". mgr (rs:%d): [%" PRId64 + " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", + pNode->vgId, pDestId->addr, index, term, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, + pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); return 0; } -int32_t syncLogReplMgrReplicateAttemptedOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { +int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { ASSERT(pMgr->restored); - SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; - int32_t batchSize = TMAX(1, pMgr->size >> (4 + pMgr->retryBackoff)); - int32_t count = 0; - int64_t nowMs = taosGetMonoTimestampMs(); - int64_t limit = pMgr->size >> 1; + SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; + int32_t batchSize = TMAX(1, pMgr->size >> (4 + pMgr->retryBackoff)); + int32_t count = 0; + int64_t nowMs = taosGetMonoTimestampMs(); + int64_t limit = pMgr->size >> 1; + SyncTerm term = -1; + SyncIndex firstIndex = -1; for (SyncIndex index = pMgr->endIndex; index <= pNode->pLogBuf->matchIndex; index++) { - if (batchSize < count++ || limit <= index - pMgr->startIndex) { + if (batchSize < count || limit <= index - pMgr->startIndex) { break; } if (pMgr->startIndex + 1 < index && pMgr->states[(index - 1) % pMgr->size].barrier) { @@ -827,7 +854,7 @@ int32_t syncLogReplMgrReplicateAttemptedOnce(SSyncLogReplMgr* pMgr, SSyncNode* p SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; bool barrier = false; SyncTerm term = -1; - if (syncLogBufferReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { + if (syncLogReplMgrReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { sError("vgId:%d, failed to replicate log entry since %s. index: %" PRId64 ", dest: 0x%016" PRIx64 "", pNode->vgId, terrstr(), index, pDestId->addr); return -1; @@ -837,6 +864,9 @@ int32_t syncLogReplMgrReplicateAttemptedOnce(SSyncLogReplMgr* pMgr, SSyncNode* p pMgr->states[pos].term = term; pMgr->states[pos].acked = false; + if (firstIndex == -1) firstIndex = index; + count++; + pMgr->endIndex = index + 1; if (barrier) { sInfo("vgId:%d, replicated sync barrier to dest: %" PRIx64 ". index: %" PRId64 ", term: %" PRId64 @@ -850,23 +880,24 @@ int32_t syncLogReplMgrReplicateAttemptedOnce(SSyncLogReplMgr* pMgr, SSyncNode* p syncLogReplMgrRetryOnNeed(pMgr, pNode); SSyncLogBuffer* pBuf = pNode->pLogBuf; - sTrace("vgId:%d, attempted to replicate %d msgs to the %d'th peer. pMgr(rs:%d): [%" PRId64 " %" PRId64 ", %" PRId64 - "), pBuf: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", - pNode->vgId, count, pMgr->peerId, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, - pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); + sTrace("vgId:%d, replicated %d msgs to peer: %" PRIx64 ". indexes: %" PRId64 "..., terms: ...%" PRId64 + ", mgr: (rs:%d) [%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 + ")", + pNode->vgId, count, pDestId->addr, firstIndex, term, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, + pMgr->endIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); return 0; } -int32_t syncLogReplMgrProcessReplyInNormalMode(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { +int32_t syncLogReplMgrProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { ASSERT(pMgr->restored == true); if (pMgr->startIndex <= pMsg->lastSendIndex && pMsg->lastSendIndex < pMgr->endIndex) { - if (pMgr->startIndex < pMgr->matchIndex && pMgr->retryBackoff > 0) { - int64_t firstSentMs = pMgr->states[pMgr->startIndex % pMgr->size].timeMs; - int64_t lastSentMs = pMgr->states[(pMgr->endIndex - 1) % pMgr->size].timeMs; - int64_t timeDiffMs = lastSentMs - firstSentMs; - if (timeDiffMs > 0 && timeDiffMs < (SYNC_LOG_REPL_RETRY_WAIT_MS << (pMgr->retryBackoff - 1))) { - pMgr->retryBackoff -= 1; - } + if (pMgr->startIndex < pMgr->matchIndex && pMgr->retryBackoff > 0) { + int64_t firstSentMs = pMgr->states[pMgr->startIndex % pMgr->size].timeMs; + int64_t lastSentMs = pMgr->states[(pMgr->endIndex - 1) % pMgr->size].timeMs; + int64_t timeDiffMs = lastSentMs - firstSentMs; + if (timeDiffMs > 0 && timeDiffMs < (SYNC_LOG_REPL_RETRY_WAIT_MS << (pMgr->retryBackoff - 1))) { + pMgr->retryBackoff -= 1; + } } pMgr->states[pMsg->lastSendIndex % pMgr->size].acked = true; pMgr->matchIndex = TMAX(pMgr->matchIndex, pMsg->matchIndex); @@ -876,7 +907,7 @@ int32_t syncLogReplMgrProcessReplyInNormalMode(SSyncLogReplMgr* pMgr, SSyncNode* pMgr->startIndex = pMgr->matchIndex; } - return syncLogReplMgrReplicateAttemptedOnce(pMgr, pNode); + return syncLogReplMgrReplicateAttempt(pMgr, pNode); } SSyncLogReplMgr* syncLogReplMgrCreate() { @@ -985,6 +1016,10 @@ void syncLogBufferDestroy(SSyncLogBuffer* pBuf) { int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex toIndex) { ASSERT(pBuf->commitIndex < toIndex && toIndex <= pBuf->endIndex); + if (toIndex == pBuf->endIndex) { + return 0; + } + sInfo("vgId:%d, rollback sync log buffer. toindex: %" PRId64 ", buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, toIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); @@ -1066,12 +1101,11 @@ SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode, return pEntry; } -int32_t syncLogBufferReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, - SRaftId* pDestId, bool* pBarrier) { +int32_t syncLogReplMgrReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, + SRaftId* pDestId, bool* pBarrier) { SSyncRaftEntry* pEntry = NULL; SRpcMsg msgOut = {0}; bool inBuf = false; - int32_t ret = -1; SyncTerm prevLogTerm = -1; SSyncLogBuffer* pBuf = pNode->pLogBuf; @@ -1097,14 +1131,13 @@ int32_t syncLogBufferReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, Syn } if (pTerm) *pTerm = pEntry->term; - int32_t code = syncBuildAppendEntriesFromRaftLog(pNode, pEntry, prevLogTerm, &msgOut); + int32_t code = syncBuildAppendEntriesFromRaftEntry(pNode, pEntry, prevLogTerm, &msgOut); if (code < 0) { sError("vgId:%d, failed to get append entries for index:%" PRId64 "", pNode->vgId, index); goto _err; } (void)syncNodeSendAppendEntries(pNode, pDestId, &msgOut); - ret = 0; sTrace("vgId:%d, replicate one msg index: %" PRId64 " term: %" PRId64 " prevterm: %" PRId64 " to dest: 0x%016" PRIx64, pNode->vgId, pEntry->index, pEntry->term, prevLogTerm, pDestId->addr); diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index c609bfba93377e89fb8eeb41c3b098359efc93b3..806949c81e2ad8598a7d2901769c53a0bfa2daa8 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -16,382 +16,219 @@ #define _DEFAULT_SOURCE #include "syncRaftCfg.h" #include "syncUtil.h" - -// file must already exist! -SRaftCfgIndex *raftCfgIndexOpen(const char *path) { - SRaftCfgIndex *pRaftCfgIndex = taosMemoryMalloc(sizeof(SRaftCfg)); - snprintf(pRaftCfgIndex->path, sizeof(pRaftCfgIndex->path), "%s", path); - - pRaftCfgIndex->pFile = taosOpenFile(pRaftCfgIndex->path, TD_FILE_READ | TD_FILE_WRITE); - ASSERT(pRaftCfgIndex->pFile != NULL); - - taosLSeekFile(pRaftCfgIndex->pFile, 0, SEEK_SET); - - int32_t bufLen = MAX_CONFIG_INDEX_COUNT * 16; - char *pBuf = taosMemoryMalloc(bufLen); - memset(pBuf, 0, bufLen); - int64_t len = taosReadFile(pRaftCfgIndex->pFile, pBuf, bufLen); - ASSERT(len > 0); - - int32_t ret = raftCfgIndexFromStr(pBuf, pRaftCfgIndex); - ASSERT(ret == 0); - - taosMemoryFree(pBuf); - - return pRaftCfgIndex; -} - -int32_t raftCfgIndexClose(SRaftCfgIndex *pRaftCfgIndex) { - if (pRaftCfgIndex != NULL) { - int64_t ret = taosCloseFile(&(pRaftCfgIndex->pFile)); - ASSERT(ret == 0); - taosMemoryFree(pRaftCfgIndex); +#include "tjson.h" + +static int32_t syncEncodeSyncCfg(const void *pObj, SJson *pJson) { + SSyncCfg *pCfg = (SSyncCfg *)pObj; + if (tjsonAddDoubleToObject(pJson, "replicaNum", pCfg->replicaNum) < 0) return -1; + if (tjsonAddDoubleToObject(pJson, "myIndex", pCfg->myIndex) < 0) return -1; + + SJson *nodeInfo = tjsonCreateArray(); + if (nodeInfo == NULL) return -1; + if (tjsonAddItemToObject(pJson, "nodeInfo", nodeInfo) < 0) return -1; + for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + SJson *info = tjsonCreateObject(); + if (info == NULL) return -1; + if (tjsonAddDoubleToObject(info, "nodePort", pCfg->nodeInfo[i].nodePort) < 0) return -1; + if (tjsonAddStringToObject(info, "nodeFqdn", pCfg->nodeInfo[i].nodeFqdn) < 0) return -1; + if (tjsonAddIntegerToObject(info, "nodeId", pCfg->nodeInfo[i].nodeId) < 0) return -1; + if (tjsonAddIntegerToObject(info, "clusterId", pCfg->nodeInfo[i].clusterId) < 0) return -1; + if (tjsonAddItemToArray(nodeInfo, info) < 0) return -1; } - return 0; -} - -int32_t raftCfgIndexPersist(SRaftCfgIndex *pRaftCfgIndex) { - ASSERT(pRaftCfgIndex != NULL); - - char *s = raftCfgIndex2Str(pRaftCfgIndex); - taosLSeekFile(pRaftCfgIndex->pFile, 0, SEEK_SET); - - int64_t ret = taosWriteFile(pRaftCfgIndex->pFile, s, strlen(s) + 1); - ASSERT(ret == strlen(s) + 1); - - taosMemoryFree(s); - taosFsyncFile(pRaftCfgIndex->pFile); - return 0; -} -int32_t raftCfgIndexAddConfigIndex(SRaftCfgIndex *pRaftCfgIndex, SyncIndex configIndex) { - ASSERT(pRaftCfgIndex->configIndexCount <= MAX_CONFIG_INDEX_COUNT); - (pRaftCfgIndex->configIndexArr)[pRaftCfgIndex->configIndexCount] = configIndex; - ++(pRaftCfgIndex->configIndexCount); return 0; } -cJSON *raftCfgIndex2Json(SRaftCfgIndex *pRaftCfgIndex) { - cJSON *pRoot = cJSON_CreateObject(); - - cJSON_AddNumberToObject(pRoot, "configIndexCount", pRaftCfgIndex->configIndexCount); - cJSON *pIndexArr = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "configIndexArr", pIndexArr); - for (int i = 0; i < pRaftCfgIndex->configIndexCount; ++i) { - char buf64[128]; - snprintf(buf64, sizeof(buf64), "%" PRId64, (pRaftCfgIndex->configIndexArr)[i]); - cJSON *pIndexObj = cJSON_CreateObject(); - cJSON_AddStringToObject(pIndexObj, "index", buf64); - cJSON_AddItemToArray(pIndexArr, pIndexObj); - } - - cJSON *pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SRaftCfgIndex", pRoot); - return pJson; -} - -char *raftCfgIndex2Str(SRaftCfgIndex *pRaftCfgIndex) { - cJSON *pJson = raftCfgIndex2Json(pRaftCfgIndex); - char *serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -int32_t raftCfgIndexFromJson(const cJSON *pRoot, SRaftCfgIndex *pRaftCfgIndex) { - cJSON *pJson = cJSON_GetObjectItem(pRoot, "SRaftCfgIndex"); - - cJSON *pJsonConfigIndexCount = cJSON_GetObjectItem(pJson, "configIndexCount"); - pRaftCfgIndex->configIndexCount = cJSON_GetNumberValue(pJsonConfigIndexCount); - - cJSON *pIndexArr = cJSON_GetObjectItem(pJson, "configIndexArr"); - int arraySize = cJSON_GetArraySize(pIndexArr); - ASSERT(arraySize == pRaftCfgIndex->configIndexCount); - - memset(pRaftCfgIndex->configIndexArr, 0, sizeof(pRaftCfgIndex->configIndexArr)); - for (int i = 0; i < arraySize; ++i) { - cJSON *pIndexObj = cJSON_GetArrayItem(pIndexArr, i); - ASSERT(pIndexObj != NULL); - - cJSON *pIndex = cJSON_GetObjectItem(pIndexObj, "index"); - ASSERT(cJSON_IsString(pIndex)); - (pRaftCfgIndex->configIndexArr)[i] = atoll(pIndex->valuestring); +static int32_t syncEncodeRaftCfg(const void *pObj, SJson *pJson) { + SRaftCfg *pCfg = (SRaftCfg *)pObj; + if (tjsonAddObject(pJson, "SSyncCfg", syncEncodeSyncCfg, (void *)&pCfg->cfg) < 0) return -1; + if (tjsonAddDoubleToObject(pJson, "isStandBy", pCfg->isStandBy) < 0) return -1; + if (tjsonAddDoubleToObject(pJson, "snapshotStrategy", pCfg->snapshotStrategy) < 0) return -1; + if (tjsonAddDoubleToObject(pJson, "batchSize", pCfg->batchSize) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "lastConfigIndex", pCfg->lastConfigIndex) < 0) return -1; + if (tjsonAddDoubleToObject(pJson, "configIndexCount", pCfg->configIndexCount) < 0) return -1; + + SJson *configIndexArr = tjsonCreateArray(); + if (configIndexArr == NULL) return -1; + if (tjsonAddItemToObject(pJson, "configIndexArr", configIndexArr) < 0) return -1; + for (int32_t i = 0; i < pCfg->configIndexCount; ++i) { + SJson *configIndex = tjsonCreateObject(); + if (configIndex == NULL) return -1; + if (tjsonAddIntegerToObject(configIndex, "index", pCfg->configIndexArr[i]) < 0) return -1; + if (tjsonAddItemToArray(configIndexArr, configIndex) < 0) return -1; } return 0; } -int32_t raftCfgIndexFromStr(const char *s, SRaftCfgIndex *pRaftCfgIndex) { - cJSON *pRoot = cJSON_Parse(s); - ASSERT(pRoot != NULL); - - int32_t ret = raftCfgIndexFromJson(pRoot, pRaftCfgIndex); - ASSERT(ret == 0); - - cJSON_Delete(pRoot); - return 0; -} - -int32_t raftCfgIndexCreateFile(const char *path) { - TdFilePtr pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE); - if (pFile == NULL) { - int32_t err = terrno; - const char *errStr = tstrerror(err); - int32_t sysErr = errno; - const char *sysErrStr = strerror(errno); - sError("create raft cfg index file error, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", err, err, errStr, sysErr, - sysErrStr); - ASSERT(0); +int32_t syncWriteCfgFile(SSyncNode *pNode) { + int32_t code = -1; + char *buffer = NULL; + SJson *pJson = NULL; + TdFilePtr pFile = NULL; + const char *realfile = pNode->configPath; + SRaftCfg *pCfg = &pNode->raftCfg; + char file[PATH_MAX] = {0}; + snprintf(file, sizeof(file), "%s.bak", realfile); - return -1; - } + terrno = TSDB_CODE_OUT_OF_MEMORY; + pJson = tjsonCreateObject(); + if (pJson == NULL) goto _OVER; + if (tjsonAddObject(pJson, "RaftCfg", syncEncodeRaftCfg, pCfg) < 0) goto _OVER; + buffer = tjsonToString(pJson); + if (buffer == NULL) goto _OVER; + terrno = 0; - SRaftCfgIndex raftCfgIndex; - memset(raftCfgIndex.configIndexArr, 0, sizeof(raftCfgIndex.configIndexArr)); - raftCfgIndex.configIndexCount = 1; - raftCfgIndex.configIndexArr[0] = -1; + pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) goto _OVER; - char *s = raftCfgIndex2Str(&raftCfgIndex); - int64_t ret = taosWriteFile(pFile, s, strlen(s) + 1); - ASSERT(ret == strlen(s) + 1); + int32_t len = strlen(buffer); + if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER; + if (taosFsyncFile(pFile) < 0) goto _OVER; - taosMemoryFree(s); taosCloseFile(&pFile); - return 0; -} - -// --------------------------------------- -// file must already exist! -SRaftCfg *raftCfgOpen(const char *path) { - SRaftCfg *pCfg = taosMemoryMalloc(sizeof(SRaftCfg)); - snprintf(pCfg->path, sizeof(pCfg->path), "%s", path); - - pCfg->pFile = taosOpenFile(pCfg->path, TD_FILE_READ | TD_FILE_WRITE); - ASSERT(pCfg->pFile != NULL); - - taosLSeekFile(pCfg->pFile, 0, SEEK_SET); + if (taosRenameFile(file, realfile) != 0) goto _OVER; - char buf[CONFIG_FILE_LEN] = {0}; - int len = taosReadFile(pCfg->pFile, buf, sizeof(buf)); - ASSERT(len > 0); + code = 0; + sInfo("vgId:%d, succeed to write sync cfg file:%s, len:%d", pNode->vgId, realfile, len); - int32_t ret = raftCfgFromStr(buf, pCfg); - ASSERT(ret == 0); +_OVER: + if (pJson != NULL) tjsonDelete(pJson); + if (buffer != NULL) taosMemoryFree(buffer); + if (pFile != NULL) taosCloseFile(&pFile); - return pCfg; -} - -int32_t raftCfgClose(SRaftCfg *pRaftCfg) { - int64_t ret = taosCloseFile(&(pRaftCfg->pFile)); - ASSERT(ret == 0); - taosMemoryFree(pRaftCfg); - return 0; -} - -int32_t raftCfgPersist(SRaftCfg *pRaftCfg) { - ASSERT(pRaftCfg != NULL); - - char *s = raftCfg2Str(pRaftCfg); - taosLSeekFile(pRaftCfg->pFile, 0, SEEK_SET); - - char buf[CONFIG_FILE_LEN] = {0}; - memset(buf, 0, sizeof(buf)); - - if (strlen(s) + 1 > CONFIG_FILE_LEN) { - sError("too long config str:%s", s); - ASSERT(0); + if (code != 0) { + if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); + sError("vgId:%d, failed to write sync cfg file:%s since %s", pNode->vgId, realfile, terrstr()); } - - snprintf(buf, sizeof(buf), "%s", s); - int64_t ret = taosWriteFile(pRaftCfg->pFile, buf, sizeof(buf)); - ASSERT(ret == sizeof(buf)); - - // int64_t ret = taosWriteFile(pRaftCfg->pFile, s, strlen(s) + 1); - // ASSERT(ret == strlen(s) + 1); - - taosMemoryFree(s); - taosFsyncFile(pRaftCfg->pFile); - return 0; -} - -int32_t raftCfgAddConfigIndex(SRaftCfg *pRaftCfg, SyncIndex configIndex) { - ASSERT(pRaftCfg->configIndexCount <= MAX_CONFIG_INDEX_COUNT); - (pRaftCfg->configIndexArr)[pRaftCfg->configIndexCount] = configIndex; - ++(pRaftCfg->configIndexCount); - return 0; + return code; } -cJSON *syncCfg2Json(SSyncCfg *pSyncCfg) { - char u64buf[128] = {0}; - cJSON *pRoot = cJSON_CreateObject(); - - if (pSyncCfg != NULL) { - cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncCfg->replicaNum); - cJSON_AddNumberToObject(pRoot, "myIndex", pSyncCfg->myIndex); - - cJSON *pNodeInfoArr = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "nodeInfo", pNodeInfoArr); - for (int i = 0; i < pSyncCfg->replicaNum; ++i) { - cJSON *pNodeInfo = cJSON_CreateObject(); - cJSON_AddNumberToObject(pNodeInfo, "nodePort", ((pSyncCfg->nodeInfo)[i]).nodePort); - cJSON_AddStringToObject(pNodeInfo, "nodeFqdn", ((pSyncCfg->nodeInfo)[i]).nodeFqdn); - cJSON_AddItemToArray(pNodeInfoArr, pNodeInfo); - } +static int32_t syncDecodeSyncCfg(const SJson *pJson, void *pObj) { + SSyncCfg *pCfg = (SSyncCfg *)pObj; + int32_t code = 0; + + tjsonGetInt32ValueFromDouble(pJson, "replicaNum", pCfg->replicaNum, code); + if (code < 0) return -1; + tjsonGetInt32ValueFromDouble(pJson, "myIndex", pCfg->myIndex, code); + if (code < 0) return -1; + + SJson *nodeInfo = tjsonGetObjectItem(pJson, "nodeInfo"); + if (nodeInfo == NULL) return -1; + pCfg->replicaNum = tjsonGetArraySize(nodeInfo); + + for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + SJson *info = tjsonGetArrayItem(nodeInfo, i); + if (info == NULL) return -1; + tjsonGetUInt16ValueFromDouble(info, "nodePort", pCfg->nodeInfo[i].nodePort, code); + if (code < 0) return -1; + code = tjsonGetStringValue(info, "nodeFqdn", pCfg->nodeInfo[i].nodeFqdn); + if (code < 0) return -1; + tjsonGetNumberValue(info, "nodeId", pCfg->nodeInfo[i].nodeId, code); + tjsonGetNumberValue(info, "clusterId", pCfg->nodeInfo[i].clusterId, code); } - return pRoot; + return 0; } -int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg) { - memset(pSyncCfg, 0, sizeof(SSyncCfg)); - // cJSON *pJson = cJSON_GetObjectItem(pRoot, "SSyncCfg"); - const cJSON *pJson = pRoot; - - cJSON *pReplicaNum = cJSON_GetObjectItem(pJson, "replicaNum"); - ASSERT(cJSON_IsNumber(pReplicaNum)); - pSyncCfg->replicaNum = cJSON_GetNumberValue(pReplicaNum); +static int32_t syncDecodeRaftCfg(const SJson *pJson, void *pObj) { + SRaftCfg *pCfg = (SRaftCfg *)pObj; + int32_t code = 0; - cJSON *pMyIndex = cJSON_GetObjectItem(pJson, "myIndex"); - ASSERT(cJSON_IsNumber(pMyIndex)); - pSyncCfg->myIndex = cJSON_GetNumberValue(pMyIndex); + if (tjsonToObject(pJson, "SSyncCfg", syncDecodeSyncCfg, (void *)&pCfg->cfg) < 0) return -1; - cJSON *pNodeInfoArr = cJSON_GetObjectItem(pJson, "nodeInfo"); - int arraySize = cJSON_GetArraySize(pNodeInfoArr); - ASSERT(arraySize == pSyncCfg->replicaNum); + tjsonGetInt8ValueFromDouble(pJson, "isStandBy", pCfg->isStandBy, code); + if (code < 0) return -1; + tjsonGetInt8ValueFromDouble(pJson, "snapshotStrategy", pCfg->snapshotStrategy, code); + if (code < 0) return -1; + tjsonGetInt32ValueFromDouble(pJson, "batchSize", pCfg->batchSize, code); + if (code < 0) return -1; + tjsonGetNumberValue(pJson, "lastConfigIndex", pCfg->lastConfigIndex, code); + if (code < 0) return -1; + tjsonGetInt32ValueFromDouble(pJson, "configIndexCount", pCfg->configIndexCount, code); - for (int i = 0; i < arraySize; ++i) { - cJSON *pNodeInfo = cJSON_GetArrayItem(pNodeInfoArr, i); - ASSERT(pNodeInfo != NULL); + SJson *configIndexArr = tjsonGetObjectItem(pJson, "configIndexArr"); + if (configIndexArr == NULL) return -1; - cJSON *pNodePort = cJSON_GetObjectItem(pNodeInfo, "nodePort"); - ASSERT(cJSON_IsNumber(pNodePort)); - ((pSyncCfg->nodeInfo)[i]).nodePort = cJSON_GetNumberValue(pNodePort); - - cJSON *pNodeFqdn = cJSON_GetObjectItem(pNodeInfo, "nodeFqdn"); - ASSERT(cJSON_IsString(pNodeFqdn)); - snprintf(((pSyncCfg->nodeInfo)[i]).nodeFqdn, sizeof(((pSyncCfg->nodeInfo)[i]).nodeFqdn), "%s", - pNodeFqdn->valuestring); + pCfg->configIndexCount = tjsonGetArraySize(configIndexArr); + for (int32_t i = 0; i < pCfg->configIndexCount; ++i) { + SJson *configIndex = tjsonGetArrayItem(configIndexArr, i); + if (configIndex == NULL) return -1; + tjsonGetNumberValue(configIndex, "index", pCfg->configIndexArr[i], code); + if (code < 0) return -1; } return 0; } -cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) { - cJSON *pRoot = cJSON_CreateObject(); - cJSON_AddItemToObject(pRoot, "SSyncCfg", syncCfg2Json(&(pRaftCfg->cfg))); - cJSON_AddNumberToObject(pRoot, "isStandBy", pRaftCfg->isStandBy); - cJSON_AddNumberToObject(pRoot, "snapshotStrategy", pRaftCfg->snapshotStrategy); - cJSON_AddNumberToObject(pRoot, "batchSize", pRaftCfg->batchSize); - - char buf64[128]; - snprintf(buf64, sizeof(buf64), "%" PRId64, pRaftCfg->lastConfigIndex); - cJSON_AddStringToObject(pRoot, "lastConfigIndex", buf64); - - cJSON_AddNumberToObject(pRoot, "configIndexCount", pRaftCfg->configIndexCount); - cJSON *pIndexArr = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "configIndexArr", pIndexArr); - for (int i = 0; i < pRaftCfg->configIndexCount; ++i) { - snprintf(buf64, sizeof(buf64), "%" PRId64, (pRaftCfg->configIndexArr)[i]); - cJSON *pIndexObj = cJSON_CreateObject(); - cJSON_AddStringToObject(pIndexObj, "index", buf64); - cJSON_AddItemToArray(pIndexArr, pIndexObj); - } - - cJSON *pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "RaftCfg", pRoot); - return pJson; -} - -char *raftCfg2Str(SRaftCfg *pRaftCfg) { - cJSON *pJson = raftCfg2Json(pRaftCfg); - char *serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} +int32_t syncReadCfgFile(SSyncNode *pNode) { + int32_t code = -1; + TdFilePtr pFile = NULL; + char *pData = NULL; + SJson *pJson = NULL; + const char *file = pNode->configPath; + SRaftCfg *pCfg = &pNode->raftCfg; -int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path) { - TdFilePtr pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE); + pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - int32_t err = terrno; - const char *errStr = tstrerror(err); - int32_t sysErr = errno; - const char *sysErrStr = strerror(errno); - sError("create raft cfg file error, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", err, err, errStr, sysErr, sysErrStr); - return -1; + terrno = TAOS_SYSTEM_ERROR(errno); + sError("vgId:%d, failed to open sync cfg file:%s since %s", pNode->vgId, file, terrstr()); + goto _OVER; } - SRaftCfg raftCfg; - raftCfg.cfg = *pCfg; - raftCfg.isStandBy = meta.isStandBy; - raftCfg.batchSize = meta.batchSize; - raftCfg.snapshotStrategy = meta.snapshotStrategy; - raftCfg.lastConfigIndex = meta.lastConfigIndex; - raftCfg.configIndexCount = 1; - memset(raftCfg.configIndexArr, 0, sizeof(raftCfg.configIndexArr)); - raftCfg.configIndexArr[0] = -1; - char *s = raftCfg2Str(&raftCfg); - - char buf[CONFIG_FILE_LEN] = {0}; - memset(buf, 0, sizeof(buf)); - ASSERT(strlen(s) + 1 <= CONFIG_FILE_LEN); - snprintf(buf, sizeof(buf), "%s", s); - int64_t ret = taosWriteFile(pFile, buf, sizeof(buf)); - ASSERT(ret == sizeof(buf)); - - // int64_t ret = taosWriteFile(pFile, s, strlen(s) + 1); - // ASSERT(ret == strlen(s) + 1); - - taosMemoryFree(s); - taosCloseFile(&pFile); - return 0; -} - -int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) { - // memset(pRaftCfg, 0, sizeof(SRaftCfg)); - cJSON *pJson = cJSON_GetObjectItem(pRoot, "RaftCfg"); + int64_t size = 0; + if (taosFStatFile(pFile, &size, NULL) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + sError("vgId:%d, failed to fstat sync cfg file:%s since %s", pNode->vgId, file, terrstr()); + goto _OVER; + } - cJSON *pJsonIsStandBy = cJSON_GetObjectItem(pJson, "isStandBy"); - pRaftCfg->isStandBy = cJSON_GetNumberValue(pJsonIsStandBy); + pData = taosMemoryMalloc(size + 1); + if (pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } - cJSON *pJsonBatchSize = cJSON_GetObjectItem(pJson, "batchSize"); - pRaftCfg->batchSize = cJSON_GetNumberValue(pJsonBatchSize); + if (taosReadFile(pFile, pData, size) != size) { + terrno = TAOS_SYSTEM_ERROR(errno); + sError("vgId:%d, failed to read sync cfg file:%s since %s", pNode->vgId, file, terrstr()); + goto _OVER; + } - cJSON *pJsonSnapshotStrategy = cJSON_GetObjectItem(pJson, "snapshotStrategy"); - pRaftCfg->snapshotStrategy = cJSON_GetNumberValue(pJsonSnapshotStrategy); + pData[size] = '\0'; - cJSON *pJsonLastConfigIndex = cJSON_GetObjectItem(pJson, "lastConfigIndex"); - pRaftCfg->lastConfigIndex = atoll(cJSON_GetStringValue(pJsonLastConfigIndex)); + pJson = tjsonParse(pData); + if (pJson == NULL) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; + goto _OVER; + } - cJSON *pJsonConfigIndexCount = cJSON_GetObjectItem(pJson, "configIndexCount"); - pRaftCfg->configIndexCount = cJSON_GetNumberValue(pJsonConfigIndexCount); + if (tjsonToObject(pJson, "RaftCfg", syncDecodeRaftCfg, (void *)pCfg) < 0) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; + goto _OVER; + } - cJSON *pIndexArr = cJSON_GetObjectItem(pJson, "configIndexArr"); - int arraySize = cJSON_GetArraySize(pIndexArr); - ASSERT(arraySize == pRaftCfg->configIndexCount); + code = 0; + sInfo("vgId:%d, succceed to read sync cfg file %s", pNode->vgId, file); - memset(pRaftCfg->configIndexArr, 0, sizeof(pRaftCfg->configIndexArr)); - for (int i = 0; i < arraySize; ++i) { - cJSON *pIndexObj = cJSON_GetArrayItem(pIndexArr, i); - ASSERT(pIndexObj != NULL); +_OVER: + if (pData != NULL) taosMemoryFree(pData); + if (pJson != NULL) cJSON_Delete(pJson); + if (pFile != NULL) taosCloseFile(&pFile); - cJSON *pIndex = cJSON_GetObjectItem(pIndexObj, "index"); - ASSERT(cJSON_IsString(pIndex)); - (pRaftCfg->configIndexArr)[i] = atoll(pIndex->valuestring); + if (code != 0) { + sError("vgId:%d, failed to read sync cfg file:%s since %s", pNode->vgId, file, terrstr()); } - - cJSON *pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); - int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg)); - ASSERT(code == 0); - return code; } -int32_t raftCfgFromStr(const char *s, SRaftCfg *pRaftCfg) { - cJSON *pRoot = cJSON_Parse(s); - ASSERT(pRoot != NULL); - - int32_t ret = raftCfgFromJson(pRoot, pRaftCfg); - ASSERT(ret == 0); +int32_t syncAddCfgIndex(SSyncNode *pNode, SyncIndex cfgIndex) { + SRaftCfg *pCfg = &pNode->raftCfg; + if (pCfg->configIndexCount <= MAX_CONFIG_INDEX_COUNT) { + return -1; + } - cJSON_Delete(pRoot); + pCfg->configIndexArr[pCfg->configIndexCount] = cfgIndex; + pCfg->configIndexCount++; return 0; -} +} \ No newline at end of file diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 018ac5bb7da3764d2b54b6dfc7995ffe9d9a47d8..ca6d3c314fbeb63fe2eafa3b93bcdc08f936da3e 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -115,8 +115,8 @@ static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncI const char* sysErrStr = strerror(errno); sNError(pData->pSyncNode, - "wal restore from snapshot error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", - snapshotIndex, err, err, errStr, sysErr, sysErrStr); + "wal restore from snapshot error, index:%" PRId64 ", err:0x%x, msg:%s, syserr:%d, sysmsg:%s", snapshotIndex, + err, errStr, sysErr, sysErrStr); return -1; } @@ -192,6 +192,8 @@ SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { return SYNC_TERM_INVALID; } +static inline bool raftLogForceSync(SSyncRaftEntry* pEntry) { return (pEntry->originalRpcType == TDMT_VND_COMMIT); } + static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; @@ -212,13 +214,16 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr int32_t sysErr = errno; const char* sysErrStr = strerror(errno); - sNError(pData->pSyncNode, "wal write error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", - pEntry->index, err, err, errStr, sysErr, sysErrStr); + sNError(pData->pSyncNode, "wal write error, index:%" PRId64 ", err:0x%x, msg:%s, syserr:%d, sysmsg:%s", + pEntry->index, err, errStr, sysErr, sysErrStr); return -1; } ASSERT(pEntry->index == index); + bool forceSync = raftLogForceSync(pEntry); + walFsync(pWal, forceSync); + sNTrace(pData->pSyncNode, "write index:%" PRId64 ", type:%s, origin type:%s, elapsed:%" PRId64, pEntry->index, TMSG_INFO(pEntry->msgType), TMSG_INFO(pEntry->originalRpcType), tsElapsed); return 0; @@ -257,11 +262,11 @@ int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncR const char* sysErrStr = strerror(errno); if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { - sNTrace(pData->pSyncNode, "wal read not exist, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", index, - err, err, errStr, sysErr, sysErrStr); + sNTrace(pData->pSyncNode, "wal read not exist, index:%" PRId64 ", err:0x%x, msg:%s, syserr:%d, sysmsg:%s", index, + err, errStr, sysErr, sysErrStr); } else { - sNTrace(pData->pSyncNode, "wal read error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", index, - err, err, errStr, sysErr, sysErrStr); + sNTrace(pData->pSyncNode, "wal read error, index:%" PRId64 ", err:0x%x, msg:%s, syserr:%d, sysmsg:%s", index, err, + errStr, sysErr, sysErrStr); } /* @@ -312,37 +317,14 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - // need not truncate - SyncIndex wallastVer = walGetLastVer(pWal); - if (fromIndex > wallastVer) { - return 0; - } - - // need not truncate - SyncIndex walCommitVer = walGetCommittedVer(pWal); - if (fromIndex <= walCommitVer) { - return 0; - } - - // delete from cache - for (SyncIndex index = fromIndex; index <= wallastVer; ++index) { - SLRUCache* pCache = pData->pSyncNode->pLogStore->pCache; - LRUHandle* h = taosLRUCacheLookup(pCache, &index, sizeof(index)); - if (h) { - sNTrace(pData->pSyncNode, "cache delete index:%" PRId64, index); - - taosLRUCacheRelease(pData->pSyncNode->pLogStore->pCache, h, true); - } - } - int32_t code = walRollback(pWal, fromIndex); if (code != 0) { int32_t err = terrno; const char* errStr = tstrerror(err); int32_t sysErr = errno; const char* sysErrStr = strerror(errno); - sError("vgId:%d, wal truncate error, from-index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", - pData->pSyncNode->vgId, fromIndex, err, err, errStr, sysErr, sysErrStr); + sError("vgId:%d, wal truncate error, from-index:%" PRId64 ", err:0x%x, msg:%s, syserr:%d, sysmsg:%s", + pData->pSyncNode->vgId, fromIndex, err, errStr, sysErr, sysErrStr); } // event log @@ -392,8 +374,8 @@ int32_t raftLogUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { const char* errStr = tstrerror(err); int32_t sysErr = errno; const char* sysErrStr = strerror(errno); - sError("vgId:%d, wal update commit index error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", - pData->pSyncNode->vgId, index, err, err, errStr, sysErr, sysErrStr); + sError("vgId:%d, wal update commit index error, index:%" PRId64 ", err:0x%x, msg:%s, syserr:%d, sysmsg:%s", + pData->pSyncNode->vgId, index, err, errStr, sysErr, sysErrStr); return -1; } return 0; diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index e328ed3d31ab3f51297c93bdea781417e9e33999..197d1463fd80791403425adaed75ab31d380e4bb 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -16,163 +16,161 @@ #define _DEFAULT_SOURCE #include "syncRaftStore.h" #include "syncUtil.h" +#include "tjson.h" -// private function -static int32_t raftStoreInit(SRaftStore *pRaftStore); -static bool raftStoreFileExist(char *path); +static int32_t raftStoreDecode(const SJson *pJson, SRaftStore *pStore) { + int32_t code = 0; -// public function -SRaftStore *raftStoreOpen(const char *path) { - int32_t ret; + tjsonGetNumberValue(pJson, "current_term", pStore->currentTerm, code); + if (code < 0) return -1; + tjsonGetNumberValue(pJson, "vote_for_addr", pStore->voteFor.addr, code); + if (code < 0) return -1; + tjsonGetInt32ValueFromDouble(pJson, "vote_for_vgid", pStore->voteFor.vgId, code); + if (code < 0) return -1; - SRaftStore *pRaftStore = taosMemoryCalloc(1, sizeof(SRaftStore)); - if (pRaftStore == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - snprintf(pRaftStore->path, sizeof(pRaftStore->path), "%s", path); - if (!raftStoreFileExist(pRaftStore->path)) { - ret = raftStoreInit(pRaftStore); - ASSERT(ret == 0); - } - - char storeBuf[RAFT_STORE_BLOCK_SIZE] = {0}; - pRaftStore->pFile = taosOpenFile(path, TD_FILE_READ | TD_FILE_WRITE); - ASSERT(pRaftStore->pFile != NULL); - - int len = taosReadFile(pRaftStore->pFile, storeBuf, RAFT_STORE_BLOCK_SIZE); - ASSERT(len > 0); - - ret = raftStoreDeserialize(pRaftStore, storeBuf, len); - ASSERT(ret == 0); - - return pRaftStore; -} - -static int32_t raftStoreInit(SRaftStore *pRaftStore) { - ASSERT(pRaftStore != NULL); - - pRaftStore->pFile = taosOpenFile(pRaftStore->path, TD_FILE_CREATE | TD_FILE_WRITE); - ASSERT(pRaftStore->pFile != NULL); - - pRaftStore->currentTerm = 0; - pRaftStore->voteFor.addr = 0; - pRaftStore->voteFor.vgId = 0; - - int32_t ret = raftStorePersist(pRaftStore); - ASSERT(ret == 0); - - taosCloseFile(&pRaftStore->pFile); - return 0; -} - -int32_t raftStoreClose(SRaftStore *pRaftStore) { - if (pRaftStore == NULL) return 0; - - taosCloseFile(&pRaftStore->pFile); - taosMemoryFree(pRaftStore); - pRaftStore = NULL; return 0; } -int32_t raftStorePersist(SRaftStore *pRaftStore) { - ASSERT(pRaftStore != NULL); - - int32_t ret; - char storeBuf[RAFT_STORE_BLOCK_SIZE] = {0}; - ret = raftStoreSerialize(pRaftStore, storeBuf, sizeof(storeBuf)); - ASSERT(ret == 0); - - taosLSeekFile(pRaftStore->pFile, 0, SEEK_SET); +int32_t raftStoreReadFile(SSyncNode *pNode) { + int32_t code = -1; + TdFilePtr pFile = NULL; + char *pData = NULL; + SJson *pJson = NULL; + const char *file = pNode->raftStorePath; + SRaftStore *pStore = &pNode->raftStore; + + if (taosStatFile(file, NULL, NULL) < 0) { + sInfo("vgId:%d, raft store file:%s not exist, use default value", pNode->vgId, file); + pStore->currentTerm = 0; + pStore->voteFor.addr = 0; + pStore->voteFor.vgId = 0; + return raftStoreWriteFile(pNode); + } - ret = taosWriteFile(pRaftStore->pFile, storeBuf, sizeof(storeBuf)); - ASSERT(ret == RAFT_STORE_BLOCK_SIZE); + pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + sError("vgId:%d, failed to open raft store file:%s since %s", pNode->vgId, file, terrstr()); + goto _OVER; + } - taosFsyncFile(pRaftStore->pFile); - return 0; -} + int64_t size = 0; + if (taosFStatFile(pFile, &size, NULL) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + sError("vgId:%d, failed to fstat raft store file:%s since %s", pNode->vgId, file, terrstr()); + goto _OVER; + } -static bool raftStoreFileExist(char *path) { - bool b = taosStatFile(path, NULL, NULL) >= 0; - return b; -} + pData = taosMemoryMalloc(size + 1); + if (pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } -int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) { - ASSERT(pRaftStore != NULL); + if (taosReadFile(pFile, pData, size) != size) { + terrno = TAOS_SYSTEM_ERROR(errno); + sError("vgId:%d, failed to read raft store file:%s since %s", pNode->vgId, file, terrstr()); + goto _OVER; + } - cJSON *pRoot = cJSON_CreateObject(); + pData[size] = '\0'; - char u64Buf[128] = {0}; - snprintf(u64Buf, sizeof(u64Buf), "%" PRIu64 "", pRaftStore->currentTerm); - cJSON_AddStringToObject(pRoot, "current_term", u64Buf); + pJson = tjsonParse(pData); + if (pJson == NULL) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; + goto _OVER; + } - snprintf(u64Buf, sizeof(u64Buf), "%" PRIu64 "", pRaftStore->voteFor.addr); - cJSON_AddStringToObject(pRoot, "vote_for_addr", u64Buf); + if (raftStoreDecode(pJson, pStore) < 0) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; + goto _OVER; + } - cJSON_AddNumberToObject(pRoot, "vote_for_vgid", pRaftStore->voteFor.vgId); + code = 0; + sInfo("vgId:%d, succceed to read raft store file %s", pNode->vgId, file); - uint64_t u64 = pRaftStore->voteFor.addr; - char host[128] = {0}; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pRoot, "addr_host", host); - cJSON_AddNumberToObject(pRoot, "addr_port", port); +_OVER: + if (pData != NULL) taosMemoryFree(pData); + if (pJson != NULL) cJSON_Delete(pJson); + if (pFile != NULL) taosCloseFile(&pFile); - char *serialized = cJSON_Print(pRoot); - int len2 = strlen(serialized); - ASSERT(len2 < len); - memset(buf, 0, len); - snprintf(buf, len, "%s", serialized); - taosMemoryFree(serialized); + if (code != 0) { + sError("vgId:%d, failed to read raft store file:%s since %s", pNode->vgId, file, terrstr()); + } + return code; +} - cJSON_Delete(pRoot); +static int32_t raftStoreEncode(SJson *pJson, SRaftStore *pStore) { + if (tjsonAddIntegerToObject(pJson, "current_term", pStore->currentTerm) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "vote_for_addr", pStore->voteFor.addr) < 0) return -1; + if (tjsonAddDoubleToObject(pJson, "vote_for_vgid", pStore->voteFor.vgId) < 0) return -1; return 0; } -int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len) { - ASSERT(pRaftStore != NULL); - - ASSERT(len > 0 && len <= RAFT_STORE_BLOCK_SIZE); - cJSON *pRoot = cJSON_Parse(buf); - - cJSON *pCurrentTerm = cJSON_GetObjectItem(pRoot, "current_term"); - ASSERT(cJSON_IsString(pCurrentTerm)); - sscanf(pCurrentTerm->valuestring, "%" PRIu64 "", &(pRaftStore->currentTerm)); - - cJSON *pVoteForAddr = cJSON_GetObjectItem(pRoot, "vote_for_addr"); - ASSERT(cJSON_IsString(pVoteForAddr)); - sscanf(pVoteForAddr->valuestring, "%" PRIu64 "", &(pRaftStore->voteFor.addr)); - - cJSON *pVoteForVgid = cJSON_GetObjectItem(pRoot, "vote_for_vgid"); - pRaftStore->voteFor.vgId = pVoteForVgid->valueint; - - cJSON_Delete(pRoot); - return 0; +int32_t raftStoreWriteFile(SSyncNode *pNode) { + int32_t code = -1; + char *buffer = NULL; + SJson *pJson = NULL; + TdFilePtr pFile = NULL; + const char *realfile = pNode->raftStorePath; + SRaftStore *pStore = &pNode->raftStore; + char file[PATH_MAX] = {0}; + snprintf(file, sizeof(file), "%s.bak", realfile); + + terrno = TSDB_CODE_OUT_OF_MEMORY; + pJson = tjsonCreateObject(); + if (pJson == NULL) goto _OVER; + if (raftStoreEncode(pJson, pStore) != 0) goto _OVER; + buffer = tjsonToString(pJson); + if (buffer == NULL) goto _OVER; + terrno = 0; + + pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) goto _OVER; + + int32_t len = strlen(buffer); + if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER; + if (taosFsyncFile(pFile) < 0) goto _OVER; + + taosCloseFile(&pFile); + if (taosRenameFile(file, realfile) != 0) goto _OVER; + + code = 0; + sInfo("vgId:%d, succeed to write raft store file:%s, len:%d", pNode->vgId, realfile, len); + +_OVER: + if (pJson != NULL) tjsonDelete(pJson); + if (buffer != NULL) taosMemoryFree(buffer); + if (pFile != NULL) taosCloseFile(&pFile); + + if (code != 0) { + if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); + sError("vgId:%d, failed to write raft store file:%s since %s", pNode->vgId, realfile, terrstr()); + } + return code; } -bool raftStoreHasVoted(SRaftStore *pRaftStore) { - bool b = syncUtilEmptyId(&(pRaftStore->voteFor)); +bool raftStoreHasVoted(SSyncNode *pNode) { + bool b = syncUtilEmptyId(&pNode->raftStore.voteFor); return (!b); } -void raftStoreVote(SRaftStore *pRaftStore, SRaftId *pRaftId) { - ASSERT(!syncUtilEmptyId(pRaftId)); - pRaftStore->voteFor = *pRaftId; - raftStorePersist(pRaftStore); +void raftStoreVote(SSyncNode *pNode, SRaftId *pRaftId) { + pNode->raftStore.voteFor = *pRaftId; + (void)raftStoreWriteFile(pNode); } -void raftStoreClearVote(SRaftStore *pRaftStore) { - pRaftStore->voteFor = EMPTY_RAFT_ID; - raftStorePersist(pRaftStore); +void raftStoreClearVote(SSyncNode *pNode) { + pNode->raftStore.voteFor = EMPTY_RAFT_ID; + (void)raftStoreWriteFile(pNode); } -void raftStoreNextTerm(SRaftStore *pRaftStore) { - ++(pRaftStore->currentTerm); - raftStorePersist(pRaftStore); +void raftStoreNextTerm(SSyncNode *pNode) { + pNode->raftStore.currentTerm++; + (void)raftStoreWriteFile(pNode); } -void raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term) { - pRaftStore->currentTerm = term; - raftStorePersist(pRaftStore); +void raftStoreSetTerm(SSyncNode *pNode, SyncTerm term) { + pNode->raftStore.currentTerm = term; + (void)raftStoreWriteFile(pNode); } diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 0f56921ec77570e85c081837e596071aeecbe483..1aa476e84e03ab46a925085ff7792ca88b0af5b4 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -107,10 +107,7 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId, bool snapsh pMsg = rpcMsg.pCont; } else { - char host[64]; - uint16_t port; - syncUtilU642Addr(pDestId->addr, host, sizeof(host), &port); - sNError(pSyncNode, "replicate to %s:%d error, next-index:%" PRId64, host, port, nextIndex); + sNError(pSyncNode, "replicate to dnode:%d error, next-index:%" PRId64, DID(pDestId), nextIndex); return -1; } } @@ -125,7 +122,7 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId, bool snapsh ASSERT(pMsg != NULL); pMsg->srcId = pSyncNode->myRaftId; pMsg->destId = *pDestId; - pMsg->term = pSyncNode->pRaftStore->currentTerm; + pMsg->term = pSyncNode->raftStore.currentTerm; pMsg->prevLogIndex = preLogIndex; pMsg->prevLogTerm = preLogTerm; pMsg->commitIndex = pSyncNode->commitIndex; @@ -171,10 +168,7 @@ int32_t syncNodeReplicateOld(SSyncNode* pSyncNode) { SRaftId* pDestId = &(pSyncNode->peersId[i]); ret = syncNodeReplicateOne(pSyncNode, pDestId, true); if (ret != 0) { - char host[64]; - int16_t port; - syncUtilU642Addr(pDestId->addr, host, sizeof(host), &port); - sError("vgId:%d, do append entries error for %s:%d", pSyncNode->vgId, host, port); + sError("vgId:%d, do append entries error for dnode:%d", pSyncNode->vgId, DID(pDestId)); } } @@ -183,7 +177,6 @@ int32_t syncNodeReplicateOld(SSyncNode* pSyncNode) { int32_t syncNodeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg) { SyncAppendEntries* pMsg = pRpcMsg->pCont; - int32_t ret = 0; pMsg->destId = *destRaftId; syncNodeSendMsgById(destRaftId, pSyncNode, pRpcMsg); return 0; @@ -229,11 +222,7 @@ int32_t syncNodeMaybeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* dest if (syncNodeNeedSendAppendEntries(pSyncNode, destRaftId, pMsg)) { ret = syncNodeSendAppendEntries(pSyncNode, destRaftId, pRpcMsg); } else { - char logBuf[128]; - char host[64]; - int16_t port; - syncUtilU642Addr(destRaftId->addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "do not repcate to %s:%d for index:%" PRId64, host, port, pMsg->prevLogIndex + 1); + sNTrace(pSyncNode, "do not repcate to dnode:%d for index:%" PRId64, DID(destRaftId), pMsg->prevLogIndex + 1); rpcFreeCont(pRpcMsg->pCont); } @@ -256,7 +245,7 @@ int32_t syncNodeHeartbeatPeers(SSyncNode* pSyncNode) { SyncHeartbeat* pSyncMsg = rpcMsg.pCont; pSyncMsg->srcId = pSyncNode->myRaftId; pSyncMsg->destId = pSyncNode->peersId[i]; - pSyncMsg->term = pSyncNode->pRaftStore->currentTerm; + pSyncMsg->term = pSyncNode->raftStore.currentTerm; pSyncMsg->commitIndex = pSyncNode->commitIndex; pSyncMsg->minMatchIndex = syncMinMatchIndex(pSyncNode); pSyncMsg->privateTerm = 0; diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 773befe1e4fdec36c5bc21ac1a6d955dd6de3322..069ea2ea88b85c91846b975513dcd681944c7ace 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -44,21 +44,12 @@ // /\ UNCHANGED <> // -static bool syncNodeOnRequestVoteLogOK(SSyncNode* pSyncNode, SyncRequestVote* pMsg) { - SyncTerm myLastTerm = syncNodeGetLastTerm(pSyncNode); - SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); - - if (pMsg->lastLogIndex < pSyncNode->commitIndex) { - sNTrace(pSyncNode, - "logok:0, {my-lterm:%" PRIu64 ", my-lindex:%" PRId64 ", recv-lterm:%" PRIu64 ", recv-lindex:%" PRId64 - ", recv-term:%" PRIu64 "}", - myLastTerm, myLastIndex, pMsg->lastLogTerm, pMsg->lastLogIndex, pMsg->term); - - return false; - } +static bool syncNodeOnRequestVoteLogOK(SSyncNode* ths, SyncRequestVote* pMsg) { + SyncTerm myLastTerm = syncNodeGetLastTerm(ths); + SyncIndex myLastIndex = syncNodeGetLastIndex(ths); if (myLastTerm == SYNC_TERM_INVALID) { - sNTrace(pSyncNode, + sNTrace(ths, "logok:0, {my-lterm:%" PRIu64 ", my-lindex:%" PRId64 ", recv-lterm:%" PRIu64 ", recv-lindex:%" PRId64 ", recv-term:%" PRIu64 "}", myLastTerm, myLastIndex, pMsg->lastLogTerm, pMsg->lastLogIndex, pMsg->term); @@ -66,22 +57,29 @@ static bool syncNodeOnRequestVoteLogOK(SSyncNode* pSyncNode, SyncRequestVote* pM } if (pMsg->lastLogTerm > myLastTerm) { - sNTrace(pSyncNode, + sNTrace(ths, "logok:1, {my-lterm:%" PRIu64 ", my-lindex:%" PRId64 ", recv-lterm:%" PRIu64 ", recv-lindex:%" PRId64 ", recv-term:%" PRIu64 "}", myLastTerm, myLastIndex, pMsg->lastLogTerm, pMsg->lastLogIndex, pMsg->term); + + if (pMsg->lastLogIndex < ths->commitIndex) { + sNWarn(ths, + "logok:1, commit rollback required. {my-lterm:%" PRIu64 ", my-lindex:%" PRId64 ", recv-lterm:%" PRIu64 + ", recv-lindex:%" PRId64 ", recv-term:%" PRIu64 "}", + myLastTerm, myLastIndex, pMsg->lastLogTerm, pMsg->lastLogIndex, pMsg->term); + } return true; } if (pMsg->lastLogTerm == myLastTerm && pMsg->lastLogIndex >= myLastIndex) { - sNTrace(pSyncNode, + sNTrace(ths, "logok:1, {my-lterm:%" PRIu64 ", my-lindex:%" PRId64 ", recv-lterm:%" PRIu64 ", recv-lindex:%" PRId64 ", recv-term:%" PRIu64 "}", myLastTerm, myLastIndex, pMsg->lastLogTerm, pMsg->lastLogIndex, pMsg->term); return true; } - sNTrace(pSyncNode, + sNTrace(ths, "logok:0, {my-lterm:%" PRIu64 ", my-lindex:%" PRId64 ", recv-lterm:%" PRIu64 ", recv-lindex:%" PRId64 ", recv-term:%" PRIu64 "}", myLastTerm, myLastIndex, pMsg->lastLogTerm, pMsg->lastLogIndex, pMsg->term); @@ -93,7 +91,7 @@ int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) { SyncRequestVote* pMsg = pRpcMsg->pCont; // if already drop replica, do not process - if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) { + if (!syncNodeInRaftGroup(ths, &pMsg->srcId)) { syncLogRecvRequestVote(ths, pMsg, -1, "not in my config"); return -1; } @@ -101,21 +99,21 @@ int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) { bool logOK = syncNodeOnRequestVoteLogOK(ths, pMsg); // maybe update term - if (pMsg->term > ths->pRaftStore->currentTerm) { + if (pMsg->term > ths->raftStore.currentTerm) { syncNodeStepDown(ths, pMsg->term); // syncNodeUpdateTerm(ths, pMsg->term); } - ASSERT(pMsg->term <= ths->pRaftStore->currentTerm); + ASSERT(pMsg->term <= ths->raftStore.currentTerm); - bool grant = (pMsg->term == ths->pRaftStore->currentTerm) && logOK && - ((!raftStoreHasVoted(ths->pRaftStore)) || (syncUtilSameId(&(ths->pRaftStore->voteFor), &(pMsg->srcId)))); + bool grant = (pMsg->term == ths->raftStore.currentTerm) && logOK && + ((!raftStoreHasVoted(ths)) || (syncUtilSameId(&ths->raftStore.voteFor, &pMsg->srcId))); if (grant) { // maybe has already voted for pMsg->srcId // vote again, no harm - raftStoreVote(ths->pRaftStore, &(pMsg->srcId)); + raftStoreVote(ths, &(pMsg->srcId)); // candidate ? - syncNodeStepDown(ths, ths->pRaftStore->currentTerm); + syncNodeStepDown(ths, ths->raftStore.currentTerm); // forbid elect for this round syncNodeResetElectTimer(ths); @@ -129,7 +127,7 @@ int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) { SyncRequestVoteReply* pReply = rpcMsg.pCont; pReply->srcId = ths->myRaftId; pReply->destId = pMsg->srcId; - pReply->term = ths->pRaftStore->currentTerm; + pReply->term = ths->raftStore.currentTerm; pReply->voteGranted = grant; // trace log @@ -137,4 +135,4 @@ int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) { syncLogSendRequestVoteReply(ths, pReply, ""); syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); return 0; -} \ No newline at end of file +} diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index 563f475070a1ec63a3af73dcf9ed6205754b04b4..a0d6cbf597bb95c9d0e73b9cb9cde9f7c94d4a98 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -49,25 +49,25 @@ int32_t syncNodeOnRequestVoteReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { } // drop stale response - if (pMsg->term < ths->pRaftStore->currentTerm) { + if (pMsg->term < ths->raftStore.currentTerm) { syncLogRecvRequestVoteReply(ths, pMsg, "drop stale response"); return -1; } - // ASSERT(!(pMsg->term > ths->pRaftStore->currentTerm)); + // ASSERT(!(pMsg->term > ths->raftStore.currentTerm)); // no need this code, because if I receive reply.term, then I must have sent for that term. - // if (pMsg->term > ths->pRaftStore->currentTerm) { + // if (pMsg->term > ths->raftStore.currentTerm) { // syncNodeUpdateTerm(ths, pMsg->term); // } - if (pMsg->term > ths->pRaftStore->currentTerm) { + if (pMsg->term > ths->raftStore.currentTerm) { syncLogRecvRequestVoteReply(ths, pMsg, "error term"); syncNodeStepDown(ths, pMsg->term); return -1; } syncLogRecvRequestVoteReply(ths, pMsg, ""); - ASSERT(pMsg->term == ths->pRaftStore->currentTerm); + ASSERT(pMsg->term == ths->raftStore.currentTerm); // This tallies votes even when the current state is not Candidate, // but they won't be looked at, so it doesn't matter. diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index 503c56b1e7d00639680e57d44f7d1d31be34a089..9373eccaef6b27fd451fa77b360972e47cbea3d7 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -118,12 +118,12 @@ static void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp) { SRespStub *pStub = (SRespStub *)taosHashIterate(pObj->pRespHash, NULL); int cnt = 0; int sum = 0; - SSyncNode *pSyncNode = pObj->data; + SSyncNode *pNode = pObj->data; SArray *delIndexArray = taosArrayInit(4, sizeof(uint64_t)); if (delIndexArray == NULL) return; - sDebug("vgId:%d, resp manager begin clean by ttl", pSyncNode->vgId); + sDebug("vgId:%d, resp manager begin clean by ttl", pNode->vgId); while (pStub) { size_t len; void *key = taosHashGetKey(pStub, &len); @@ -140,20 +140,18 @@ static void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp) { .lastConfigIndex = SYNC_INDEX_INVALID, .isWeak = false, .code = TSDB_CODE_SYN_TIMEOUT, - .state = pSyncNode->state, + .state = pNode->state, .seqNum = *pSeqNum, .term = SYNC_TERM_INVALID, - .currentTerm = pSyncNode->pRaftStore->currentTerm, + .currentTerm = pNode->raftStore.currentTerm, .flag = 0, }; pStub->rpcMsg.pCont = NULL; pStub->rpcMsg.contLen = 0; - // TODO: and make rpcMsg body, call commit cb - // pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &pStub->rpcMsg, cbMeta); SRpcMsg rpcMsg = {.info = pStub->rpcMsg.info, .code = TSDB_CODE_SYN_TIMEOUT}; - sInfo("vgId:%d, message handle:%p expired, type:%s ahandle:%p", pSyncNode->vgId, rpcMsg.info.handle, + sInfo("vgId:%d, message handle:%p expired, type:%s ahandle:%p", pNode->vgId, rpcMsg.info.handle, TMSG_INFO(pStub->rpcMsg.msgType), rpcMsg.info.ahandle); rpcSendResponse(&rpcMsg); } @@ -162,12 +160,12 @@ static void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp) { } int32_t arraySize = taosArrayGetSize(delIndexArray); - sDebug("vgId:%d, resp manager end clean by ttl, sum:%d, cnt:%d, array-size:%d", pSyncNode->vgId, sum, cnt, arraySize); + sDebug("vgId:%d, resp manager end clean by ttl, sum:%d, cnt:%d, array-size:%d", pNode->vgId, sum, cnt, arraySize); for (int32_t i = 0; i < arraySize; ++i) { uint64_t *pSeqNum = taosArrayGet(delIndexArray, i); taosHashRemove(pObj->pRespHash, pSeqNum, sizeof(uint64_t)); - sDebug("vgId:%d, resp manager clean by ttl, seq:%" PRId64, pSyncNode->vgId, *pSeqNum); + sDebug("vgId:%d, resp manager clean by ttl, seq:%" PRId64, pNode->vgId, *pSeqNum); } taosArrayDestroy(delIndexArray); } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 30324c1113a8434604a4a97b3b0e87c6460125e6..880c76e4ddebe6ff443b59769fdcdb28a477a65f 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -43,19 +43,17 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI pSender->sendingMS = SYNC_SNAPSHOT_RETRY_MS; pSender->pSyncNode = pSyncNode; pSender->replicaIndex = replicaIndex; - pSender->term = pSyncNode->pRaftStore->currentTerm; + pSender->term = pSyncNode->raftStore.currentTerm; pSender->startTime = 0; pSender->endTime = 0; pSender->pSyncNode->pFsm->FpGetSnapshotInfo(pSender->pSyncNode->pFsm, &pSender->snapshot); pSender->finish = false; - sDebug("vgId:%d, snapshot sender create", pSender->pSyncNode->vgId); return pSender; } void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { if (pSender == NULL) return; - sDebug("vgId:%d, snapshot sender destroy", pSender->pSyncNode->vgId); // free current block if (pSender->pCurrentBlock != NULL) { @@ -76,12 +74,6 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { bool snapshotSenderIsStart(SSyncSnapshotSender *pSender) { return pSender->start; } int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { - if (snapshotSenderIsStart(pSender)) { - sSError(pSender, "vgId:%d, snapshot sender is already start"); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - return -1; - } - pSender->start = true; pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; @@ -96,9 +88,9 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { pSender->snapshot.lastApplyTerm = SYNC_TERM_INVALID; pSender->snapshot.lastConfigIndex = SYNC_INDEX_INVALID; - memset(&(pSender->lastConfig), 0, sizeof(pSender->lastConfig)); + memset(&pSender->lastConfig, 0, sizeof(pSender->lastConfig)); pSender->sendingMS = 0; - pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; + pSender->term = pSender->pSyncNode->raftStore.currentTerm; pSender->startTime = taosGetTimestampMs(); pSender->lastSendTime = pSender->startTime; pSender->finish = false; @@ -112,8 +104,8 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { SyncSnapshotSend *pMsg = rpcMsg.pCont; pMsg->srcId = pSender->pSyncNode->myRaftId; - pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; - pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; + pMsg->destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; + pMsg->term = pSender->pSyncNode->raftStore.currentTerm; pMsg->beginIndex = pSender->snapshotParam.start; pMsg->lastIndex = pSender->snapshot.lastApplyIndex; pMsg->lastTerm = pSender->snapshot.lastApplyTerm; @@ -123,7 +115,6 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { pMsg->seq = SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT; // event log - sSDebug(pSender, "snapshot sender start"); syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, "snapshot sender start"); // send msg @@ -135,7 +126,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { return 0; } -int32_t snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) { +void snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) { sSDebug(pSender, "snapshot sender stop, finish:%d reader:%p", finish, pSender->pReader); // update flag @@ -155,13 +146,11 @@ int32_t snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) { pSender->pCurrentBlock = NULL; pSender->blockLen = 0; } - - return 0; } // when sender receive ack, call this function to send msg from seq // seq = ack + 1, already updated -int32_t snapshotSend(SSyncSnapshotSender *pSender) { +static int32_t snapshotSend(SSyncSnapshotSender *pSender) { // free memory last time (current seq - 1) if (pSender->pCurrentBlock != NULL) { taosMemoryFree(pSender->pCurrentBlock); @@ -178,8 +167,8 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { } if (pSender->blockLen > 0) { - sSDebug(pSender, "snapshot sender continue to read, blockLen:%d seq:%d", pSender->blockLen, pSender->seq); // has read data + sSDebug(pSender, "snapshot sender continue to read, blockLen:%d seq:%d", pSender->blockLen, pSender->seq); } else { // read finish, update seq to end pSender->seq = SYNC_SNAPSHOT_SEQ_END; @@ -195,15 +184,14 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { SyncSnapshotSend *pMsg = rpcMsg.pCont; pMsg->srcId = pSender->pSyncNode->myRaftId; - pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; - pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; + pMsg->destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; + pMsg->term = pSender->pSyncNode->raftStore.currentTerm; pMsg->beginIndex = pSender->snapshotParam.start; pMsg->lastIndex = pSender->snapshot.lastApplyIndex; pMsg->lastTerm = pSender->snapshot.lastApplyTerm; pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex; pMsg->lastConfig = pSender->lastConfig; pMsg->seq = pSender->seq; - // pMsg->privateTerm = pSender->privateTerm; if (pSender->pCurrentBlock != NULL) { memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); @@ -211,10 +199,8 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { // event log if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) { - sSDebug(pSender, "snapshot sender finish, seq:%d", pSender->seq); syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, "snapshot sender finish"); } else { - sSDebug(pSender, "snapshot sender sending, seq:%d", pSender->seq); syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, "snapshot sender sending"); } @@ -239,8 +225,8 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { SyncSnapshotSend *pMsg = rpcMsg.pCont; pMsg->srcId = pSender->pSyncNode->myRaftId; - pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; - pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; + pMsg->destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; + pMsg->term = pSender->pSyncNode->raftStore.currentTerm; pMsg->beginIndex = pSender->snapshotParam.start; pMsg->lastIndex = pSender->snapshot.lastApplyIndex; pMsg->lastTerm = pSender->snapshot.lastApplyTerm; @@ -249,12 +235,10 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { pMsg->seq = pSender->seq; if (pSender->pCurrentBlock != NULL && pSender->blockLen > 0) { - // pMsg->privateTerm = pSender->privateTerm; memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); } // event log - sSDebug(pSender, "snapshot sender resend, seq:%d", pSender->seq); syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, "snapshot sender resend"); // send msg @@ -294,19 +278,16 @@ int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) { } if (snapshotSenderIsStart(pSender)) { - sSError(pSender, "snapshot sender already start, ignore"); + sSInfo(pSender, "snapshot sender already start, ignore"); return 0; } if (pSender->finish && taosGetTimestampMs() - pSender->endTime < SNAPSHOT_WAIT_MS) { sSInfo(pSender, "snapshot sender start too frequently, ignore"); - return 1; + return 0; } - char host[64]; - uint16_t port; - syncUtilU642Addr(pDestId->addr, host, sizeof(host), &port); - sSInfo(pSender, "snapshot sender start for peer:%s:%u", host, port); + sSInfo(pSender, "snapshot sender start"); int32_t code = snapshotSenderStart(pSender); if (code != 0) { @@ -333,19 +314,17 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId from pReceiver->pWriter = NULL; pReceiver->pSyncNode = pSyncNode; pReceiver->fromId = fromId; - pReceiver->term = pSyncNode->pRaftStore->currentTerm; + pReceiver->term = pSyncNode->raftStore.currentTerm; pReceiver->snapshot.data = NULL; pReceiver->snapshot.lastApplyIndex = SYNC_INDEX_INVALID; pReceiver->snapshot.lastApplyTerm = 0; pReceiver->snapshot.lastConfigIndex = SYNC_INDEX_INVALID; - sDebug("vgId:%d, snapshot receiver create", pSyncNode->vgId); return pReceiver; } void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) { if (pReceiver == NULL) return; - sDebug("vgId:%d, snapshot receiver destroy", pReceiver->pSyncNode->vgId); // close writer if (pReceiver->pWriter != NULL) { @@ -363,31 +342,7 @@ void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) { bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { return pReceiver->start; } -// force stop -void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) { - sRInfo(pReceiver, "snapshot receiver force stop, writer:%p"); - - // force close, abandon incomplete data - if (pReceiver->pWriter != NULL) { - // event log - int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false, - &pReceiver->snapshot); - if (ret != 0) { - sRInfo(pReceiver, "snapshot receiver force stop failed since %s", terrstr()); - } - pReceiver->pWriter = NULL; - } - - pReceiver->start = false; -} - -int32_t snapshotReceiverStartWriter(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg) { - if (!snapshotReceiverIsStart(pReceiver)) { - sRError(pReceiver, "snapshot receiver is not start"); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - return -1; - } - +static int32_t snapshotReceiverStartWriter(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg) { if (pReceiver->pWriter != NULL) { sRError(pReceiver, "vgId:%d, snapshot receiver writer is not null"); terrno = TSDB_CODE_SYN_INTERNAL_ERROR; @@ -417,26 +372,25 @@ int32_t snapshotReceiverStartWriter(SSyncSnapshotReceiver *pReceiver, SyncSnapsh return 0; } -int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pPreMsg) { +void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pPreMsg) { if (snapshotReceiverIsStart(pReceiver)) { sRInfo(pReceiver, "snapshot receiver has started"); - return 0; + return; } pReceiver->start = true; pReceiver->ack = SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT; - pReceiver->term = pReceiver->pSyncNode->pRaftStore->currentTerm; + pReceiver->term = pReceiver->pSyncNode->raftStore.currentTerm; pReceiver->fromId = pPreMsg->srcId; pReceiver->startTime = pPreMsg->startTime; // event log sRInfo(pReceiver, "snapshot receiver is start"); - return 0; } // just set start = false -// FpSnapshotStopWrite should not be called, assert writer == NULL -int32_t snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { +// FpSnapshotStopWrite should not be called +void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { sRInfo(pReceiver, "snapshot receiver stop, not apply, writer:%p", pReceiver->pWriter); if (pReceiver->pWriter != NULL) { @@ -451,17 +405,10 @@ int32_t snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { } pReceiver->start = false; - return 0; } // when recv last snapshot block, apply data into snapshot static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) { - if (pMsg->seq != SYNC_SNAPSHOT_SEQ_END) { - sRError(pReceiver, "snapshot receiver seq:%d is invalid", pMsg->seq); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - return -1; - } - int32_t code = 0; if (pReceiver->pWriter != NULL) { // write data @@ -490,9 +437,9 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap } // maybe update term - if (pReceiver->snapshot.lastApplyTerm > pReceiver->pSyncNode->pRaftStore->currentTerm) { - pReceiver->pSyncNode->pRaftStore->currentTerm = pReceiver->snapshot.lastApplyTerm; - raftStorePersist(pReceiver->pSyncNode->pRaftStore); + if (pReceiver->snapshot.lastApplyTerm > pReceiver->pSyncNode->raftStore.currentTerm) { + pReceiver->pSyncNode->raftStore.currentTerm = pReceiver->snapshot.lastApplyTerm; + (void)raftStoreWriteFile(pReceiver->pSyncNode); } // stop writer, apply data @@ -523,7 +470,7 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap static int32_t snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) { if (pMsg->seq != pReceiver->ack + 1) { sRError(pReceiver, "snapshot receiver invalid seq, ack:%d seq:%d", pReceiver->ack, pMsg->seq); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + terrno = TSDB_CODE_SYN_INVALID_SNAPSHOT_MSG; return -1; } @@ -583,6 +530,7 @@ SyncIndex syncNodeGetSnapBeginIndex(SSyncNode *ths) { static int32_t syncNodeOnSnapshotPre(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; int64_t timeNow = taosGetTimestampMs(); + int32_t code = 0; if (snapshotReceiverIsStart(pReceiver)) { // already start @@ -594,14 +542,14 @@ static int32_t syncNodeOnSnapshotPre(SSyncNode *pSyncNode, SyncSnapshotSend *pMs sRInfo(pReceiver, "snapshot receiver startTime:%" PRId64 " == msg startTime:%" PRId64 " send reply", pReceiver->startTime, pMsg->startTime); goto _SEND_REPLY; - } else { // ignore - sRInfo(pReceiver, "snapshot receiver startTime:%" PRId64 " < msg startTime:%" PRId64 " ignore", - pReceiver->startTime, pMsg->startTime); - return 0; + sRError(pReceiver, "snapshot receiver startTime:%" PRId64 " < msg startTime:%" PRId64 " ignore", + pReceiver->startTime, pMsg->startTime); + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + code = terrno; + goto _SEND_REPLY; } - } else { // start new sRInfo(pReceiver, "snapshot receiver not start yet so start new one"); @@ -612,7 +560,8 @@ _START_RECEIVER: if (timeNow - pMsg->startTime > SNAPSHOT_MAX_CLOCK_SKEW_MS) { sRError(pReceiver, "snapshot receiver time skew too much, now:%" PRId64 " msg startTime:%" PRId64, timeNow, pMsg->startTime); - return -1; + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + code = terrno; } else { // waiting for clock match while (timeNow < pMsg->startTime) { @@ -624,7 +573,7 @@ _START_RECEIVER: if (snapshotReceiverIsStart(pReceiver)) { sRInfo(pReceiver, "snapshot receiver already start and force stop pre one"); - snapshotReceiverForceStop(pReceiver); + snapshotReceiverStop(pReceiver); } snapshotReceiverStart(pReceiver, pMsg); // set start-time same with sender @@ -643,12 +592,12 @@ _SEND_REPLY: SyncSnapshotRsp *pRspMsg = rpcMsg.pCont; pRspMsg->srcId = pSyncNode->myRaftId; pRspMsg->destId = pMsg->srcId; - pRspMsg->term = pSyncNode->pRaftStore->currentTerm; + pRspMsg->term = pSyncNode->raftStore.currentTerm; pRspMsg->lastIndex = pMsg->lastIndex; pRspMsg->lastTerm = pMsg->lastTerm; pRspMsg->startTime = pReceiver->startTime; pRspMsg->ack = pMsg->seq; // receiver maybe already closed - pRspMsg->code = 0; + pRspMsg->code = code; pRspMsg->snapBeginIndex = syncNodeGetSnapBeginIndex(pSyncNode); // send msg @@ -658,26 +607,36 @@ _SEND_REPLY: return -1; } - return 0; + return code; } static int32_t syncNodeOnSnapshotBegin(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // condition 1 SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; + int32_t code = TSDB_CODE_SYN_INTERNAL_ERROR; if (!snapshotReceiverIsStart(pReceiver)) { - sRError(pReceiver, "snapshot receiver not start"); - return -1; + sRError(pReceiver, "snapshot receiver begin failed since not start"); + goto _SEND_REPLY; } if (pReceiver->startTime != pMsg->startTime) { - sRError(pReceiver, "snapshot receiver startTime:%" PRId64 " not equal to msg startTime:%" PRId64, + sRError(pReceiver, "snapshot receiver begin failed since startTime:%" PRId64 " not equal to msg startTime:%" PRId64, pReceiver->startTime, pMsg->startTime); - return -1; + goto _SEND_REPLY; } // start writer - snapshotReceiverStartWriter(pReceiver, pMsg); + if (snapshotReceiverStartWriter(pReceiver, pMsg) != 0) { + sRError(pReceiver, "snapshot receiver begin failed since start writer failed"); + goto _SEND_REPLY; + } + + code = 0; +_SEND_REPLY: + if (code != 0 && terrno != 0) { + code = terrno; + } // build msg SRpcMsg rpcMsg = {0}; @@ -689,12 +648,12 @@ static int32_t syncNodeOnSnapshotBegin(SSyncNode *pSyncNode, SyncSnapshotSend *p SyncSnapshotRsp *pRspMsg = rpcMsg.pCont; pRspMsg->srcId = pSyncNode->myRaftId; pRspMsg->destId = pMsg->srcId; - pRspMsg->term = pSyncNode->pRaftStore->currentTerm; + pRspMsg->term = pSyncNode->raftStore.currentTerm; pRspMsg->lastIndex = pMsg->lastIndex; pRspMsg->lastTerm = pMsg->lastTerm; pRspMsg->startTime = pReceiver->startTime; pRspMsg->ack = pReceiver->ack; // receiver maybe already closed - pRspMsg->code = 0; + pRspMsg->code = code; pRspMsg->snapBeginIndex = pReceiver->snapshotParam.start; // send msg @@ -704,10 +663,10 @@ static int32_t syncNodeOnSnapshotBegin(SSyncNode *pSyncNode, SyncSnapshotSend *p return -1; } - return 0; + return code; } -static int32_t syncNodeOnSnapshotTransfering(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { +static int32_t syncNodeOnSnapshotReceive(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // condition 4 // transfering SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; @@ -721,8 +680,12 @@ static int32_t syncNodeOnSnapshotTransfering(SSyncNode *pSyncNode, SyncSnapshotS timeNow = taosGetTimestampMs(); } + int32_t code = 0; if (snapshotReceiverGotData(pReceiver, pMsg) != 0) { - return -1; + code = terrno; + if (code >= SYNC_SNAPSHOT_SEQ_INVALID) { + code = TSDB_CODE_SYN_INTERNAL_ERROR; + } } // build msg @@ -735,22 +698,22 @@ static int32_t syncNodeOnSnapshotTransfering(SSyncNode *pSyncNode, SyncSnapshotS SyncSnapshotRsp *pRspMsg = rpcMsg.pCont; pRspMsg->srcId = pSyncNode->myRaftId; pRspMsg->destId = pMsg->srcId; - pRspMsg->term = pSyncNode->pRaftStore->currentTerm; + pRspMsg->term = pSyncNode->raftStore.currentTerm; pRspMsg->lastIndex = pMsg->lastIndex; pRspMsg->lastTerm = pMsg->lastTerm; pRspMsg->startTime = pReceiver->startTime; pRspMsg->ack = pReceiver->ack; // receiver maybe already closed - pRspMsg->code = 0; + pRspMsg->code = code; pRspMsg->snapBeginIndex = pReceiver->snapshotParam.start; // send msg - syncLogSendSyncSnapshotRsp(pSyncNode, pRspMsg, "snapshot receiver receiving"); + syncLogSendSyncSnapshotRsp(pSyncNode, pRspMsg, "snapshot receiver received"); if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) { sRError(pReceiver, "snapshot receiver send resp failed since %s", terrstr()); return -1; } - return 0; + return code; } static int32_t syncNodeOnSnapshotEnd(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { @@ -782,12 +745,12 @@ static int32_t syncNodeOnSnapshotEnd(SSyncNode *pSyncNode, SyncSnapshotSend *pMs SyncSnapshotRsp *pRspMsg = rpcMsg.pCont; pRspMsg->srcId = pSyncNode->myRaftId; pRspMsg->destId = pMsg->srcId; - pRspMsg->term = pSyncNode->pRaftStore->currentTerm; + pRspMsg->term = pSyncNode->raftStore.currentTerm; pRspMsg->lastIndex = pMsg->lastIndex; pRspMsg->lastTerm = pMsg->lastTerm; pRspMsg->startTime = pReceiver->startTime; pRspMsg->ack = pReceiver->ack; // receiver maybe already closed - pRspMsg->code = 0; + pRspMsg->code = code; pRspMsg->snapBeginIndex = pReceiver->snapshotParam.start; // send msg @@ -797,7 +760,7 @@ static int32_t syncNodeOnSnapshotEnd(SSyncNode *pSyncNode, SyncSnapshotSend *pMs return -1; } - return 0; + return code; } // receiver on message @@ -827,70 +790,65 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { // if already drop replica, do not process if (!syncNodeInRaftGroup(pSyncNode, &pMsg->srcId)) { syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "not in my config"); - return 0; + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + return -1; } - if (pMsg->term < pSyncNode->pRaftStore->currentTerm) { + if (pMsg->term < pSyncNode->raftStore.currentTerm) { syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "reject since small term"); - return 0; + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + return -1; } - if (pMsg->term > pSyncNode->pRaftStore->currentTerm) { + if (pMsg->term > pSyncNode->raftStore.currentTerm) { syncNodeStepDown(pSyncNode, pMsg->term); } syncNodeResetElectTimer(pSyncNode); // state, term, seq/ack + int32_t code = 0; if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { - if (pMsg->term == pSyncNode->pRaftStore->currentTerm) { + if (pMsg->term == pSyncNode->raftStore.currentTerm) { if (pMsg->seq == SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT) { syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq pre-snapshot"); - syncNodeOnSnapshotPre(pSyncNode, pMsg); + code = syncNodeOnSnapshotPre(pSyncNode, pMsg); } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) { syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq begin"); - syncNodeOnSnapshotBegin(pSyncNode, pMsg); + code = syncNodeOnSnapshotBegin(pSyncNode, pMsg); } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq end"); - syncNodeOnSnapshotEnd(pSyncNode, pMsg); + code = syncNodeOnSnapshotEnd(pSyncNode, pMsg); if (syncLogBufferReInit(pSyncNode->pLogBuf, pSyncNode) != 0) { sRError(pReceiver, "failed to reinit log buffer since %s", terrstr()); - return -1; + code = -1; } } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { // force close, no response syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process force stop"); - snapshotReceiverForceStop(pReceiver); + snapshotReceiverStop(pReceiver); } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { - syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq"); - syncNodeOnSnapshotTransfering(pSyncNode, pMsg); + syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq data"); + code = syncNodeOnSnapshotReceive(pSyncNode, pMsg); } else { // error log sRError(pReceiver, "snapshot receiver recv error seq:%d, my ack:%d", pMsg->seq, pReceiver->ack); - return -1; + code = -1; } } else { // error log sRError(pReceiver, "snapshot receiver term not equal"); - return -1; + code = -1; } } else { // error log sRError(pReceiver, "snapshot receiver not follower"); - return -1; + code = -1; } - return 0; + return code; } -int32_t syncNodeOnSnapshotReplyPre(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { - // get sender - SSyncSnapshotSender *pSender = syncNodeGetSnapshotSender(pSyncNode, &(pMsg->srcId)); - if (pSender == NULL) { - sNError(pSyncNode, "prepare snapshot error since sender is null"); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - return -1; - } - +static int32_t syncNodeOnSnapshotPreRsp(SSyncNode *pSyncNode, SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) { SSnapshot snapshot = {0}; pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); @@ -912,7 +870,7 @@ int32_t syncNodeOnSnapshotReplyPre(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) pSender->snapshot = snapshot; // start reader - int32_t code = pSyncNode->pFsm->FpSnapshotStartRead(pSyncNode->pFsm, &(pSender->snapshotParam), &(pSender->pReader)); + int32_t code = pSyncNode->pFsm->FpSnapshotStartRead(pSyncNode->pFsm, &pSender->snapshotParam, &pSender->pReader); if (code != 0) { sSError(pSender, "prepare snapshot failed since %s", terrstr()); return -1; @@ -933,8 +891,8 @@ int32_t syncNodeOnSnapshotReplyPre(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) SyncSnapshotSend *pSendMsg = rpcMsg.pCont; pSendMsg->srcId = pSender->pSyncNode->myRaftId; - pSendMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; - pSendMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; + pSendMsg->destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; + pSendMsg->term = pSender->pSyncNode->raftStore.currentTerm; pSendMsg->beginIndex = pSender->snapshotParam.start; pSendMsg->lastIndex = pSender->snapshot.lastApplyIndex; pSendMsg->lastTerm = pSender->snapshot.lastApplyTerm; @@ -963,8 +921,9 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { SyncSnapshotRsp *pMsg = pRpcMsg->pCont; // if already drop replica, do not process - if (!syncNodeInRaftGroup(pSyncNode, &(pMsg->srcId))) { + if (!syncNodeInRaftGroup(pSyncNode, &pMsg->srcId)) { syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "maybe replica already dropped"); + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; return -1; } @@ -976,76 +935,103 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { return -1; } + // state, term, seq/ack + if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "snapshot sender not leader"); + sSError(pSender, "snapshot sender not leader"); + terrno = TSDB_CODE_SYN_NOT_LEADER; + goto _ERROR; + } + if (pMsg->startTime != pSender->startTime) { - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "sender:% " PRId64 " receiver:%" PRId64 " time not match"); - return -1; + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "snapshot sender and receiver time not match"); + sSError(pSender, "sender:%" PRId64 " receiver:%" PRId64 " time not match, code:0x%x", pMsg->startTime, + pSender->startTime, pMsg->code); + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + goto _ERROR; } - // state, term, seq/ack - if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { - if (pMsg->term == pSyncNode->pRaftStore->currentTerm) { - // prepare , send begin msg - if (pMsg->ack == SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT) { - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq pre-snapshot"); - syncNodeOnSnapshotReplyPre(pSyncNode, pMsg); - return 0; - } + if (pMsg->term != pSyncNode->raftStore.currentTerm) { + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "snapshot sender and receiver term not match"); + sSError(pSender, "snapshot sender term not equal, msg term:%" PRId64 " currentTerm:%" PRId64, pMsg->term, + pSyncNode->raftStore.currentTerm); + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + goto _ERROR; + } - if (pMsg->ack == SYNC_SNAPSHOT_SEQ_BEGIN) { - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq begin"); - if (snapshotSenderUpdateProgress(pSender, pMsg) != 0) { - return -1; - } + if (pMsg->code != 0) { + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "receive error code"); + sSError(pSender, "snapshot sender receive error code:0x%x and stop sender", pMsg->code); + terrno = pMsg->code; + goto _ERROR; + } - if (snapshotSend(pSender) != 0) { - return -1; - } - return 0; - } + // prepare , send begin msg + if (pMsg->ack == SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT) { + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq pre-snapshot"); + return syncNodeOnSnapshotPreRsp(pSyncNode, pSender, pMsg); + } - // receive ack is finish, close sender - if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) { - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq end"); - snapshotSenderStop(pSender, true); - SSyncLogReplMgr *pMgr = syncNodeGetLogReplMgr(pSyncNode, &pMsg->srcId); - if (pMgr) { - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "reset repl mgr"); - syncLogReplMgrReset(pMgr); - } - return 0; - } + if (pSender->pReader == NULL || pSender->finish) { + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "snapshot sender invalid"); + sSError(pSender, "snapshot sender invalid, pReader:%p finish:%d", pMsg->code, pSender->pReader, pSender->finish); + terrno = pMsg->code; + goto _ERROR; + } - // send next msg - if (pMsg->ack == pSender->seq) { - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq"); - // update sender ack - if (snapshotSenderUpdateProgress(pSender, pMsg) != 0) { - return -1; - } - if (snapshotSend(pSender) != 0) { - return -1; - } + if (pMsg->ack == SYNC_SNAPSHOT_SEQ_BEGIN) { + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq begin"); + if (snapshotSenderUpdateProgress(pSender, pMsg) != 0) { + return -1; + } - } else if (pMsg->ack == pSender->seq - 1) { - // maybe resend - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq and resend"); - snapshotReSend(pSender); + if (snapshotSend(pSender) != 0) { + return -1; + } + return 0; + } - } else { - // error log - sSError(pSender, "snapshot sender recv error ack:%d, my seq:%d", pMsg->ack, pSender->seq); - return -1; - } - } else { - // error log - sSError(pSender, "snapshot sender term not equal"); + // receive ack is finish, close sender + if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) { + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq end"); + snapshotSenderStop(pSender, true); + SSyncLogReplMgr *pMgr = syncNodeGetLogReplMgr(pSyncNode, &pMsg->srcId); + syncLogReplMgrReset(pMgr); + return 0; + } + + // send next msg + if (pMsg->ack == pSender->seq) { + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq data"); + // update sender ack + if (snapshotSenderUpdateProgress(pSender, pMsg) != 0) { + return -1; + } + if (snapshotSend(pSender) != 0) { + return -1; + } + } else if (pMsg->ack == pSender->seq - 1) { + // maybe resend + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq and resend"); + if (snapshotReSend(pSender) != 0) { return -1; } } else { // error log - sSError(pSender, "snapshot sender not leader"); + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "receive error ack"); + sSError(pSender, "snapshot sender receive error ack:%d, my seq:%d", pMsg->ack, pSender->seq); + snapshotSenderStop(pSender, true); + SSyncLogReplMgr *pMgr = syncNodeGetLogReplMgr(pSyncNode, &pMsg->srcId); + syncLogReplMgrReset(pMgr); return -1; } return 0; + +_ERROR: + snapshotSenderStop(pSender, true); + SSyncLogReplMgr *pMgr = syncNodeGetLogReplMgr(pSyncNode, &pMsg->srcId); + syncLogReplMgrReset(pMgr); + + return -1; } diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 16e593d0e49695f3ec7ac02d47388106bcb9d281..859183db9529ace6bf914fafbc381ca388880dea 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -24,33 +24,35 @@ #include "syncUtil.h" static void syncNodeCleanConfigIndex(SSyncNode* ths) { +#if 0 int32_t newArrIndex = 0; SyncIndex newConfigIndexArr[MAX_CONFIG_INDEX_COUNT] = {0}; SSnapshot snapshot = {0}; ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot); if (snapshot.lastApplyIndex != SYNC_INDEX_INVALID) { - for (int32_t i = 0; i < ths->pRaftCfg->configIndexCount; ++i) { - if (ths->pRaftCfg->configIndexArr[i] < snapshot.lastConfigIndex) { + for (int32_t i = 0; i < ths->raftCfg.configIndexCount; ++i) { + if (ths->raftCfg.configIndexArr[i] < snapshot.lastConfigIndex) { // pass } else { // save - newConfigIndexArr[newArrIndex] = ths->pRaftCfg->configIndexArr[i]; + newConfigIndexArr[newArrIndex] = ths->raftCfg.configIndexArr[i]; ++newArrIndex; } } - int32_t oldCnt = ths->pRaftCfg->configIndexCount; - ths->pRaftCfg->configIndexCount = newArrIndex; - memcpy(ths->pRaftCfg->configIndexArr, newConfigIndexArr, sizeof(newConfigIndexArr)); + int32_t oldCnt = ths->raftCfg.configIndexCount; + ths->raftCfg.configIndexCount = newArrIndex; + memcpy(ths->raftCfg.configIndexArr, newConfigIndexArr, sizeof(newConfigIndexArr)); - int32_t code = raftCfgPersist(ths->pRaftCfg); + int32_t code = syncWriteCfgFile(ths); if (code != 0) { sNFatal(ths, "failed to persist cfg"); } else { - sNTrace(ths, "clean config index arr, old-cnt:%d, new-cnt:%d", oldCnt, ths->pRaftCfg->configIndexCount); + sNTrace(ths, "clean config index arr, old-cnt:%d, new-cnt:%d", oldCnt, ths->raftCfg.configIndexCount); } } +#endif } static int32_t syncNodeTimerRoutine(SSyncNode* ths) { diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 49a24bebde3774640c2636e7964a2c18e3a1266f..b246d9a79d58c153d9e5b12775965929f9ec7adf 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -17,46 +17,27 @@ #include "syncUtil.h" #include "syncIndexMgr.h" #include "syncMessage.h" +#include "syncPipeline.h" #include "syncRaftCfg.h" #include "syncRaftStore.h" #include "syncSnapshot.h" -extern void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port); - -uint64_t syncUtilAddr2U64(const char* host, uint16_t port) { - uint32_t hostU32 = taosGetIpv4FromFqdn(host); - if (hostU32 == (uint32_t)-1) { - sError("failed to resolve ipv4 addr, host:%s", host); - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; +void syncCfg2SimpleStr(const SSyncCfg* pCfg, char* buf, int32_t bufLen) { + int32_t len = snprintf(buf, bufLen, "{num:%d, as:%d, [", pCfg->replicaNum, pCfg->myIndex); + for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + len += snprintf(buf + len, bufLen - len, "%s:%d", pCfg->nodeInfo[i].nodeFqdn, pCfg->nodeInfo[i].nodePort); + if (i < pCfg->replicaNum - 1) { + len += snprintf(buf + len, bufLen - len, "%s", ", "); + } } - - uint64_t u64 = (((uint64_t)hostU32) << 32) | (((uint32_t)port) << 16); - return u64; -} - -void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port) { - uint32_t hostU32 = (uint32_t)((u64 >> 32) & 0x00000000FFFFFFFF); - - struct in_addr addr = {.s_addr = hostU32}; - taosInetNtoa(addr, host, len); - *port = (uint16_t)((u64 & 0x00000000FFFF0000) >> 16); + len += snprintf(buf + len, bufLen - len, "%s", "]}"); } void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet) { pEpSet->inUse = 0; - pEpSet->numOfEps = 0; - addEpIntoEpSet(pEpSet, pInfo->nodeFqdn, pInfo->nodePort); -} - -void syncUtilRaftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) { - char host[TSDB_FQDN_LEN] = {0}; - uint16_t port = 0; - - syncUtilU642Addr(raftId->addr, host, sizeof(host), &port); - pEpSet->inUse = 0; - pEpSet->numOfEps = 0; - addEpIntoEpSet(pEpSet, host, port); + pEpSet->numOfEps = 1; + pEpSet->eps[0].port = pInfo->nodePort; + tstrncpy(pEpSet->eps[0].fqdn, pInfo->nodeFqdn, TSDB_FQDN_LEN); } bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId) { @@ -69,13 +50,18 @@ bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* char ipbuf[128] = {0}; tinet_ntoa(ipbuf, ipv4); - raftId->addr = syncUtilAddr2U64(ipbuf, pInfo->nodePort); + raftId->addr = SYNC_ADDR(pInfo); raftId->vgId = vgId; + + sInfo("vgId:%d, sync addr:%" PRIu64 ", dnode:%d cluster:%" PRId64 " fqdn:%s ip:%s port:%u ipv4:%u", vgId, + raftId->addr, pInfo->nodeId, pInfo->clusterId, pInfo->nodeFqdn, ipbuf, pInfo->nodePort, ipv4); return true; } bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2) { - return pId1->addr == pId2->addr && pId1->vgId == pId2->vgId; + if (pId1->addr == pId2->addr && pId1->vgId == pId2->vgId) return true; + if ((CID(pId1) == 0 || CID(pId2) == 0) && (DID(pId1) == DID(pId2)) && pId1->vgId == pId2->vgId) return true; + return false; } bool syncUtilEmptyId(const SRaftId* pId) { return (pId->addr == 0 && pId->vgId == 0); } @@ -91,117 +77,67 @@ int32_t syncUtilElectRandomMS(int32_t min, int32_t max) { int32_t syncUtilQuorum(int32_t replicaNum) { return replicaNum / 2 + 1; } -cJSON* syncUtilRaftId2Json(const SRaftId* p) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", p->addr); - cJSON_AddStringToObject(pRoot, "addr", u64buf); - char host[128] = {0}; - uint16_t port; - syncUtilU642Addr(p->addr, host, sizeof(host), &port); - cJSON_AddStringToObject(pRoot, "host", host); - cJSON_AddNumberToObject(pRoot, "port", port); - cJSON_AddNumberToObject(pRoot, "vgId", p->vgId); - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SRaftId", pRoot); - return pJson; -} - -static inline bool syncUtilCanPrint(char c) { - if (c >= 32 && c <= 126) { - return true; - } else { - return false; - } -} - -char* syncUtilPrintBin(char* ptr, uint32_t len) { - int64_t memLen = (int64_t)(len + 1); - char* s = taosMemoryMalloc(memLen); - ASSERT(s != NULL); - memset(s, 0, len + 1); - memcpy(s, ptr, len); - - for (int32_t i = 0; i < len; ++i) { - if (!syncUtilCanPrint(s[i])) { - s[i] = '.'; - } - } - return s; -} - -char* syncUtilPrintBin2(char* ptr, uint32_t len) { - uint32_t len2 = len * 4 + 1; - char* s = taosMemoryMalloc(len2); - ASSERT(s != NULL); - memset(s, 0, len2); - - char* p = s; - for (int32_t i = 0; i < len; ++i) { - int32_t n = sprintf(p, "%d,", ptr[i]); - p += n; - } - return s; -} - void syncUtilMsgHtoN(void* msg) { SMsgHead* pHead = msg; pHead->contLen = htonl(pHead->contLen); pHead->vgId = htonl(pHead->vgId); } -void syncUtilMsgNtoH(void* msg) { - SMsgHead* pHead = msg; - pHead->contLen = ntohl(pHead->contLen); - pHead->vgId = ntohl(pHead->vgId); -} - bool syncUtilUserPreCommit(tmsg_t msgType) { return msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_LEADER_TRANSFER; } bool syncUtilUserRollback(tmsg_t msgType) { return msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_LEADER_TRANSFER; } -void syncCfg2SimpleStr(const SSyncCfg* pCfg, char* buf, int32_t bufLen) { - int32_t len = snprintf(buf, bufLen, "{r-num:%d, my:%d, ", pCfg->replicaNum, pCfg->myIndex); - - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { - if (i < pCfg->replicaNum - 1) { - len += snprintf(buf + len, bufLen - len, "%s:%d, ", pCfg->nodeInfo[i].nodeFqdn, pCfg->nodeInfo[i].nodePort); - } else { - len += snprintf(buf + len, bufLen - len, "%s:%d}", pCfg->nodeInfo[i].nodeFqdn, pCfg->nodeInfo[i].nodePort); - } - } -} - // for leader static void syncHearbeatReplyTime2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { - int32_t len = 5; - + int32_t len = 0; + len += snprintf(buf + len, bufLen - len, "%s", "{"); for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { int64_t tsMs = syncIndexMgrGetRecvTime(pSyncNode->pMatchIndex, &(pSyncNode->replicasId[i])); - + len += snprintf(buf + len, bufLen - len, "%d:%" PRId64, i, tsMs); if (i < pSyncNode->replicaNum - 1) { - len += snprintf(buf + len, bufLen - len, "%d:%" PRId64 ",", i, tsMs); - } else { - len += snprintf(buf + len, bufLen - len, "%d:%" PRId64 "}", i, tsMs); + len += snprintf(buf + len, bufLen - len, "%s", ","); } } + len += snprintf(buf + len, bufLen - len, "%s", "}"); } // for follower static void syncHearbeatTime2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { - int32_t len = 4; - + int32_t len = 0; + len += snprintf(buf + len, bufLen - len, "%s", "{"); for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { int64_t tsMs = syncIndexMgrGetRecvTime(pSyncNode->pNextIndex, &(pSyncNode->replicasId[i])); - + len += snprintf(buf + len, bufLen - len, "%d:%" PRId64, i, tsMs); if (i < pSyncNode->replicaNum - 1) { - len += snprintf(buf + len, bufLen - len, "%d:%" PRId64 ",", i, tsMs); - } else { - len += snprintf(buf + len, bufLen - len, "%d:%" PRId64 "}", i, tsMs); + len += snprintf(buf + len, bufLen - len, "%s", ","); } } + len += snprintf(buf + len, bufLen - len, "%s", "}"); +} + +static void syncLogBufferStates2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { + SSyncLogBuffer* pBuf = pSyncNode->pLogBuf; + if (pBuf == NULL) { + return; + } + int len = 0; + len += snprintf(buf + len, bufLen - len, "[%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pBuf->startIndex, + pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); +} + +static void syncLogReplMgrStates2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { + int len = 0; + len += snprintf(buf + len, bufLen - len, "%s", "{"); + for (int32_t i = 0; i < pSyncNode->replicaNum; i++) { + SSyncLogReplMgr* pMgr = pSyncNode->logReplMgrs[i]; + if (pMgr == NULL) break; + len += snprintf(buf + len, bufLen - len, "%d:%d [%" PRId64 " %" PRId64 ", %" PRId64 ")", i, pMgr->restored, + pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex); + if (i + 1 < pSyncNode->replicaNum) { + len += snprintf(buf + len, bufLen - len, "%s", ", "); + } + } + len += snprintf(buf + len, bufLen - len, "%s", "}"); } static void syncPeerState2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { @@ -222,8 +158,8 @@ static void syncPeerState2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { } void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNode* pNode, const char* format, ...) { - if (pNode == NULL || pNode->pRaftCfg == NULL || pNode->pRaftStore == NULL || pNode->pLogStore == NULL) return; - int64_t currentTerm = pNode->pRaftStore->currentTerm; + if (pNode == NULL || pNode->pLogStore == NULL) return; + int64_t currentTerm = pNode->raftStore.currentTerm; // save error code, otherwise it will be overwritten int32_t errCode = terrno; @@ -243,24 +179,21 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo int32_t cacheHit = pNode->pLogStore->cacheHit; int32_t cacheMiss = pNode->pLogStore->cacheMiss; - char cfgStr[1024]; - if (pNode->pRaftCfg != NULL) { - syncCfg2SimpleStr(&(pNode->pRaftCfg->cfg), cfgStr, sizeof(cfgStr)); - } else { - return; - } + char cfgStr[1024] = ""; + syncCfg2SimpleStr(&pNode->raftCfg.cfg, cfgStr, sizeof(cfgStr)); - char peerStr[1024] = "{"; - syncPeerState2Str(pNode, peerStr, sizeof(peerStr)); + char replMgrStatesStr[1024] = ""; + syncLogReplMgrStates2Str(pNode, replMgrStatesStr, sizeof(replMgrStatesStr)); - char hbrTimeStr[256] = "hbr:{"; + char bufferStatesStr[256] = ""; + syncLogBufferStates2Str(pNode, bufferStatesStr, sizeof(bufferStatesStr)); + + char hbrTimeStr[256] = ""; syncHearbeatReplyTime2Str(pNode, hbrTimeStr, sizeof(hbrTimeStr)); - char hbTimeStr[256] = "hb:{"; + char hbTimeStr[256] = ""; syncHearbeatTime2Str(pNode, hbTimeStr, sizeof(hbTimeStr)); - int32_t quorum = syncNodeDynamicQuorum(pNode); - char eventLog[512]; // {0}; va_list argpointer; va_start(argpointer, format); @@ -274,28 +207,28 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo // restore error code terrno = errCode; - - if (pNode != NULL && pNode->pRaftCfg != NULL) { + + if (pNode != NULL) { taosPrintLog(flags, level, dflag, "vgId:%d, %s, sync:%s, term:%" PRIu64 ", commit-index:%" PRId64 ", first-ver:%" PRId64 ", last-ver:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64 ", snap-term:%" PRIu64 - ", elect-times:%d, as-leader-times:%d, cfg-ch-times:%d, hit:%d, mis:%d, hb-slow:%d, hbr-slow:%d, " + ", elect-times:%d, as-leader-times:%d, cfg-ch-times:%d, hb-slow:%d, hbr-slow:%d, " "aq-items:%d, snaping:%" PRId64 ", replicas:%d, last-cfg:%" PRId64 - ", chging:%d, restore:%d, quorum:%d, elect-lc-timer:%" PRId64 ", hb:%" PRId64 ", %s, %s, %s, %s", + ", chging:%d, restore:%d, quorum:%d, elect-lc-timer:%" PRId64 ", hb:%" PRId64 + ", buffer:%s, repl-mgrs:%s, members:%s, hb:%s, hb-reply:%s", pNode->vgId, eventLog, syncStr(pNode->state), currentTerm, pNode->commitIndex, logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pNode->electNum, - pNode->becomeLeaderNum, pNode->configChangeNum, cacheHit, cacheMiss, pNode->hbSlowNum, - pNode->hbrSlowNum, aqItems, pNode->snapshottingIndex, pNode->replicaNum, - pNode->pRaftCfg->lastConfigIndex, pNode->changing, pNode->restoreFinish, quorum, - pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr, hbTimeStr, - hbrTimeStr); + pNode->becomeLeaderNum, pNode->configChangeNum, pNode->hbSlowNum, pNode->hbrSlowNum, aqItems, + pNode->snapshottingIndex, pNode->replicaNum, pNode->raftCfg.lastConfigIndex, pNode->changing, + pNode->restoreFinish, syncNodeDynamicQuorum(pNode), pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, + bufferStatesStr, replMgrStatesStr, cfgStr, hbTimeStr, hbrTimeStr); } } void syncPrintSnapshotSenderLog(const char* flags, ELogLevel level, int32_t dflag, SSyncSnapshotSender* pSender, const char* format, ...) { SSyncNode* pNode = pSender->pSyncNode; - if (pNode == NULL || pNode->pRaftCfg == NULL || pNode->pRaftStore == NULL || pNode->pLogStore == NULL) return; + if (pNode == NULL || pNode->pLogStore == NULL) return; SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; if (pNode->pFsm != NULL && pNode->pFsm->FpGetSnapshotInfo != NULL) { @@ -309,18 +242,12 @@ void syncPrintSnapshotSenderLog(const char* flags, ELogLevel level, int32_t dfla logBeginIndex = pNode->pLogStore->syncLogBeginIndex(pNode->pLogStore); } - char cfgStr[1024]; - syncCfg2SimpleStr(&(pNode->pRaftCfg->cfg), cfgStr, sizeof(cfgStr)); + char cfgStr[1024] = ""; + syncCfg2SimpleStr(&pNode->raftCfg.cfg, cfgStr, sizeof(cfgStr)); char peerStr[1024] = "{"; syncPeerState2Str(pNode, peerStr, sizeof(peerStr)); - int32_t quorum = syncNodeDynamicQuorum(pNode); - SRaftId destId = pNode->replicasId[pSender->replicaIndex]; - char host[64]; - uint16_t port; - syncUtilU642Addr(destId.addr, host, sizeof(host), &port); - char eventLog[512]; // {0}; va_list argpointer; va_start(argpointer, format); @@ -330,24 +257,24 @@ void syncPrintSnapshotSenderLog(const char* flags, ELogLevel level, int32_t dfla taosPrintLog(flags, level, dflag, "vgId:%d, %s, sync:%s, {%p s-param:%" PRId64 " e-param:%" PRId64 " laindex:%" PRId64 " laterm:%" PRIu64 " lcindex:%" PRId64 - " seq:%d ack:%d finish:%d replica-index:%d %s:%d}" + " seq:%d ack:%d finish:%d replica-index:%d dnode:%d}" ", tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64 ", snap-tm:%" PRIu64 ", sby:%d, stgy:%d, bch:%d, r-num:%d, lcfg:%" PRId64 ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s, %s", pNode->vgId, eventLog, syncStr(pNode->state), pSender, pSender->snapshotParam.start, pSender->snapshotParam.end, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->seq, pSender->ack, pSender->finish, pSender->replicaIndex, - host, port, pNode->pRaftStore->currentTerm, pNode->commitIndex, logBeginIndex, logLastIndex, - pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pNode->pRaftCfg->isStandBy, - pNode->pRaftCfg->snapshotStrategy, pNode->pRaftCfg->batchSize, pNode->replicaNum, - pNode->pRaftCfg->lastConfigIndex, pNode->changing, pNode->restoreFinish, quorum, + DID(&pNode->replicasId[pSender->replicaIndex]), pNode->raftStore.currentTerm, pNode->commitIndex, + logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, + pNode->raftCfg.isStandBy, pNode->raftCfg.snapshotStrategy, pNode->raftCfg.batchSize, pNode->replicaNum, + pNode->raftCfg.lastConfigIndex, pNode->changing, pNode->restoreFinish, syncNodeDynamicQuorum(pNode), pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr); } void syncPrintSnapshotReceiverLog(const char* flags, ELogLevel level, int32_t dflag, SSyncSnapshotReceiver* pReceiver, const char* format, ...) { SSyncNode* pNode = pReceiver->pSyncNode; - if (pNode == NULL || pNode->pRaftCfg == NULL || pNode->pRaftStore == NULL || pNode->pLogStore == NULL) return; + if (pNode == NULL || pNode->pLogStore == NULL) return; SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; if (pNode->pFsm != NULL && pNode->pFsm->FpGetSnapshotInfo != NULL) { @@ -361,18 +288,12 @@ void syncPrintSnapshotReceiverLog(const char* flags, ELogLevel level, int32_t df logBeginIndex = pNode->pLogStore->syncLogBeginIndex(pNode->pLogStore); } - char cfgStr[1024]; - syncCfg2SimpleStr(&(pNode->pRaftCfg->cfg), cfgStr, sizeof(cfgStr)); + char cfgStr[1024] = ""; + syncCfg2SimpleStr(&pNode->raftCfg.cfg, cfgStr, sizeof(cfgStr)); char peerStr[1024] = "{"; syncPeerState2Str(pNode, peerStr, sizeof(peerStr)); - int32_t quorum = syncNodeDynamicQuorum(pNode); - SRaftId fromId = pReceiver->fromId; - char host[128]; - uint16_t port; - syncUtilU642Addr(fromId.addr, host, sizeof(host), &port); - char eventLog[512]; // {0}; va_list argpointer; va_start(argpointer, format); @@ -381,19 +302,19 @@ void syncPrintSnapshotReceiverLog(const char* flags, ELogLevel level, int32_t df taosPrintLog(flags, level, dflag, "vgId:%d, %s, sync:%s," - " {%p start:%d ack:%d term:%" PRIu64 " start-time:%" PRId64 " from:%s:%d s-param:%" PRId64 + " {%p start:%d ack:%d term:%" PRIu64 " start-time:%" PRId64 " from dnode:%d s-param:%" PRId64 " e-param:%" PRId64 " laindex:%" PRId64 " laterm:%" PRIu64 " lcindex:%" PRId64 "}" ", tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64 ", snap-tm:%" PRIu64 ", sby:%d, stgy:%d, bch:%d, r-num:%d, lcfg:%" PRId64 ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s, %s", pNode->vgId, eventLog, syncStr(pNode->state), pReceiver, pReceiver->start, pReceiver->ack, - pReceiver->term, pReceiver->startTime, host, port, pReceiver->snapshotParam.start, + pReceiver->term, pReceiver->startTime, DID(&pReceiver->fromId), pReceiver->snapshotParam.start, pReceiver->snapshotParam.end, pReceiver->snapshot.lastApplyIndex, pReceiver->snapshot.lastApplyTerm, - pReceiver->snapshot.lastConfigIndex, pNode->pRaftStore->currentTerm, pNode->commitIndex, logBeginIndex, + pReceiver->snapshot.lastConfigIndex, pNode->raftStore.currentTerm, pNode->commitIndex, logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, - pNode->pRaftCfg->isStandBy, pNode->pRaftCfg->snapshotStrategy, pNode->pRaftCfg->batchSize, - pNode->replicaNum, pNode->pRaftCfg->lastConfigIndex, pNode->changing, pNode->restoreFinish, quorum, + pNode->raftCfg.isStandBy, pNode->raftCfg.snapshotStrategy, pNode->raftCfg.batchSize, pNode->replicaNum, + pNode->raftCfg.lastConfigIndex, pNode->changing, pNode->restoreFinish, syncNodeDynamicQuorum(pNode), pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr); } @@ -408,56 +329,37 @@ void syncLogRecvTimer(SSyncNode* pSyncNode, const SyncTimeout* pMsg, const char* } void syncLogRecvLocalCmd(SSyncNode* pSyncNode, const SyncLocalCmd* pMsg, const char* s) { - if (!(sDebugFlag & DEBUG_TRACE)) return; - sNTrace(pSyncNode, "recv sync-local-cmd {cmd:%d-%s, sd-new-term:%" PRId64 ", fc-index:%" PRId64 "}, %s", pMsg->cmd, - syncLocalCmdGetStr(pMsg->cmd), pMsg->sdNewTerm, pMsg->fcIndex, s); + syncLocalCmdGetStr(pMsg->cmd), pMsg->currentTerm, pMsg->commitIndex, s); } void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { - if (!(sDebugFlag & DEBUG_TRACE)) return; - - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, - "send sync-append-entries-reply to %s:%d, {term:%" PRId64 ", pterm:%" PRId64 + "send sync-append-entries-reply to dnode:%d, {term:%" PRId64 ", pterm:%" PRId64 ", success:%d, lsend-index:%" PRId64 ", match:%" PRId64 "}, %s", - host, port, pMsg->term, pMsg->lastMatchTerm, pMsg->success, pMsg->lastSendIndex, pMsg->matchIndex, s); + DID(&pMsg->destId), pMsg->term, pMsg->lastMatchTerm, pMsg->success, pMsg->lastSendIndex, pMsg->matchIndex, s); } void syncLogRecvAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { - if (!(sDebugFlag & DEBUG_TRACE)) return; - - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, - "recv sync-append-entries-reply from %s:%d {term:%" PRId64 ", pterm:%" PRId64 + "recv sync-append-entries-reply from dnode:%d {term:%" PRId64 ", pterm:%" PRId64 ", success:%d, lsend-index:%" PRId64 ", match:%" PRId64 "}, %s", - host, port, pMsg->term, pMsg->lastMatchTerm, pMsg->success, pMsg->lastSendIndex, pMsg->matchIndex, s); + DID(&pMsg->srcId), pMsg->term, pMsg->lastMatchTerm, pMsg->success, pMsg->lastSendIndex, pMsg->matchIndex, s); } void syncLogSendHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, bool printX, int64_t timerElapsed, int64_t execTime) { - if (!(sDebugFlag & DEBUG_TRACE)) return; - - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - if (printX) { sNTrace(pSyncNode, - "send sync-heartbeat to %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64 + "send sync-heartbeat to dnode:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64 "}, x", - host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp); + DID(&pMsg->destId), pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp); } else { sNTrace(pSyncNode, - "send sync-heartbeat to %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64 + "send sync-heartbeat to dnode:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64 "}, timer-elapsed:%" PRId64 ", next-exec:%" PRId64, - host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, timerElapsed, execTime); + DID(&pMsg->destId), pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, timerElapsed, + execTime); } } @@ -465,183 +367,109 @@ void syncLogRecvHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, int64 if (timeDiff > SYNC_HEARTBEAT_SLOW_MS) { pSyncNode->hbSlowNum++; - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); sNInfo(pSyncNode, - "recv sync-heartbeat from %s:%d slow {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64 - "}, %s, net elapsed:%" PRId64, - host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, s, timeDiff); + "recv sync-heartbeat from dnode:%d slow {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 + ", ts:%" PRId64 "}, %s, net elapsed:%" PRId64, + DID(&pMsg->srcId), pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, s, timeDiff); } - if (!(sDebugFlag & DEBUG_TRACE)) return; - - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "recv sync-heartbeat from %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64 + "recv sync-heartbeat from dnode:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64 "}, %s, net elapsed:%" PRId64, - host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, s, timeDiff); + DID(&pMsg->srcId), pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, s, timeDiff); } void syncLogSendHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, const char* s) { - if (!(sDebugFlag & DEBUG_TRACE)) return; - - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - - sNTrace(pSyncNode, "send sync-heartbeat-reply from %s:%d {term:%" PRId64 ", ts:%" PRId64 "}, %s", host, port, - pMsg->term, pMsg->timeStamp, s); + sNTrace(pSyncNode, "send sync-heartbeat-reply from dnode:%d {term:%" PRId64 ", ts:%" PRId64 "}, %s", + DID(&pMsg->destId), pMsg->term, pMsg->timeStamp, s); } void syncLogRecvHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, int64_t timeDiff, const char* s) { if (timeDiff > SYNC_HEARTBEAT_REPLY_SLOW_MS) { pSyncNode->hbrSlowNum++; - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "recv sync-heartbeat-reply from %s:%d slow {term:%" PRId64 ", ts:%" PRId64 "}, %s, net elapsed:%" PRId64, - host, port, pMsg->term, pMsg->timeStamp, s, timeDiff); + "recv sync-heartbeat-reply from dnode:%d slow {term:%" PRId64 ", ts:%" PRId64 "}, %s, net elapsed:%" PRId64, + DID(&pMsg->srcId), pMsg->term, pMsg->timeStamp, s, timeDiff); } - if (!(sDebugFlag & DEBUG_TRACE)) return; - - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "recv sync-heartbeat-reply from %s:%d {term:%" PRId64 ", ts:%" PRId64 "}, %s, net elapsed:%" PRId64, host, - port, pMsg->term, pMsg->timeStamp, s, timeDiff); + "recv sync-heartbeat-reply from dnode:%d {term:%" PRId64 ", ts:%" PRId64 "}, %s, net elapsed:%" PRId64, + DID(&pMsg->srcId), pMsg->term, pMsg->timeStamp, s, timeDiff); } void syncLogSendSyncSnapshotSend(SSyncNode* pSyncNode, const SyncSnapshotSend* pMsg, const char* s) { - if (!(sDebugFlag & DEBUG_DEBUG)) return; - - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - sNDebug(pSyncNode, - "send sync-snapshot-send to %s:%u, %s, seq:%d, term:%" PRId64 ", begin:%" PRId64 ", end:%" PRId64 + "send sync-snapshot-send to dnode:%d, %s, seq:%d, term:%" PRId64 ", begin:%" PRId64 ", end:%" PRId64 ", lterm:%" PRId64 ", stime:%" PRId64, - host, port, s, pMsg->seq, pMsg->term, pMsg->beginIndex, pMsg->lastIndex, pMsg->lastTerm, pMsg->startTime); + DID(&pMsg->destId), s, pMsg->seq, pMsg->term, pMsg->beginIndex, pMsg->lastIndex, pMsg->lastTerm, + pMsg->startTime); } void syncLogRecvSyncSnapshotSend(SSyncNode* pSyncNode, const SyncSnapshotSend* pMsg, const char* s) { - if (!(sDebugFlag & DEBUG_DEBUG)) return; - - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sNDebug(pSyncNode, - "recv sync-snapshot-send from %s:%u, %s, seq:%d, term:%" PRId64 ", begin:%" PRId64 ", lst:%" PRId64 + "recv sync-snapshot-send from dnode:%d, %s, seq:%d, term:%" PRId64 ", begin:%" PRId64 ", lst:%" PRId64 ", lterm:%" PRId64 ", stime:%" PRId64 ", len:%u", - host, port, s, pMsg->seq, pMsg->term, pMsg->beginIndex, pMsg->lastIndex, pMsg->lastTerm, pMsg->startTime, - pMsg->dataLen); + DID(&pMsg->srcId), s, pMsg->seq, pMsg->term, pMsg->beginIndex, pMsg->lastIndex, pMsg->lastTerm, + pMsg->startTime, pMsg->dataLen); } void syncLogSendSyncSnapshotRsp(SSyncNode* pSyncNode, const SyncSnapshotRsp* pMsg, const char* s) { - if (!(sDebugFlag & DEBUG_DEBUG)) return; - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - sNDebug(pSyncNode, - "send sync-snapshot-rsp to %s:%u, %s, ack:%d, term:%" PRId64 ", begin:%" PRId64 ", lst:%" PRId64 + "send sync-snapshot-rsp to dnode:%d, %s, ack:%d, term:%" PRId64 ", begin:%" PRId64 ", lst:%" PRId64 ", lterm:%" PRId64 ", stime:%" PRId64, - host, port, s, pMsg->ack, pMsg->term, pMsg->snapBeginIndex, pMsg->lastIndex, pMsg->lastTerm, pMsg->startTime); + DID(&pMsg->destId), s, pMsg->ack, pMsg->term, pMsg->snapBeginIndex, pMsg->lastIndex, pMsg->lastTerm, + pMsg->startTime); } void syncLogRecvSyncSnapshotRsp(SSyncNode* pSyncNode, const SyncSnapshotRsp* pMsg, const char* s) { - if (!(sDebugFlag & DEBUG_DEBUG)) return; - - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sNDebug(pSyncNode, - "recv sync-snapshot-rsp from %s:%u, %s, ack:%d, term:%" PRId64 ", begin:%" PRId64 ", lst:%" PRId64 + "recv sync-snapshot-rsp from dnode:%d, %s, ack:%d, term:%" PRId64 ", begin:%" PRId64 ", lst:%" PRId64 ", lterm:%" PRId64 ", stime:%" PRId64, - host, port, s, pMsg->ack, pMsg->term, pMsg->snapBeginIndex, pMsg->lastIndex, pMsg->lastTerm, pMsg->startTime); + DID(&pMsg->srcId), s, pMsg->ack, pMsg->term, pMsg->snapBeginIndex, pMsg->lastIndex, pMsg->lastTerm, + pMsg->startTime); } void syncLogRecvAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s) { - if (!(sDebugFlag & DEBUG_TRACE)) return; - - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, - "recv sync-append-entries from %s:%d {term:%" PRId64 ", pre-index:%" PRId64 ", pre-term:%" PRId64 + "recv sync-append-entries from dnode:%d {term:%" PRId64 ", pre-index:%" PRId64 ", pre-term:%" PRId64 ", cmt:%" PRId64 ", pterm:%" PRId64 ", datalen:%d}, %s", - host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->commitIndex, pMsg->privateTerm, + DID(&pMsg->srcId), pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->commitIndex, pMsg->privateTerm, pMsg->dataLen, s); } void syncLogSendAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s) { - if (!(sDebugFlag & DEBUG_TRACE)) return; - - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "send sync-append-entries to %s:%d, {term:%" PRId64 ", pre-index:%" PRId64 ", pre-term:%" PRId64 + "send sync-append-entries to dnode:%d, {term:%" PRId64 ", pre-index:%" PRId64 ", pre-term:%" PRId64 ", lsend-index:%" PRId64 ", cmt:%" PRId64 ", datalen:%d}, %s", - host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, (pMsg->prevLogIndex + 1), pMsg->commitIndex, - pMsg->dataLen, s); + DID(&pMsg->destId), pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, (pMsg->prevLogIndex + 1), + pMsg->commitIndex, pMsg->dataLen, s); } void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, int32_t voteGranted, const char* s) { - // if (!(sDebugFlag & DEBUG_TRACE)) return; - - char logBuf[256]; - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - if (voteGranted == -1) { sNInfo(pSyncNode, - "recv sync-request-vote from %s:%d, {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, %s", host, - port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); + "recv sync-request-vote from dnode:%d, {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, %s", + DID(&pMsg->srcId), pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); } else { sNInfo(pSyncNode, - "recv sync-request-vote from %s:%d, {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, granted:%d", - host, port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, voteGranted); + "recv sync-request-vote from dnode:%d, {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 + "}, granted:%d", + DID(&pMsg->srcId), pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, voteGranted); } } void syncLogSendRequestVote(SSyncNode* pNode, const SyncRequestVote* pMsg, const char* s) { - // if (!(sDebugFlag & DEBUG_TRACE)) return; - - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - sNInfo(pNode, "send sync-request-vote to %s:%d {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, %s", host, - port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); + sNInfo(pNode, "send sync-request-vote to dnode:%d {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, %s", + DID(&pMsg->destId), pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); } void syncLogRecvRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s) { - // if (!(sDebugFlag & DEBUG_TRACE)) return; - - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sNInfo(pSyncNode, "recv sync-request-vote-reply from %s:%d {term:%" PRId64 ", grant:%d}, %s", host, port, pMsg->term, - pMsg->voteGranted, s); + sNInfo(pSyncNode, "recv sync-request-vote-reply from dnode:%d {term:%" PRId64 ", grant:%d}, %s", DID(&pMsg->srcId), + pMsg->term, pMsg->voteGranted, s); } void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s) { - // if (!(sDebugFlag & DEBUG_TRACE)) return; - - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - sNInfo(pSyncNode, "send sync-request-vote-reply to %s:%d {term:%" PRId64 ", grant:%d}, %s", host, port, pMsg->term, - pMsg->voteGranted, s); + sNInfo(pSyncNode, "send sync-request-vote-reply to dnode:%d {term:%" PRId64 ", grant:%d}, %s", DID(&pMsg->destId), + pMsg->term, pMsg->voteGranted, s); } diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index 8a0a35ce336bfa9d1ed1d01fb96b79a98da24edb..6bd9625276c898184341cec291abed6c304700fb 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -23,21 +23,21 @@ static void voteGrantedClearVotes(SVotesGranted *pVotesGranted) { pVotesGranted->votes = 0; } -SVotesGranted *voteGrantedCreate(SSyncNode *pSyncNode) { +SVotesGranted *voteGrantedCreate(SSyncNode *pNode) { SVotesGranted *pVotesGranted = taosMemoryCalloc(1, sizeof(SVotesGranted)); if (pVotesGranted == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pVotesGranted->replicas = &(pSyncNode->replicasId); - pVotesGranted->replicaNum = pSyncNode->replicaNum; + pVotesGranted->replicas = &pNode->replicasId; + pVotesGranted->replicaNum = pNode->replicaNum; voteGrantedClearVotes(pVotesGranted); pVotesGranted->term = 0; - pVotesGranted->quorum = pSyncNode->quorum; + pVotesGranted->quorum = pNode->quorum; pVotesGranted->toLeader = false; - pVotesGranted->pSyncNode = pSyncNode; + pVotesGranted->pNode = pNode; return pVotesGranted; } @@ -48,33 +48,33 @@ void voteGrantedDestroy(SVotesGranted *pVotesGranted) { } } -void voteGrantedUpdate(SVotesGranted *pVotesGranted, SSyncNode *pSyncNode) { - pVotesGranted->replicas = &(pSyncNode->replicasId); - pVotesGranted->replicaNum = pSyncNode->replicaNum; +void voteGrantedUpdate(SVotesGranted *pVotesGranted, SSyncNode *pNode) { + pVotesGranted->replicas = &pNode->replicasId; + pVotesGranted->replicaNum = pNode->replicaNum; voteGrantedClearVotes(pVotesGranted); pVotesGranted->term = 0; - pVotesGranted->quorum = pSyncNode->quorum; + pVotesGranted->quorum = pNode->quorum; pVotesGranted->toLeader = false; - pVotesGranted->pSyncNode = pSyncNode; + pVotesGranted->pNode = pNode; } bool voteGrantedMajority(SVotesGranted *pVotesGranted) { return pVotesGranted->votes >= pVotesGranted->quorum; } void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { if (!pMsg->voteGranted) { - sNFatal(pVotesGranted->pSyncNode, "vote granted should be true"); + sNFatal(pVotesGranted->pNode, "vote granted should be true"); return; } if (pMsg->term != pVotesGranted->term) { - sNTrace(pVotesGranted->pSyncNode, "vote grant term:%" PRId64 " not matched with msg term:%" PRId64, - pVotesGranted->term, pMsg->term); + sNTrace(pVotesGranted->pNode, "vote grant term:%" PRId64 " not matched with msg term:%" PRId64, pVotesGranted->term, + pMsg->term); return; } - if (!syncUtilSameId(&pVotesGranted->pSyncNode->myRaftId, &pMsg->destId)) { - sNFatal(pVotesGranted->pSyncNode, "vote granted raftId not matched with msg"); + if (!syncUtilSameId(&pVotesGranted->pNode->myRaftId, &pMsg->destId)) { + sNFatal(pVotesGranted->pNode, "vote granted raftId not matched with msg"); return; } @@ -86,7 +86,7 @@ void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { } } if ((j == -1) || !(j >= 0 && j < pVotesGranted->replicaNum)) { - sNFatal(pVotesGranted->pSyncNode, "invalid msg srcId, index:%d", j); + sNFatal(pVotesGranted->pNode, "invalid msg srcId, index:%d", j); return; } @@ -96,7 +96,7 @@ void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { } if (pVotesGranted->votes > pVotesGranted->replicaNum) { - sNFatal(pVotesGranted->pSyncNode, "votes:%d not matched with replicaNum:%d", pVotesGranted->votes, + sNFatal(pVotesGranted->pNode, "votes:%d not matched with replicaNum:%d", pVotesGranted->votes, pVotesGranted->replicaNum); return; } @@ -108,17 +108,17 @@ void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term) { pVotesGranted->toLeader = false; } -SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode) { +SVotesRespond *votesRespondCreate(SSyncNode *pNode) { SVotesRespond *pVotesRespond = taosMemoryCalloc(1, sizeof(SVotesRespond)); if (pVotesRespond == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pVotesRespond->replicas = &(pSyncNode->replicasId); - pVotesRespond->replicaNum = pSyncNode->replicaNum; + pVotesRespond->replicas = &pNode->replicasId; + pVotesRespond->replicaNum = pNode->replicaNum; pVotesRespond->term = 0; - pVotesRespond->pSyncNode = pSyncNode; + pVotesRespond->pNode = pNode; return pVotesRespond; } @@ -129,11 +129,11 @@ void votesRespondDestory(SVotesRespond *pVotesRespond) { } } -void votesRespondUpdate(SVotesRespond *pVotesRespond, SSyncNode *pSyncNode) { - pVotesRespond->replicas = &(pSyncNode->replicasId); - pVotesRespond->replicaNum = pSyncNode->replicaNum; +void votesRespondUpdate(SVotesRespond *pVotesRespond, SSyncNode *pNode) { + pVotesRespond->replicas = &pNode->replicasId; + pVotesRespond->replicaNum = pNode->replicaNum; pVotesRespond->term = 0; - pVotesRespond->pSyncNode = pSyncNode; + pVotesRespond->pNode = pNode; } bool votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId) { @@ -149,7 +149,7 @@ bool votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId) { void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *pMsg) { if (pVotesRespond->term != pMsg->term) { - sNTrace(pVotesRespond->pSyncNode, "vote respond add error"); + sNTrace(pVotesRespond->pNode, "vote respond add error"); return; } @@ -160,7 +160,7 @@ void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *p } } - sNFatal(pVotesRespond->pSyncNode, "votes respond not found"); + sNFatal(pVotesRespond->pNode, "votes respond not found"); } void votesRespondReset(SVotesRespond *pVotesRespond, SyncTerm term) { diff --git a/source/libs/sync/test/syncLocalCmdTest.cpp b/source/libs/sync/test/syncLocalCmdTest.cpp index 8003cce7cc7e386223d5fc9116acc1f9cdec229c..2c839d0acb7ab6301fca49222ef37456f2b11099 100644 --- a/source/libs/sync/test/syncLocalCmdTest.cpp +++ b/source/libs/sync/test/syncLocalCmdTest.cpp @@ -16,8 +16,8 @@ SyncLocalCmd *createMsg() { pMsg->srcId.vgId = 100; pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); pMsg->destId.vgId = 100; - pMsg->sdNewTerm = 123; - pMsg->fcIndex = 456; + // pMsg->sdNewTerm = 123; + // pMsg->fcIndex = 456; pMsg->cmd = SYNC_LOCAL_CMD_STEP_DOWN; return pMsg; diff --git a/source/libs/sync/test/syncRaftCfgIndexTest.cpp b/source/libs/sync/test/syncRaftCfgIndexTest.cpp index e6d3f23b58e471609b6127c497a34176731fb182..54000654431b01ecebe453269d47c07c3d4fb570 100644 --- a/source/libs/sync/test/syncRaftCfgIndexTest.cpp +++ b/source/libs/sync/test/syncRaftCfgIndexTest.cpp @@ -51,26 +51,26 @@ SSyncCfg* createSyncCfg() { const char* pFile = "./raft_config_index.json"; void test1() { - int32_t code = raftCfgIndexCreateFile(pFile); - ASSERT(code == 0); + // int32_t code = raftCfgIndexCreateFile(pFile); + // ASSERT(code == 0); - SRaftCfgIndex* pRaftCfgIndex = raftCfgIndexOpen(pFile); + // SRaftCfgIndex* pRaftCfgIndex = raftCfgIndexOpen(pFile); - raftCfgIndexClose(pRaftCfgIndex); + // raftCfgIndexClose(pRaftCfgIndex); } void test2() { - SRaftCfgIndex* pRaftCfgIndex = raftCfgIndexOpen(pFile); - for (int i = 0; i < 500; ++i) { - raftCfgIndexAddConfigIndex(pRaftCfgIndex, i); - } - raftCfgIndexPersist(pRaftCfgIndex); - raftCfgIndexClose(pRaftCfgIndex); + // SRaftCfgIndex* pRaftCfgIndex = raftCfgIndexOpen(pFile); + // for (int i = 0; i < 500; ++i) { + // raftCfgIndexAddConfigIndex(pRaftCfgIndex, i); + // } + // raftCfgIndexPersist(pRaftCfgIndex); + // raftCfgIndexClose(pRaftCfgIndex); } void test3() { - SRaftCfgIndex* pRaftCfgIndex = raftCfgIndexOpen(pFile); - raftCfgIndexClose(pRaftCfgIndex); + // SRaftCfgIndex* pRaftCfgIndex = raftCfgIndexOpen(pFile); + // raftCfgIndexClose(pRaftCfgIndex); } int main() { diff --git a/source/libs/sync/test/syncRaftCfgTest.cpp b/source/libs/sync/test/syncRaftCfgTest.cpp index c841a68fde081a0c5fc666dc33752dd62c2e321d..8e63b4ca09d00484e33e8af9123aded673ffa51c 100644 --- a/source/libs/sync/test/syncRaftCfgTest.cpp +++ b/source/libs/sync/test/syncRaftCfgTest.cpp @@ -67,12 +67,12 @@ void test3() { if (taosCheckExistFile(s)) { printf("%s file: %s already exist! \n", (char*)__FUNCTION__, s); } else { - SRaftCfgMeta meta; - meta.isStandBy = 7; - meta.snapshotStrategy = 9; - meta.batchSize = 10; - meta.lastConfigIndex = 789; - raftCfgCreateFile(pCfg, meta, s); + // SRaftCfgMeta meta; + // meta.isStandBy = 7; + // meta.snapshotStrategy = 9; + // meta.batchSize = 10; + // meta.lastConfigIndex = 789; + // raftCfgCreateFile(pCfg, meta, s); printf("%s create json file: %s \n", (char*)__FUNCTION__, s); } @@ -80,37 +80,37 @@ void test3() { } void test4() { - SRaftCfg* pCfg = raftCfgOpen("./test3_raft_cfg.json"); - assert(pCfg != NULL); + // SRaftCfg* pCfg = raftCfgOpen("./test3_raft_cfg.json"); + // assert(pCfg != NULL); - int32_t ret = raftCfgClose(pCfg); - assert(ret == 0); + // int32_t ret = raftCfgClose(pCfg); + // assert(ret == 0); } void test5() { - SRaftCfg* pCfg = raftCfgOpen("./test3_raft_cfg.json"); - assert(pCfg != NULL); + // SRaftCfg* pCfg = raftCfgOpen("./test3_raft_cfg.json"); + // assert(pCfg != NULL); - pCfg->cfg.myIndex = taosGetTimestampSec(); - pCfg->isStandBy += 2; - pCfg->snapshotStrategy += 3; - pCfg->batchSize += 4; - pCfg->lastConfigIndex += 1000; + // pCfg->cfg.myIndex = taosGetTimestampSec(); + // pCfg->isStandBy += 2; + // pCfg->snapshotStrategy += 3; + // pCfg->batchSize += 4; + // pCfg->lastConfigIndex += 1000; - pCfg->configIndexCount = 5; - for (int i = 0; i < MAX_CONFIG_INDEX_COUNT; ++i) { - (pCfg->configIndexArr)[i] = -1; - } - for (int i = 0; i < pCfg->configIndexCount; ++i) { - (pCfg->configIndexArr)[i] = i * 100; - } + // pCfg->configIndexCount = 5; + // for (int i = 0; i < MAX_CONFIG_INDEX_COUNT; ++i) { + // (pCfg->configIndexArr)[i] = -1; + // } + // for (int i = 0; i < pCfg->configIndexCount; ++i) { + // (pCfg->configIndexArr)[i] = i * 100; + // } - raftCfgPersist(pCfg); + // // raftCfgPersist(pCfg); - printf("%s update json file: %s myIndex->%d \n", (char*)__FUNCTION__, "./test3_raft_cfg.json", pCfg->cfg.myIndex); + // printf("%s update json file: %s myIndex->%d \n", (char*)__FUNCTION__, "./test3_raft_cfg.json", pCfg->cfg.myIndex); - int32_t ret = raftCfgClose(pCfg); - assert(ret == 0); + // int32_t ret = raftCfgClose(pCfg); + // assert(ret == 0); } int main() { diff --git a/source/libs/sync/test/syncRaftStoreTest.cpp b/source/libs/sync/test/syncRaftStoreTest.cpp index 87798a7d808db32f176901468cf8c7dc3a6dd044..a8022184ef6885e9fb3c8e17e032bce94587b46f 100644 --- a/source/libs/sync/test/syncRaftStoreTest.cpp +++ b/source/libs/sync/test/syncRaftStoreTest.cpp @@ -33,35 +33,35 @@ int main() { initRaftId(); - SRaftStore* pRaftStore = raftStoreOpen("./test_raft_store.json"); - assert(pRaftStore != NULL); - raftStoreLog2((char*)"==raftStoreOpen==", pRaftStore); + // SRaftStore* pRaftStore = raftStoreOpen("./test_raft_store.json"); + // assert(pRaftStore != NULL); + // raftStoreLog2((char*)"==raftStoreOpen==", pRaftStore); - raftStoreSetTerm(pRaftStore, 100); - raftStoreLog2((char*)"==raftStoreSetTerm==", pRaftStore); + // raftStoreSetTerm(pRaftStore, 100); + // raftStoreLog2((char*)"==raftStoreSetTerm==", pRaftStore); - raftStoreVote(pRaftStore, &ids[0]); - raftStoreLog2((char*)"==raftStoreVote==", pRaftStore); + // raftStoreVote(pRaftStore, &ids[0]); + // raftStoreLog2((char*)"==raftStoreVote==", pRaftStore); - raftStoreClearVote(pRaftStore); - raftStoreLog2((char*)"==raftStoreClearVote==", pRaftStore); + // raftStoreClearVote(pRaftStore); + // raftStoreLog2((char*)"==raftStoreClearVote==", pRaftStore); - raftStoreVote(pRaftStore, &ids[1]); - raftStoreLog2((char*)"==raftStoreVote==", pRaftStore); + // raftStoreVote(pRaftStore, &ids[1]); + // raftStoreLog2((char*)"==raftStoreVote==", pRaftStore); - raftStoreNextTerm(pRaftStore); - raftStoreLog2((char*)"==raftStoreNextTerm==", pRaftStore); + // raftStoreNextTerm(pRaftStore); + // raftStoreLog2((char*)"==raftStoreNextTerm==", pRaftStore); - raftStoreNextTerm(pRaftStore); - raftStoreLog2((char*)"==raftStoreNextTerm==", pRaftStore); + // raftStoreNextTerm(pRaftStore); + // raftStoreLog2((char*)"==raftStoreNextTerm==", pRaftStore); - raftStoreNextTerm(pRaftStore); - raftStoreLog2((char*)"==raftStoreNextTerm==", pRaftStore); + // raftStoreNextTerm(pRaftStore); + // raftStoreLog2((char*)"==raftStoreNextTerm==", pRaftStore); - raftStoreNextTerm(pRaftStore); - raftStoreLog2((char*)"==raftStoreNextTerm==", pRaftStore); + // raftStoreNextTerm(pRaftStore); + // raftStoreLog2((char*)"==raftStoreNextTerm==", pRaftStore); - raftStoreClose(pRaftStore); + // raftStoreClose(pRaftStore); return 0; } diff --git a/source/libs/sync/test/syncSnapshotReceiverTest.cpp b/source/libs/sync/test/syncSnapshotReceiverTest.cpp index 49b06a7d1bfca855d661e1c332f951b8c0b7f657..1fca04a1adf866460c1c089d2b6d554c7bc46ecc 100644 --- a/source/libs/sync/test/syncSnapshotReceiverTest.cpp +++ b/source/libs/sync/test/syncSnapshotReceiverTest.cpp @@ -29,7 +29,7 @@ int32_t SnapshotDoWrite(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_ SSyncSnapshotReceiver* createReceiver() { SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(*pSyncNode)); - pSyncNode->pRaftStore = (SRaftStore*)taosMemoryMalloc(sizeof(*(pSyncNode->pRaftStore))); + // pSyncNode->pRaftStore = (SRaftStore*)taosMemoryMalloc(sizeof(*(pSyncNode->pRaftStore))); pSyncNode->pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(*(pSyncNode->pFsm))); #if 0 diff --git a/source/libs/sync/test/syncSnapshotSenderTest.cpp b/source/libs/sync/test/syncSnapshotSenderTest.cpp index bb697d541acbbec35289e723bb351c7117a3f3e6..a1768c2ce50302af7a592f5f8fee5174b550f8fc 100644 --- a/source/libs/sync/test/syncSnapshotSenderTest.cpp +++ b/source/libs/sync/test/syncSnapshotSenderTest.cpp @@ -29,7 +29,7 @@ int32_t SnapshotDoWrite(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_ SSyncSnapshotSender* createSender() { SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(*pSyncNode)); - pSyncNode->pRaftStore = (SRaftStore*)taosMemoryMalloc(sizeof(*(pSyncNode->pRaftStore))); + // pSyncNode->pRaftStore = (SRaftStore*)taosMemoryMalloc(sizeof(*(pSyncNode->pRaftStore))); pSyncNode->pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(*(pSyncNode->pFsm))); #if 0 diff --git a/source/libs/sync/test/sync_test_lib/inc/syncTest.h b/source/libs/sync/test/sync_test_lib/inc/syncTest.h index ae4cc6ed6f7839eced1f829822b22d4b03f62208..fc52e83aa7c22f383e13bfbd3395d5fe3cb0b73e 100644 --- a/source/libs/sync/test/sync_test_lib/inc/syncTest.h +++ b/source/libs/sync/test/sync_test_lib/inc/syncTest.h @@ -474,6 +474,11 @@ void syncLocalCmdPrint2(char* s, const SyncLocalCmd* pMsg); void syncLocalCmdLog(const SyncLocalCmd* pMsg); void syncLocalCmdLog2(char* s, const SyncLocalCmd* pMsg); +char* syncUtilPrintBin(char* ptr, uint32_t len); +char* syncUtilPrintBin2(char* ptr, uint32_t len); +void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port); +uint64_t syncUtilAddr2U64(const char* host, uint16_t port); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/test/sync_test_lib/src/syncBatch.c b/source/libs/sync/test/sync_test_lib/src/syncBatch.c index cb4bed1e671e40ec58e028a4a614e6c5db955dc5..4e0b53467102507c74abde4d05ca3361767c41e1 100644 --- a/source/libs/sync/test/sync_test_lib/src/syncBatch.c +++ b/source/libs/sync/test/sync_test_lib/src/syncBatch.c @@ -311,7 +311,7 @@ cJSON* syncAppendEntriesBatch2Json(const SyncAppendEntriesBatch* pMsg) { cJSON* pTmp = pSrcId; char host[128] = {0}; uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); + // syncUtilU642Addr(u64, host, sizeof(host), &port); cJSON_AddStringToObject(pTmp, "addr_host", host); cJSON_AddNumberToObject(pTmp, "addr_port", port); } diff --git a/source/libs/sync/test/sync_test_lib/src/syncIO.c b/source/libs/sync/test/sync_test_lib/src/syncIO.c index 4b305f823ff3a1002c4e8c29ed0c3afe02c371a7..2e0078558662f5a8aaee5af792f2c28a0cbec5b2 100644 --- a/source/libs/sync/test/sync_test_lib/src/syncIO.c +++ b/source/libs/sync/test/sync_test_lib/src/syncIO.c @@ -73,7 +73,7 @@ int32_t syncIOSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { int32_t ret = 0; { - syncUtilMsgNtoH(pMsg->pCont); + // syncUtilMsgNtoH(pMsg->pCont); char logBuf[256] = {0}; snprintf(logBuf, sizeof(logBuf), "==syncIOSendMsg== %s:%d msgType:%d", pEpSet->eps[0].fqdn, pEpSet->eps[0].port, @@ -376,7 +376,7 @@ static void *syncIOConsumerFunc(void *param) { } static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { - syncUtilMsgNtoH(pMsg->pCont); + // syncUtilMsgNtoH(pMsg->pCont); syncRpcMsgLog2((char *)"==syncIOProcessRequest==", pMsg); SSyncIO *io = pParent; @@ -432,9 +432,9 @@ static void syncIOTickQ(void *param, void *tmrId) { SSyncIO *io = (SSyncIO *)param; SRaftId srcId, destId; - srcId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port); + // srcId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port); srcId.vgId = -1; - destId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port); + // destId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port); destId.vgId = -1; SyncPingReply *pMsg = syncPingReplyBuild2(&srcId, &destId, -1, "syncIOTickQ"); @@ -454,9 +454,9 @@ static void syncIOTickPing(void *param, void *tmrId) { SSyncIO *io = (SSyncIO *)param; SRaftId srcId, destId; - srcId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port); + // srcId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port); srcId.vgId = -1; - destId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port); + // destId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port); destId.vgId = -1; SyncPing *pMsg = syncPingBuild2(&srcId, &destId, -1, "syncIOTickPing"); // SyncPing *pMsg = syncPingBuild3(&srcId, &destId); @@ -470,4 +470,68 @@ static void syncIOTickPing(void *param, void *tmrId) { taosTmrReset(syncIOTickPing, io->pingTimerMS, io, io->timerMgr, &io->pingTimer); } -void syncEntryDestory(SSyncRaftEntry* pEntry) {} \ No newline at end of file +void syncEntryDestory(SSyncRaftEntry* pEntry) {} + + +void syncUtilMsgNtoH(void* msg) { + SMsgHead* pHead = msg; + pHead->contLen = ntohl(pHead->contLen); + pHead->vgId = ntohl(pHead->vgId); +} + +static inline bool syncUtilCanPrint(char c) { + if (c >= 32 && c <= 126) { + return true; + } else { + return false; + } +} + +char* syncUtilPrintBin(char* ptr, uint32_t len) { + int64_t memLen = (int64_t)(len + 1); + char* s = taosMemoryMalloc(memLen); + ASSERT(s != NULL); + memset(s, 0, len + 1); + memcpy(s, ptr, len); + + for (int32_t i = 0; i < len; ++i) { + if (!syncUtilCanPrint(s[i])) { + s[i] = '.'; + } + } + return s; +} + +char* syncUtilPrintBin2(char* ptr, uint32_t len) { + uint32_t len2 = len * 4 + 1; + char* s = taosMemoryMalloc(len2); + ASSERT(s != NULL); + memset(s, 0, len2); + + char* p = s; + for (int32_t i = 0; i < len; ++i) { + int32_t n = sprintf(p, "%d,", ptr[i]); + p += n; + } + return s; +} + +void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port) { + uint32_t hostU32 = (uint32_t)((u64 >> 32) & 0x00000000FFFFFFFF); + + struct in_addr addr = {.s_addr = hostU32}; + taosInetNtoa(addr, host, len); + *port = (uint16_t)((u64 & 0x00000000FFFF0000) >> 16); +} + +uint64_t syncUtilAddr2U64(const char* host, uint16_t port) { + uint32_t hostU32 = taosGetIpv4FromFqdn(host); + if (hostU32 == (uint32_t)-1) { + sError("failed to resolve ipv4 addr, host:%s", host); + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + uint64_t u64 = (((uint64_t)hostU32) << 32) | (((uint32_t)port) << 16); + return u64; +} \ No newline at end of file diff --git a/source/libs/sync/test/sync_test_lib/src/syncIndexMgrDebug.c b/source/libs/sync/test/sync_test_lib/src/syncIndexMgrDebug.c index 1d3198c51d285f7a3de97e1c59b0676e1b15142c..c951eef7616c0afbe20cab19047113fbe342e3ac 100644 --- a/source/libs/sync/test/sync_test_lib/src/syncIndexMgrDebug.c +++ b/source/libs/sync/test/sync_test_lib/src/syncIndexMgrDebug.c @@ -53,7 +53,7 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { cJSON *pReplicas = cJSON_CreateArray(); cJSON_AddItemToObject(pRoot, "replicas", pReplicas); for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { - cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pSyncIndexMgr->replicas))[i])); + // cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pSyncIndexMgr->replicas))[i])); } { @@ -76,7 +76,7 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { cJSON_AddItemToObject(pRoot, "privateTerm", pIndex); } - snprintf(u64buf, sizeof(u64buf), "%p", pSyncIndexMgr->pSyncNode); + // snprintf(u64buf, sizeof(u64buf), "%p", pSyncIndexMgr->pSyncNode); cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); } diff --git a/source/libs/sync/test/sync_test_lib/src/syncMainDebug.c b/source/libs/sync/test/sync_test_lib/src/syncMainDebug.c index 6b461da0e562ed1c153003f1cd7d098699d88737..1dbf4fb4fb70333d6a0cf0cb0a61138e58877504 100644 --- a/source/libs/sync/test/sync_test_lib/src/syncMainDebug.c +++ b/source/libs/sync/test/sync_test_lib/src/syncMainDebug.c @@ -23,7 +23,7 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { if (pSyncNode != NULL) { // init by SSyncInfo cJSON_AddNumberToObject(pRoot, "vgId", pSyncNode->vgId); - cJSON_AddItemToObject(pRoot, "SRaftCfg", raftCfg2Json(pSyncNode->pRaftCfg)); + // cJSON_AddItemToObject(pRoot, "SRaftCfg", raftCfg2Json(pSyncNode->pRaftCfg)); cJSON_AddStringToObject(pRoot, "path", pSyncNode->path); cJSON_AddStringToObject(pRoot, "raftStorePath", pSyncNode->raftStorePath); cJSON_AddStringToObject(pRoot, "configPath", pSyncNode->configPath); @@ -44,8 +44,8 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { // init internal cJSON* pMe = syncUtilNodeInfo2Json(&pSyncNode->myNodeInfo); cJSON_AddItemToObject(pRoot, "myNodeInfo", pMe); - cJSON* pRaftId = syncUtilRaftId2Json(&pSyncNode->myRaftId); - cJSON_AddItemToObject(pRoot, "myRaftId", pRaftId); + // cJSON* pRaftId = syncUtilRaftId2Json(&pSyncNode->myRaftId); + // cJSON_AddItemToObject(pRoot, "myRaftId", pRaftId); cJSON_AddNumberToObject(pRoot, "peersNum", pSyncNode->peersNum); cJSON* pPeers = cJSON_CreateArray(); @@ -56,22 +56,22 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { cJSON* pPeersId = cJSON_CreateArray(); cJSON_AddItemToObject(pRoot, "peersId", pPeersId); for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { - cJSON_AddItemToArray(pPeersId, syncUtilRaftId2Json(&pSyncNode->peersId[i])); + // cJSON_AddItemToArray(pPeersId, syncUtilRaftId2Json(&pSyncNode->peersId[i])); } cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncNode->replicaNum); cJSON* pReplicasId = cJSON_CreateArray(); cJSON_AddItemToObject(pRoot, "replicasId", pReplicasId); for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { - cJSON_AddItemToArray(pReplicasId, syncUtilRaftId2Json(&pSyncNode->replicasId[i])); + // cJSON_AddItemToArray(pReplicasId, syncUtilRaftId2Json(&pSyncNode->replicasId[i])); } // raft algorithm snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pFsm); cJSON_AddStringToObject(pRoot, "pFsm", u64buf); cJSON_AddNumberToObject(pRoot, "quorum", pSyncNode->quorum); - cJSON* pLaderCache = syncUtilRaftId2Json(&pSyncNode->leaderCache); - cJSON_AddItemToObject(pRoot, "leaderCache", pLaderCache); + // cJSON* pLaderCache = syncUtilRaftId2Json(&pSyncNode->leaderCache); + // cJSON_AddItemToObject(pRoot, "leaderCache", pLaderCache); // life cycle snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->rid); @@ -80,7 +80,7 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { // tla+ server vars cJSON_AddNumberToObject(pRoot, "state", pSyncNode->state); cJSON_AddStringToObject(pRoot, "state_str", syncStr(pSyncNode->state)); - cJSON_AddItemToObject(pRoot, "pRaftStore", raftStore2Json(pSyncNode->pRaftStore)); + // cJSON_AddItemToObject(pRoot, "pRaftStore", raftStore2Json(&pSyncNode.raftStore)); // tla+ candidate vars cJSON_AddItemToObject(pRoot, "pVotesGranted", voteGranted2Json(pSyncNode->pVotesGranted)); @@ -199,9 +199,9 @@ inline char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { ", sby:%d, " "r-num:%d, " "lcfg:%" PRId64 ", chging:%d, rsto:%d", - pSyncNode->vgId, syncStr(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, - logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, - pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing, pSyncNode->restoreFinish); + pSyncNode->vgId, syncStr(pSyncNode->state), pSyncNode->raftStore.currentTerm, pSyncNode->commitIndex, + logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->raftCfg.isStandBy, pSyncNode->replicaNum, + pSyncNode->raftCfg.lastConfigIndex, pSyncNode->changing, pSyncNode->restoreFinish); return s; } @@ -243,7 +243,7 @@ int32_t syncNodePingPeers(SSyncNode* pSyncNode) { int32_t syncNodePingAll(SSyncNode* pSyncNode) { int32_t ret = 0; - for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) { SRaftId* destId = &(pSyncNode->replicasId[i]); SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, destId, pSyncNode->vgId); ret = syncNodePing(pSyncNode, destId, pMsg); diff --git a/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c b/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c index 3b6007df6bd346986a7e6a7b46ddb0300761d2fc..5f011ffe6902be629d82023d28fdfb7508fc68d7 100644 --- a/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c +++ b/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c @@ -181,7 +181,7 @@ cJSON* syncPing2Json(const SyncPing* pMsg) { cJSON* pTmp = pSrcId; char host[128] = {0}; uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); + // syncUtilU642Addr(u64, host, sizeof(host), &port); cJSON_AddStringToObject(pTmp, "addr_host", host); cJSON_AddNumberToObject(pTmp, "addr_port", port); } @@ -748,7 +748,7 @@ cJSON* syncSnapshotSend2Json(const SyncSnapshotSend* pMsg) { snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->lastConfigIndex); cJSON_AddStringToObject(pRoot, "lastConfigIndex", u64buf); - cJSON_AddItemToObject(pRoot, "lastConfig", syncCfg2Json((SSyncCfg*)&(pMsg->lastConfig))); + // cJSON_AddItemToObject(pRoot, "lastConfig", syncCfg2Json((SSyncCfg*)&(pMsg->lastConfig))); snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->lastTerm); cJSON_AddStringToObject(pRoot, "lastTerm", u64buf); @@ -2858,11 +2858,11 @@ cJSON* syncLocalCmd2Json(const SyncLocalCmd* pMsg) { cJSON_AddNumberToObject(pRoot, "cmd", pMsg->cmd); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->sdNewTerm); - cJSON_AddStringToObject(pRoot, "sd-new-term", u64buf); + // snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->sdNewTerm); + // cJSON_AddStringToObject(pRoot, "sd-new-term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->fcIndex); - cJSON_AddStringToObject(pRoot, "fc-index", u64buf); + // snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->fcIndex); + // cJSON_AddStringToObject(pRoot, "fc-index", u64buf); } cJSON* pJson = cJSON_CreateObject(); diff --git a/source/libs/sync/test/sync_test_lib/src/syncRaftCfgDebug.c b/source/libs/sync/test/sync_test_lib/src/syncRaftCfgDebug.c index 50ecc5d47804d54a4fa00aab34ec169706fcc1a5..bafa22ea640a508ffc7b3b8f51596fca9a7e3b22 100644 --- a/source/libs/sync/test/sync_test_lib/src/syncRaftCfgDebug.c +++ b/source/libs/sync/test/sync_test_lib/src/syncRaftCfgDebug.c @@ -17,18 +17,19 @@ #include "syncTest.h" char *syncCfg2Str(SSyncCfg *pSyncCfg) { - cJSON *pJson = syncCfg2Json(pSyncCfg); - char *serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; + // cJSON *pJson = syncCfg2Json(pSyncCfg); + // char *serialized = cJSON_Print(pJson); + // cJSON_Delete(pJson); + // return serialized; + return ""; } int32_t syncCfgFromStr(const char *s, SSyncCfg *pSyncCfg) { cJSON *pRoot = cJSON_Parse(s); ASSERT(pRoot != NULL); - int32_t ret = syncCfgFromJson(pRoot, pSyncCfg); - ASSERT(ret == 0); + // int32_t ret = syncCfgFromJson(pRoot, pSyncCfg); + // ASSERT(ret == 0); cJSON_Delete(pRoot); return 0; diff --git a/source/libs/sync/test/sync_test_lib/src/syncRaftStoreDebug.c b/source/libs/sync/test/sync_test_lib/src/syncRaftStoreDebug.c index c462b3275dfaa9298aa312a5564be3148a5d3483..f6cd381e547b6c0d38782b29bafb0665d0c05379 100644 --- a/source/libs/sync/test/sync_test_lib/src/syncRaftStoreDebug.c +++ b/source/libs/sync/test/sync_test_lib/src/syncRaftStoreDebug.c @@ -41,8 +41,8 @@ cJSON *raftStore2Json(SRaftStore *pRaftStore) { cJSON_AddNumberToObject(pVoteFor, "vgId", pRaftStore->voteFor.vgId); cJSON_AddItemToObject(pRoot, "voteFor", pVoteFor); - int hasVoted = raftStoreHasVoted(pRaftStore); - cJSON_AddNumberToObject(pRoot, "hasVoted", hasVoted); + // int hasVoted = raftStoreHasVoted(pRaftStore); + // cJSON_AddNumberToObject(pRoot, "hasVoted", hasVoted); } cJSON *pJson = cJSON_CreateObject(); diff --git a/source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c b/source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c index f1237e528251d19501d09326782e5659abd34895..d8740de16ae46214c8c5ed2ab2e1d622c003c300 100644 --- a/source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c +++ b/source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c @@ -137,7 +137,7 @@ int32_t syncNodeOnPreSnapshot(SSyncNode *ths, SyncPreSnapshot *pMsg) { SyncPreSnapshotReply *pMsgReply = syncPreSnapshotReplyBuild(ths->vgId); pMsgReply->srcId = ths->myRaftId; pMsgReply->destId = pMsg->srcId; - pMsgReply->term = ths->pRaftStore->currentTerm; + pMsgReply->term = ths->raftStore.currentTerm; SSyncLogStoreData *pData = ths->pLogStore->data; SWal *pWal = pData->pWal; diff --git a/source/libs/sync/test/sync_test_lib/src/syncUtilDebug.c b/source/libs/sync/test/sync_test_lib/src/syncUtilDebug.c index b12f4e76a58fad992cb5b3ea545966e43469ea00..6802148578d159ff546276faf2be9f1171465aeb 100644 --- a/source/libs/sync/test/sync_test_lib/src/syncUtilDebug.c +++ b/source/libs/sync/test/sync_test_lib/src/syncUtilDebug.c @@ -29,8 +29,9 @@ cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p) { } char* syncUtilRaftId2Str(const SRaftId* p) { - cJSON* pJson = syncUtilRaftId2Json(p); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; + // cJSON* pJson = syncUtilRaftId2Json(p); + // char* serialized = cJSON_Print(pJson); + // cJSON_Delete(pJson); + // return serialized; + return ""; } diff --git a/source/libs/sync/test/sync_test_lib/src/syncVoteMgrDebug.c b/source/libs/sync/test/sync_test_lib/src/syncVoteMgrDebug.c index f1c26a97aa443f9fa27580b4b2138192ecc09401..f3ea5e07fd1e649084e15fe53d03d6606f987f16 100644 --- a/source/libs/sync/test/sync_test_lib/src/syncVoteMgrDebug.c +++ b/source/libs/sync/test/sync_test_lib/src/syncVoteMgrDebug.c @@ -25,7 +25,7 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { cJSON *pReplicas = cJSON_CreateArray(); cJSON_AddItemToObject(pRoot, "replicas", pReplicas); for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { - cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas))[i])); + // cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas))[i])); } int32_t *arr = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * pVotesGranted->replicaNum); for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { @@ -40,7 +40,7 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { cJSON_AddStringToObject(pRoot, "term", u64buf); cJSON_AddNumberToObject(pRoot, "quorum", pVotesGranted->quorum); cJSON_AddNumberToObject(pRoot, "toLeader", pVotesGranted->toLeader); - snprintf(u64buf, sizeof(u64buf), "%p", pVotesGranted->pSyncNode); + snprintf(u64buf, sizeof(u64buf), "%p", pVotesGranted->pNode); cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); bool majority = voteGrantedMajority(pVotesGranted); @@ -68,7 +68,7 @@ cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { cJSON *pReplicas = cJSON_CreateArray(); cJSON_AddItemToObject(pRoot, "replicas", pReplicas); for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { - cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesRespond->replicas))[i])); + // cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesRespond->replicas))[i])); } int32_t respondNum = 0; int32_t *arr = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * pVotesRespond->replicaNum); @@ -85,7 +85,7 @@ cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pVotesRespond->term); cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pVotesRespond->pSyncNode); + snprintf(u64buf, sizeof(u64buf), "%p", pVotesRespond->pNode); cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); } diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 10a99bb1fab525a9ff4569d55c3d688bb099ee46..0e20941b3ae472986d5116616eda05af93a83dc1 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -74,7 +74,12 @@ int32_t tdbTbcUpsert(TBC *pTbc, const void *pKey, int nKey, const void *pData, i int32_t tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg, int flags); -int32_t tdbTxnClose(TXN *pTxn); +int32_t tdbTxnCloseImpl(TXN *pTxn); +#define tdbTxnClose(pTxn) \ + do { \ + tdbTxnCloseImpl(pTxn); \ + (pTxn) = NULL; \ + } while (0) // other void tdbFree(void *); diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 5c1f264460630d12249cafa5ea20f8df5e279976..4cd6a39bfe00f56631abea8dfd3c935dff1e580b 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -74,7 +74,10 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg SBTree *pBt; int ret; - ASSERT(keyLen != 0); + if (keyLen == 0) { + tdbError("tdb/btree-open: key len cannot be zero."); + return -1; + } *ppBt = NULL; @@ -152,7 +155,11 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg tdbPostCommit(pPager->pEnv, txn); } - ASSERT(pgno != 0); + if (pgno == 0) { + tdbError("tdb/btree-open: pgno cannot be zero."); + tdbOsFree(pBt); + return -1; + } pBt->root = pgno; /* // TODO: pBt->root @@ -192,7 +199,7 @@ int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, in ret = tdbBtcMoveTo(&btc, pKey, kLen, &c); if (ret < 0) { tdbBtcClose(&btc); - ASSERT(0); + tdbError("tdb/btree-insert: btc move to failed with ret: %d.", ret); return -1; } @@ -202,17 +209,17 @@ int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, in if (c > 0) { btc.idx++; } else if (c == 0) { - // dup key not allowed - tdbError("unable to insert dup key. pKey: %p, kLen: %d, btc: %p, pTxn: %p", pKey, kLen, &btc, pTxn); - // ASSERT(0); + // dup key not allowed with insert + tdbBtcClose(&btc); + tdbError("tdb/btree-insert: dup key. pKey: %p, kLen: %d, btc: %p, pTxn: %p", pKey, kLen, &btc, pTxn); return -1; } } ret = tdbBtcUpsert(&btc, pKey, kLen, pVal, vLen, 1); if (ret < 0) { - ASSERT(0); tdbBtcClose(&btc); + tdbError("tdb/btree-insert: btc upsert failed with ret: %d.", ret); return -1; } @@ -233,7 +240,7 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) { ret = tdbBtcMoveTo(&btc, pKey, kLen, &c); if (ret < 0) { tdbBtcClose(&btc); - ASSERT(0); + tdbError("tdb/btree-delete: btc move to failed with ret: %d.", ret); return -1; } @@ -264,7 +271,7 @@ int tdbBtreeUpsert(SBTree *pBt, const void *pKey, int nKey, const void *pData, i // move the cursor ret = tdbBtcMoveTo(&btc, pKey, nKey, &c); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btree-upsert: btc move to failed with ret: %d.", ret); tdbBtcClose(&btc); return -1; } @@ -280,8 +287,8 @@ int tdbBtreeUpsert(SBTree *pBt, const void *pKey, int nKey, const void *pData, i ret = tdbBtcUpsert(&btc, pKey, nKey, pData, nData, c); if (ret < 0) { - ASSERT(0); tdbBtcClose(&btc); + tdbError("tdb/btree-upsert: btc upsert failed with ret: %d.", ret); return -1; } @@ -309,7 +316,8 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL ret = tdbBtcMoveTo(&btc, pKey, kLen, &cret); if (ret < 0) { tdbBtcClose(&btc); - ASSERT(0); + tdbError("tdb/btree-pget: btc move to failed with ret: %d.", ret); + return -1; } if (btc.idx < 0 || cret) { @@ -325,7 +333,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL pTKey = tdbRealloc(*ppKey, cd.kLen); if (pTKey == NULL) { tdbBtcClose(&btc); - ASSERT(0); + tdbError("tdb/btree-pget: realloc pTKey failed."); return -1; } *ppKey = pTKey; @@ -337,7 +345,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL pTVal = tdbRealloc(*ppVal, cd.vLen); if (pTVal == NULL) { tdbBtcClose(&btc); - ASSERT(0); + tdbError("tdb/btree-pget: realloc pTVal failed."); return -1; } *ppVal = pTVal; @@ -350,7 +358,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL } if (TDB_CELLDECODER_FREE_VAL(&cd)) { - tdbDebug("tdb btc/pget/2 decoder: %p pVal free: %p", &cd, cd.pVal); + tdbTrace("tdb btc/pget/2 decoder: %p pVal free: %p", &cd, cd.pVal); tdbFree(cd.pVal); } @@ -366,7 +374,9 @@ static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2 int mlen; int cret; - ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); + if (ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL)) { + // -1 is less than + } mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; cret = memcmp(pKey1, pKey2, mlen); @@ -381,36 +391,7 @@ static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2 } return cret; } -/* -static int tdbBtreeOpenImpl(SBTree *pBt) { - // Try to get the root page of the an existing btree - SPgno pgno; - SPage *pPage; - int ret; - - { - // 1. TODO: Search the main DB to check if the DB exists - ret = tdbPagerOpenDB(pBt->pPager, &pgno, true, pBt); - ASSERT(ret == 0); - } - if (pgno != 0) { - pBt->root = pgno; - return 0; - } - - // Try to create a new database - ret = tdbPagerAllocPage(pBt->pPager, &pgno); - if (ret < 0) { - ASSERT(0); - return -1; - } - - ASSERT(pgno != 0); - pBt->root = pgno; - return 0; -} -*/ int tdbBtreeInitPage(SPage *pPage, void *arg, int init) { SBTree *pBt; u8 flags; @@ -546,11 +527,15 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx nOlds = 3; } for (int i = 0; i < nOlds; i++) { - ASSERT(sIdx + i <= nCells); + if (ASSERT(sIdx + i <= nCells)) { + return -1; + } SPgno pgno; if (sIdx + i == nCells) { - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pParent)); + if (ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pParent))) { + return -1; + } pgno = ((SIntHdr *)(pParent->pData))->pgno; } else { pCell = tdbPageGetCell(pParent, sIdx + i); @@ -560,7 +545,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ret = tdbPagerFetchPage(pBt->pPager, &pgno, pOlds + i, tdbBtreeInitPage, &((SBtreeInitPageArg){.pBt = pBt, .flags = 0}), pTxn); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btree-balance: fetch page failed with ret: %d.", ret); return -1; } @@ -679,7 +664,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx szRCell = tdbBtreeCellSize(pPage, pCell, 0, NULL, NULL); } - ASSERT(infoNews[iNew - 1].cnt > 0); + if (ASSERT(infoNews[iNew - 1].cnt > 0)) { + return -1; + } if (infoNews[iNew].size + szRCell >= infoNews[iNew - 1].size - szRCell) { break; @@ -722,7 +709,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx iarg.flags = flags; ret = tdbPagerFetchPage(pBt->pPager, &pgno, pNews + iNew, tdbBtreeInitPage, &iarg, pTxn); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btree-balance: fetch page failed with ret: %d.", ret); + return -1; } ret = tdbPagerWrite(pBt->pPager, pNews[iNew]); @@ -766,8 +754,12 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx pCell = tdbPageGetCell(pPage, oIdx); szCell = tdbBtreeCellSize(pPage, pCell, 0, NULL, NULL); - ASSERT(nNewCells <= infoNews[iNew].cnt); - ASSERT(iNew < nNews); + if (ASSERT(nNewCells <= infoNews[iNew].cnt)) { + return -1; + } + if (ASSERT(iNew < nNews)) { + return -1; + } if (nNewCells < infoNews[iNew].cnt) { tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); @@ -806,14 +798,20 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } } } else { - ASSERT(childNotLeaf); - ASSERT(iNew < nNews - 1); + if (ASSERT(childNotLeaf)) { + return -1; + } + if (ASSERT(iNew < nNews - 1)) { + return -1; + } // set current new page right-most child ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; // insert to parent as divider cell - ASSERT(iNew < nNews - 1); + if (ASSERT(iNew < nNews - 1)) { + return -1; + } ((SPgno *)pCell)[0] = TDB_PAGE_PGNO(pNews[iNew]); tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); @@ -828,7 +826,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } if (childNotLeaf) { - ASSERT(TDB_PAGE_TOTAL_CELLS(pNews[nNews - 1]) == infoNews[nNews - 1].cnt); + if (ASSERT(TDB_PAGE_TOTAL_CELLS(pNews[nNews - 1]) == infoNews[nNews - 1].cnt)) { + return -1; + } ((SIntHdr *)(pNews[nNews - 1]->pData))->pgno = rPgno; SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; @@ -1018,7 +1018,9 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const nLeft -= kLen; // pack partial val to local if any space left if (nLocal > nHeader + kLen + sizeof(SPgno)) { - ASSERT(pVal != NULL && vLen != 0); + if (ASSERT(pVal != NULL && vLen != 0)) { + return -1; + } memcpy(pCell + nHeader + kLen, pVal, nLocal - nHeader - kLen - sizeof(SPgno)); nLeft -= nLocal - nHeader - kLen - sizeof(SPgno); } @@ -1063,11 +1065,11 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const } else { int nLeftKey = kLen; // pack partial key and nextPgno - memcpy(pCell + nHeader, pKey, nLocal - 4); - nLeft -= nLocal - 4; - nLeftKey -= nLocal - 4; + memcpy(pCell + nHeader, pKey, nLocal - nHeader - sizeof(pgno)); + nLeft -= nLocal - nHeader - sizeof(pgno); + nLeftKey -= nLocal - nHeader - sizeof(pgno); - memcpy(pCell + nHeader + nLocal - 4, &pgno, sizeof(pgno)); + memcpy(pCell + nLocal - sizeof(pgno), &pgno, sizeof(pgno)); int lastKeyPageSpace = 0; // pack left key & val to ovpages @@ -1087,9 +1089,12 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const if (lastKeyPage) { if (lastKeyPageSpace >= vLen) { - memcpy(pBuf + kLen - nLeftKey, pVal, vLen); + if (vLen > 0) { + memcpy(pBuf + kLen - nLeftKey, pVal, vLen); + + nLeft -= vLen; + } - nLeft -= vLen; pgno = 0; } else { memcpy(pBuf + kLen - nLeftKey, pVal, lastKeyPageSpace); @@ -1111,7 +1116,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const } } - memcpy(pBuf + kLen - nLeft, &pgno, sizeof(pgno)); + memcpy(pBuf + bytes, &pgno, sizeof(pgno)); ret = tdbPageInsertCell(ofp, 0, pBuf, bytes + sizeof(pgno), 0); if (ret < 0) { @@ -1180,9 +1185,15 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo int nPayload; int ret; - ASSERT(pPage->kLen == TDB_VARIANT_LEN || pPage->kLen == kLen); - ASSERT(pPage->vLen == TDB_VARIANT_LEN || pPage->vLen == vLen); - ASSERT(pKey != NULL && kLen > 0); + if (ASSERT(pPage->kLen == TDB_VARIANT_LEN || pPage->kLen == kLen)) { + return -1; + } + if (ASSERT(pPage->vLen == TDB_VARIANT_LEN || pPage->vLen == vLen)) { + return -1; + } + if (ASSERT(pKey != NULL && kLen > 0)) { + return -1; + } nPayload = 0; nHeader = 0; @@ -1191,7 +1202,10 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo // 1. Encode Header part /* Encode SPgno if interior page */ if (!leaf) { - ASSERT(pPage->vLen == sizeof(SPgno)); + if (pPage->vLen != sizeof(SPgno)) { + tdbError("tdb/btree-encode-cell: invalid cell."); + return -1; + } ((SPgno *)(pCell + nHeader))[0] = ((SPgno *)pVal)[0]; nHeader = nHeader + sizeof(SPgno); @@ -1216,8 +1230,8 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo ret = tdbBtreeEncodePayload(pPage, pCell, nHeader, pKey, kLen, pVal, vLen, &nPayload, pTxn, pBt); if (ret < 0) { // TODO - ASSERT(0); - return 0; + tdbError("tdb/btree-encode-cell: encode payload failed with ret: %d.", ret); + return -1; } *szCell = nHeader + nPayload; @@ -1234,7 +1248,10 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, int vLen = pDecoder->vLen; if (pDecoder->pVal) { - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pPage)); + if (TDB_BTREE_PAGE_IS_LEAF(pPage)) { + tdbError("tdb/btree-decode-payload: leaf page with non-null pVal."); + return -1; + } nPayload = pDecoder->kLen; } else { nPayload = pDecoder->kLen + pDecoder->vLen; @@ -1313,11 +1330,11 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, } TDB_CELLDECODER_SET_FREE_KEY(pDecoder); - memcpy(pDecoder->pKey, pCell + nHeader, nLocal - 4); - nLeft -= nLocal - 4; - nLeftKey -= nLocal - 4; + memcpy(pDecoder->pKey, pCell + nHeader, nLocal - nHeader - sizeof(pgno)); + nLeft -= nLocal - nHeader - sizeof(pgno); + nLeftKey -= nLocal - nHeader - sizeof(pgno); - memcpy(&pgno, pCell + nHeader + nLocal - 4, sizeof(pgno)); + memcpy(&pgno, pCell + nLocal - sizeof(pgno), sizeof(pgno)); int lastKeyPageSpace = 0; // load left key & val to ovpages @@ -1343,9 +1360,11 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, if (lastKeyPage) { if (lastKeyPageSpace >= vLen) { - pDecoder->pVal = ofpCell + kLen - nLeftKey; + if (vLen > 0) { + pDecoder->pVal = ofpCell + kLen - nLeftKey; - nLeft -= vLen; + nLeft -= vLen; + } pgno = 0; } else { // read partial val to local @@ -1435,7 +1454,10 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD // 1. Decode header part if (!leaf) { - ASSERT(pPage->vLen == sizeof(SPgno)); + if (pPage->vLen != sizeof(SPgno)) { + tdbError("tdb/btree-decode-cell: invalid cell."); + return -1; + } pDecoder->pgno = ((SPgno *)(pCell + nHeader))[0]; pDecoder->pVal = (u8 *)(&(pDecoder->pgno)); @@ -1449,7 +1471,10 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD } if (pPage->vLen == TDB_VARIANT_LEN) { - ASSERT(leaf); + if (!leaf) { + tdbError("tdb/btree-decode-cell: not a leaf page."); + return -1; + } nHeader += tdbGetVarInt(pCell + nHeader, &(pDecoder->vLen)); } else { pDecoder->vLen = pPage->vLen; @@ -1481,7 +1506,10 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN * } if (pPage->vLen == TDB_VARIANT_LEN) { - ASSERT(leaf); + if (!leaf) { + tdbError("tdb/btree-cell-size: not a leaf page."); + return -1; + } nHeader += tdbGetVarInt(pCell + nHeader, &vLen); } else if (leaf) { vLen = pPage->vLen; @@ -1577,29 +1605,42 @@ int tdbBtcMoveToFirst(SBTC *pBtc) { ret = tdbPagerFetchPage(pPager, &pBt->root, &(pBtc->pPage), tdbBtreeInitPage, &((SBtreeInitPageArg){.pBt = pBt, .flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF}), pBtc->pTxn); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-tofirst: fetch page failed with ret: %d.", ret); return -1; } - ASSERT(TDB_BTREE_PAGE_IS_ROOT(pBtc->pPage)); + if (!TDB_BTREE_PAGE_IS_ROOT(pBtc->pPage)) { + tdbError("tdb/btc-move-tofirst: not a root page"); + return -1; + } pBtc->iPage = 0; if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) > 0) { pBtc->idx = 0; } else { // no any data, point to an invalid position - ASSERT(TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + if (!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + tdbError("tdb/btc-move-to-first: not a leaf page."); + return -1; + } + pBtc->idx = -1; return 0; } } else { - ASSERT(0); + // TODO + tdbError("tdb/btc-move-to-first: move from a dirty cursor."); + return -1; #if 0 // move from a position int iPage = 0; for (; iPage < pBtc->iPage; iPage++) { - ASSERT(pBtc->idxStack[iPage] >= 0); + if (pBtc->idxStack[iPage] < 0) { + tdbError("tdb/btc-move-to-first: invalid idx: %d.", pBtc->idxStack[iPage]); + return -1; + } + if (pBtc->idxStack[iPage]) break; } @@ -1621,7 +1662,7 @@ int tdbBtcMoveToFirst(SBTC *pBtc) { ret = tdbBtcMoveDownward(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-tofirst: btc move downward failed with ret: %d.", ret); return -1; } @@ -1646,7 +1687,7 @@ int tdbBtcMoveToLast(SBTC *pBtc) { ret = tdbPagerFetchPage(pPager, &pBt->root, &(pBtc->pPage), tdbBtreeInitPage, &((SBtreeInitPageArg){.pBt = pBt, .flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF}), pBtc->pTxn); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-tolast: fetch page failed with ret: %d.", ret); return -1; } @@ -1656,18 +1697,28 @@ int tdbBtcMoveToLast(SBTC *pBtc) { pBtc->idx = TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage) ? nCells - 1 : nCells; } else { // no data at all, point to an invalid position - ASSERT(TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + if (!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + tdbError("tdb/btc-move-to-last: not a leaf page."); + return -1; + } + pBtc->idx = -1; return 0; } } else { - ASSERT(0); + // TODO + tdbError("tdb/btc-move-to-last: move from a dirty cursor."); + return -1; #if 0 int iPage = 0; // downward search for (; iPage < pBtc->iPage; iPage++) { - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pBtc->pgStack[iPage])); + if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pgStack[iPage])) { + tdbError("tdb/btc-move-to-last: leaf page in cursor stack."); + return -1; + } + nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pgStack[iPage]); if (pBtc->idxStack[iPage] != nCells) break; } @@ -1694,7 +1745,7 @@ int tdbBtcMoveToLast(SBTC *pBtc) { ret = tdbBtcMoveDownward(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-tolast: btc move downward failed with ret: %d.", ret); return -1; } @@ -1742,6 +1793,10 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { } memcpy(pVal, cd.pVal, cd.vLen); + if (TDB_CELLDECODER_FREE_VAL(&cd)) { + tdbTrace("tdb/btree-next decoder: %p pVal free: %p", &cd, cd.pVal); + tdbFree(cd.pVal); + } } else { pVal = NULL; } @@ -1752,7 +1807,7 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { ret = tdbBtcMoveToNext(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btree-next: btc move to next failed with ret: %d.", ret); return -1; } @@ -1798,7 +1853,7 @@ int tdbBtreePrev(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { ret = tdbBtcMoveToPrev(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btree-prev: btc move to prev failed with ret: %d.", ret); return -1; } @@ -1810,7 +1865,10 @@ int tdbBtcMoveToNext(SBTC *pBtc) { int ret; SCell *pCell; - ASSERT(TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + if (!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + tdbError("tdb/btc-move-to-next: not a leaf page."); + return -1; + } if (pBtc->idx < 0) return -1; @@ -1829,7 +1887,10 @@ int tdbBtcMoveToNext(SBTC *pBtc) { tdbBtcMoveUpward(pBtc); pBtc->idx++; - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + tdbError("tdb/btree-decode-cell: should not be a leaf page here."); + return -1; + } if (pBtc->idx <= TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { break; } @@ -1841,7 +1902,7 @@ int tdbBtcMoveToNext(SBTC *pBtc) { ret = tdbBtcMoveDownward(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-tonext: btc move downward failed with ret: %d.", ret); return -1; } @@ -1893,8 +1954,15 @@ static int tdbBtcMoveDownward(SBTC *pBtc) { SPgno pgno; SCell *pCell; - ASSERT(pBtc->idx >= 0); - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + if (pBtc->idx < 0) { + tdbError("tdb/btc-move-downward: invalid idx: %d.", pBtc->idx); + return -1; + } + + if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + tdbError("tdb/btc-move-downward: should not be a leaf page here."); + return -1; + } if (pBtc->idx < TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); @@ -1903,7 +1971,10 @@ static int tdbBtcMoveDownward(SBTC *pBtc) { pgno = ((SIntHdr *)pBtc->pPage->pData)->pgno; } - ASSERT(pgno); + if (!pgno) { + tdbError("tdb/btc-move-downward: invalid pgno."); + return -1; + } pBtc->pgStack[pBtc->iPage] = pBtc->pPage; pBtc->idxStack[pBtc->iPage] = pBtc->idx; @@ -1914,7 +1985,7 @@ static int tdbBtcMoveDownward(SBTC *pBtc) { ret = tdbPagerFetchPage(pBtc->pBt->pPager, &pgno, &pBtc->pPage, tdbBtreeInitPage, &((SBtreeInitPageArg){.pBt = pBtc->pBt, .flags = 0}), pBtc->pTxn); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-downward: fetch page failed with ret: %d.", ret); return -1; } @@ -1969,7 +2040,10 @@ int tdbBtcDelete(SBTC *pBtc) { int nKey; int ret; - ASSERT(idx >= 0 && idx < nCells); + if (idx < 0 || idx >= nCells) { + tdbError("tdb/btc-delete: idx: %d out of range[%d, %d).", idx, 0, nCells); + return -1; + } // drop the cell on the leaf ret = tdbPagerWrite(pPager, pBtc->pPage); @@ -2007,7 +2081,7 @@ int tdbBtcDelete(SBTC *pBtc) { ret = tdbPageUpdateCell(pPage, idx, pCell, szCell, pBtc->pTxn, pBtc->pBt); if (ret < 0) { tdbOsFree(pCell); - ASSERT(0); + tdbError("tdb/btc-delete: page update cell failed with ret: %d.", ret); return -1; } tdbOsFree(pCell); @@ -2018,11 +2092,14 @@ int tdbBtcDelete(SBTC *pBtc) { } } else { // delete the leaf page and do balance - ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0); + if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) != 0) { + tdbError("tdb/btc-delete: page to be deleted should be empty."); + return -1; + } ret = tdbBtreeBalance(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-delete: btree balance failed with ret: %d.", ret); return -1; } } @@ -2039,13 +2116,16 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int void *pBuf; int ret; - ASSERT(pBtc->idx >= 0); + if (pBtc->idx < 0) { + tdbError("tdb/btc-upsert: invalid idx: %d.", pBtc->idx); + return -1; + } // alloc space szBuf = kLen + nData + 14; pBuf = tdbRealloc(pBtc->pBt->pBuf, pBtc->pBt->pageSize > szBuf ? szBuf : pBtc->pBt->pageSize); if (pBuf == NULL) { - ASSERT(0); + tdbError("tdb/btc-upsert: realloc pBuf failed."); return -1; } pBtc->pBt->pBuf = pBuf; @@ -2054,7 +2134,7 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int // encode cell ret = tdbBtreeEncodeCell(pBtc->pPage, pKey, kLen, pData, nData, pCell, &szCell, pBtc->pTxn, pBtc->pBt); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-upsert: btree encode cell failed with ret: %d.", ret); return -1; } @@ -2067,16 +2147,22 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int // insert or update if (insert) { - ASSERT(pBtc->idx <= nCells); + if (pBtc->idx > nCells) { + tdbError("tdb/btc-upsert: invalid idx: %d, nCells: %d.", pBtc->idx, nCells); + return -1; + } ret = tdbPageInsertCell(pBtc->pPage, pBtc->idx, pCell, szCell, 0); } else { - ASSERT(pBtc->idx < nCells); + if (pBtc->idx >= nCells) { + tdbError("tdb/btc-upsert: invalid idx: %d, nCells: %d.", pBtc->idx, nCells); + return -1; + } ret = tdbPageUpdateCell(pBtc->pPage, pBtc->idx, pCell, szCell, pBtc->pTxn, pBtc->pBt); } if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-upsert: page insert/update cell failed with ret: %d.", ret); return -1; } @@ -2084,7 +2170,7 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int if (pBtc->pPage->nOverflow > 0) { ret = tdbBtreeBalance(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-upsert: btree balance failed with ret: %d.", ret); return -1; } } @@ -2109,8 +2195,8 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { &((SBtreeInitPageArg){.pBt = pBt, .flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF}), pBtc->pTxn); if (ret < 0) { // TODO - ASSERT(0); - return 0; + tdbError("tdb/btc-move-to: fetch page failed with ret: %d.", ret); + return -1; } pBtc->iPage = 0; @@ -2118,7 +2204,9 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { // for empty tree, just return with an invalid position if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0) return 0; } else { - ASSERT(0); + // TODO + tdbError("tdb/btc-move-to: move from a dirty cursor."); + return -1; #if 0 SPage *pPage; int idx; @@ -2130,7 +2218,10 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { idx = pBtc->idxStack[iPage]; nCells = TDB_PAGE_TOTAL_CELLS(pPage); - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pPage)); + if (TDB_BTREE_PAGE_IS_LEAF(pPage)) { + tdbError("tdb/btc-move-to: leaf page in cursor stack."); + return -1; + } // check if key <= current position if (idx < nCells) { @@ -2233,7 +2324,10 @@ int tdbBtcClose(SBTC *pBtc) { if (pBtc->iPage < 0) return 0; for (;;) { - ASSERT(pBtc->pPage); + if (NULL == pBtc->pPage) { + tdbError("tdb/btc-close: null ptr pPage."); + return -1; + } tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage, pBtc->pTxn); diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index c79279c658a746e83a40eaed6fd6cb8cbe6e31d7..952c49db73a2c8986d67c8e327175ad00e778ae6 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -247,7 +247,10 @@ void tdbEnvRemovePager(TDB *pDb, SPager *pPager) { // remove from the list for (ppPager = &pDb->pgrList; *ppPager && (*ppPager != pPager); ppPager = &((*ppPager)->pNext)) { } - ASSERT(*ppPager == pPager); + if (*ppPager != pPager) { + tdbError("tdb/db: invalid pPager: %p, *ppPager: %p", pPager, *ppPager); + return; + } *ppPager = pPager->pNext; // remove from hash @@ -255,7 +258,10 @@ void tdbEnvRemovePager(TDB *pDb, SPager *pPager) { ppPager = &pDb->pgrHash[hash % pDb->nPgrHash]; for (; *ppPager && *ppPager != pPager; ppPager = &((*ppPager)->pHashNext)) { } - ASSERT(*ppPager == pPager); + if (*ppPager != pPager) { + tdbError("tdb/db: invalid pPager: %p, *ppPager: %p", pPager, *ppPager); + return; + } *ppPager = pPager->pNext; // decrease the counter diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 4896568c7f5ce45ce816f16d3eec0436cf51dfc5..262f3d27e648cfebb4f081c725f443756cb6a06d 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -236,10 +236,10 @@ void tdbPCacheInvalidatePage(SPCache *pCache, SPager *pPager, SPgno pgno) { void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) { i32 nRef; - ASSERT(pTxn); - - // nRef = tdbUnrefPage(pPage); - // ASSERT(nRef >= 0); + if (!pTxn) { + tdbError("tdb/pcache: null ptr pTxn, release failed."); + return; + } tdbPCacheLock(pCache); nRef = tdbUnrefPage(pPage); @@ -275,7 +275,10 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) SPage *pPage = NULL; SPage *pPageH = NULL; - ASSERT(pTxn); + if (!pTxn) { + tdbError("tdb/pcache: null ptr pTxn, fetch impl failed."); + return NULL; + } // 1. Search the hash table pPage = pCache->pgHash[tdbPCachePageHash(pPgid) % pCache->nHash]; @@ -315,8 +318,8 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) if (!pPage && pTxn->xMalloc != NULL) { ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg); if (ret < 0 || pPage == NULL) { - // TODO - ASSERT(0); + tdbError("tdb/pcache: ret: %" PRId32 " pPage: %p, page create failed.", ret, pPage); + // TODO: recycle other backup pages return NULL; } @@ -370,7 +373,11 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) static void tdbPCachePinPage(SPCache *pCache, SPage *pPage) { if (pPage->pLruNext != NULL) { - ASSERT(tdbGetPageRef(pPage) == 0); + int32_t nRef = tdbGetPageRef(pPage); + if (nRef != 0) { + tdbError("tdb/pcache: pin page's ref not zero: %" PRId32, nRef); + return; + } pPage->pLruPrev->pLruNext = pPage->pLruNext; pPage->pLruNext->pLruPrev = pPage->pLruPrev; @@ -383,13 +390,23 @@ static void tdbPCachePinPage(SPCache *pCache, SPage *pPage) { } static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) { - i32 nRef; - - ASSERT(pPage->isLocal); - ASSERT(!pPage->isDirty); - ASSERT(tdbGetPageRef(pPage) == 0); - - ASSERT(pPage->pLruNext == NULL); + i32 nRef = tdbGetPageRef(pPage); + if (nRef != 0) { + tdbError("tdb/pcache: unpin page's ref not zero: %" PRId32, nRef); + return; + } + if (!pPage->isLocal) { + tdbError("tdb/pcache: unpin page's not local: %" PRIu8, pPage->isLocal); + return; + } + if (pPage->isDirty) { + tdbError("tdb/pcache: unpin page's dirty: %" PRIu8, pPage->isDirty); + return; + } + if (NULL != pPage->pLruNext) { + tdbError("tdb/pcache: unpin page's pLruNext not null."); + return; + } tdbTrace("pCache:%p unpin page %p/%d, nPages:%d, pgno:%d, ", pCache, pPage, pPage->id, pCache->nPages, TDB_PAGE_PGNO(pPage)); diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index 50dc8e0a6506abf40cd85b33acdd0e50f737168b..f19c3f28c05e4ec586328320228c59966ad1e3ba 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -43,9 +43,15 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) u8 *ptr; int size; - ASSERT(xMalloc); + if (!xMalloc) { + tdbError("tdb/page-create: null xMalloc."); + return -1; + } - ASSERT(TDB_IS_PGSIZE_VLD(pageSize)); + if (!TDB_IS_PGSIZE_VLD(pageSize)) { + tdbError("tdb/page-create: invalid pageSize: %d.", pageSize); + return -1; + } *ppPage = NULL; size = pageSize + sizeof(*pPage); @@ -69,16 +75,24 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) *ppPage = pPage; - tdbTrace("page/create: %p/%d %p", pPage, pPage->id, xMalloc); + tdbTrace("tdb/page-create: %p/%d %p", pPage, pPage->id, xMalloc); return 0; } int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) { u8 *ptr; - tdbTrace("page/destroy: %p/%d %p", pPage, pPage->id, xFree); - ASSERT(!pPage->isDirty); - ASSERT(xFree); + tdbTrace("tdb/page-destroy: %p/%d %p", pPage, pPage->id, xFree); + + if (pPage->isDirty) { + tdbError("tdb/page-destroy: dirty page: %" PRIu8 ".", pPage->isDirty); + return -1; + } + + if (!xFree) { + tdbError("tdb/page-destroy: null xFree."); + return -1; + } for (int iOvfl = 0; iOvfl < pPage->nOverflow; iOvfl++) { tdbTrace("tdbPage/destroy/free ovfl cell: %p/%p", pPage->apOvfl[iOvfl], pPage); @@ -105,7 +119,10 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell pPage->nOverflow = 0; pPage->xCellSize = xCellSize; - ASSERT((u8 *)pPage->pPageFtr == pPage->pFreeEnd); + if ((u8 *)pPage->pPageFtr != pPage->pFreeEnd) { + tdbError("tdb/page-zero: invalid page, pFreeEnd: %p, pPageFtr: %p", pPage->pFreeEnd, pPage->pPageFtr); + return; + } } void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)) { @@ -121,8 +138,15 @@ void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell pPage->nOverflow = 0; pPage->xCellSize = xCellSize; - ASSERT(pPage->pFreeEnd >= pPage->pFreeStart); - ASSERT(pPage->pFreeEnd - pPage->pFreeStart <= TDB_PAGE_NFREE(pPage)); + if (pPage->pFreeEnd < pPage->pFreeStart) { + tdbError("tdb/page-init: invalid page, pFreeEnd: %p, pFreeStart: %p", pPage->pFreeEnd, pPage->pFreeStart); + return; + } + if (pPage->pFreeEnd - pPage->pFreeStart > TDB_PAGE_NFREE(pPage)) { + tdbError("tdb/page-init: invalid page, pFreeEnd: %p, pFreeStart: %p, NFREE: %d", pPage->pFreeEnd, pPage->pFreeStart, + TDB_PAGE_NFREE(pPage)); + return; + } } int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl) { @@ -132,7 +156,11 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl int lidx; // local idx SCell *pNewCell; - ASSERT(szCell <= TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)); + if (szCell > TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)) { + tdbError("tdb/page-insert-cell: invalid page, szCell: %d, max free: %lu", szCell, + TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)); + return -1; + } nFree = TDB_PAGE_NFREE(pPage); nCells = TDB_PAGE_NCELLS(pPage); @@ -176,7 +204,11 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl TDB_PAGE_CELL_OFFSET_AT_SET(pPage, lidx, pNewCell - pPage->pData); TDB_PAGE_NCELLS_SET(pPage, nCells + 1); - ASSERT(pPage->pFreeStart == pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)); + if (pPage->pFreeStart != pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)) { + tdbError("tdb/page-insert-cell: invalid page, pFreeStart: %p, pCellIdx: %p, nCells: %d", pPage->pFreeStart, + pPage->pCellIdx, nCells); + return -1; + } } for (; iOvfl < pPage->nOverflow; iOvfl++) { @@ -200,7 +232,10 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) { nCells = TDB_PAGE_NCELLS(pPage); - ASSERT(idx >= 0 && idx < nCells + pPage->nOverflow); + if (idx < 0 || idx >= nCells + pPage->nOverflow) { + tdbError("tdb/page-drop-cell: idx: %d out of range, nCells: %d, nOvfl: %d.", idx, nCells, pPage->nOverflow); + return -1; + } iOvfl = 0; for (; iOvfl < pPage->nOverflow; iOvfl++) { @@ -228,7 +263,10 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) { for (; iOvfl < pPage->nOverflow; iOvfl++) { pPage->aiOvfl[iOvfl]--; - ASSERT(pPage->aiOvfl[iOvfl] > 0); + if (pPage->aiOvfl[iOvfl] <= 0) { + tdbError("tdb/page-drop-cell: invalid ai idx: %d", pPage->aiOvfl[iOvfl]); + return -1; + } } return 0; @@ -240,12 +278,19 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { pToPage->pFreeStart = pToPage->pPageHdr + (pFromPage->pFreeStart - pFromPage->pPageHdr); pToPage->pFreeEnd = (u8 *)(pToPage->pPageFtr) - ((u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); - ASSERT(pToPage->pFreeEnd >= pToPage->pFreeStart); + if (pToPage->pFreeEnd < pToPage->pFreeStart) { + tdbError("tdb/page-copy: invalid to page, pFreeStart: %p, pFreeEnd: %p", pToPage->pFreeStart, pToPage->pFreeEnd); + return; + } memcpy(pToPage->pPageHdr, pFromPage->pPageHdr, pFromPage->pFreeStart - pFromPage->pPageHdr); memcpy(pToPage->pFreeEnd, pFromPage->pFreeEnd, (u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); - ASSERT(TDB_PAGE_CCELLS(pToPage) == pToPage->pFreeEnd - pToPage->pData); + if (TDB_PAGE_CCELLS(pToPage) != pToPage->pFreeEnd - pToPage->pData) { + tdbError("tdb/page-copy: invalid to page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pToPage), + pToPage->pFreeEnd - pToPage->pData); + return; + } delta = (pToPage->pPageHdr - pToPage->pData) - (pFromPage->pPageHdr - pFromPage->pData); if (delta != 0) { @@ -295,8 +340,16 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { *ppCell = NULL; nFree = TDB_PAGE_NFREE(pPage); - ASSERT(nFree >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)); - ASSERT(TDB_PAGE_CCELLS(pPage) == pPage->pFreeEnd - pPage->pData); + if (nFree < szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { + tdbError("tdb/page-allocate: invalid cell size, nFree: %d, szCell: %d, szOffset: %d", nFree, szCell, + TDB_PAGE_OFFSET_SIZE(pPage)); + return -1; + } + if (TDB_PAGE_CCELLS(pPage) != pPage->pFreeEnd - pPage->pData) { + tdbError("tdb/page-allocate: invalid page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pPage), + pPage->pFreeEnd - pPage->pData); + return -1; + } // 1. Try to allocate from the free space block area if (pPage->pFreeEnd - pPage->pFreeStart >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { @@ -308,7 +361,10 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { // 2. Try to allocate from the page free list cellFree = TDB_PAGE_FCELL(pPage); - ASSERT(cellFree == 0 || cellFree >= pPage->pFreeEnd - pPage->pData); + if (cellFree != 0 && cellFree < pPage->pFreeEnd - pPage->pData) { + tdbError("tdb/page-allocate: cellFree: %d, pFreeEnd: %p, pData: %p.", cellFree, pPage->pFreeEnd, pPage->pData); + return -1; + } if (cellFree && pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) { SCell *pPrevFreeCell = NULL; int szPrevFreeCell; @@ -353,16 +409,30 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { // 3. Try to dfragment and allocate again tdbPageDefragment(pPage); - ASSERT(pPage->pFreeEnd - pPage->pFreeStart == nFree); - ASSERT(nFree == TDB_PAGE_NFREE(pPage)); - ASSERT(pPage->pFreeEnd - pPage->pData == TDB_PAGE_CCELLS(pPage)); + if (pPage->pFreeEnd - pPage->pFreeStart != nFree) { + tdbError("tdb/page-allocate: nFree: %d, pFreeStart: %p, pFreeEnd: %p.", nFree, pPage->pFreeStart, pPage->pFreeEnd); + return -1; + } + if (TDB_PAGE_NFREE(pPage) != nFree) { + tdbError("tdb/page-allocate: nFree: %d, page free: %d.", nFree, TDB_PAGE_NFREE(pPage)); + return -1; + } + if (pPage->pFreeEnd - pPage->pData != TDB_PAGE_CCELLS(pPage)) { + tdbError("tdb/page-allocate: ccells: %d, pFreeStart: %p, pData: %p.", TDB_PAGE_CCELLS(pPage), pPage->pFreeStart, + pPage->pData); + return -1; + } pPage->pFreeEnd -= szCell; pCell = pPage->pFreeEnd; TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); _alloc_finish: - ASSERT(pCell); + if (NULL == pCell) { + tdbError("tdb/page-allocate: null ptr pCell."); + return -1; + } + pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); TDB_PAGE_NFREE_SET(pPage, nFree - szCell - TDB_PAGE_OFFSET_SIZE(pPage)); *ppCell = pCell; @@ -375,9 +445,18 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) { u8 *dest; u8 *src; - ASSERT(pCell >= pPage->pFreeEnd); - ASSERT(pCell + szCell <= (u8 *)(pPage->pPageFtr)); - ASSERT(pCell == TDB_PAGE_CELL_AT(pPage, idx)); + if (pCell < pPage->pFreeEnd) { + tdbError("tdb/page-free: invalid cell, cell: %p, free end: %p", pCell, pPage->pFreeEnd); + return -1; + } + if (pCell + szCell > (u8 *)(pPage->pPageFtr)) { + tdbError("tdb/page-free: cell crosses page footer, cell: %p, size: %d footer: %p", pCell, szCell, pPage->pFreeEnd); + return -1; + } + if (pCell != TDB_PAGE_CELL_AT(pPage, idx)) { + tdbError("tdb/page-free: cell pos incorrect, cell: %p, pos: %p", pCell, TDB_PAGE_CELL_AT(pPage, idx)); + return -1; + } nFree = TDB_PAGE_NFREE(pPage); @@ -390,7 +469,8 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) { pPage->pPageMethods->setFreeCellInfo(pCell, szCell, cellFree); TDB_PAGE_FCELL_SET(pPage, pCell - pPage->pData); } else { - ASSERT(0); + tdbError("tdb/page-free: invalid cell size: %d", szCell); + return -1; } } @@ -417,7 +497,10 @@ static int tdbPageDefragment(SPage *pPage) { nFree = TDB_PAGE_NFREE(pPage); nCells = TDB_PAGE_NCELLS(pPage); - ASSERT(pPage->pFreeEnd - pPage->pFreeStart < nFree); + if (pPage->pFreeEnd - pPage->pFreeStart >= nFree) { + tdbError("tdb/page-defragment: invalid free range, nFree: %d.", nFree); + return -1; + } // Loop to compact the page content // Here we use an O(n^2) algorithm to do the job since @@ -443,11 +526,19 @@ static int tdbPageDefragment(SPage *pPage) { } } - ASSERT(pCell != NULL); + if (NULL == pCell) { + tdbError("tdb/page-defragment: null ptr pCell."); + return -1; + } szCell = (*pPage->xCellSize)(pPage, pCell, 0, NULL, NULL); - ASSERT(pCell + szCell <= pNextCell); + if (pCell + szCell > pNextCell) { + tdbError("tdb/page-defragment: invalid cell range, pCell: %p, szCell: %d, pNextCell: %p.", pCell, szCell, + pNextCell); + return -1; + } + if (pCell + szCell < pNextCell) { memmove(pNextCell - szCell, pCell, szCell); } @@ -457,7 +548,11 @@ static int tdbPageDefragment(SPage *pPage) { TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pNextCell - pPage->pData); } - ASSERT(pPage->pFreeEnd - pPage->pFreeStart == nFree); + if (pPage->pFreeEnd - pPage->pFreeStart != nFree) { + tdbError("tdb/page-defragment: invalid free range, nFree: %d.", nFree); + return -1; + } + TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); TDB_PAGE_FCELL_SET(pPage, 0); @@ -483,39 +578,59 @@ typedef struct { // cellNum static inline int getPageCellNum(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellNum; } static inline void setPageCellNum(SPage *pPage, int cellNum) { - ASSERT(cellNum < 65536); + if (cellNum >= 65536) { + tdbError("tdb/page-set-cell-num: invalid cellNum: %d.", cellNum); + return; + } ((SPageHdr *)(pPage->pPageHdr))[0].cellNum = (u16)cellNum; } // cellBody static inline int getPageCellBody(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellBody; } static inline void setPageCellBody(SPage *pPage, int cellBody) { - ASSERT(cellBody < 65536); + if (cellBody >= 65536) { + tdbError("tdb/page-set-cell-body: invalid cellBody: %d.", cellBody); + return; + } ((SPageHdr *)(pPage->pPageHdr))[0].cellBody = (u16)cellBody; } // cellFree static inline int getPageCellFree(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellFree; } static inline void setPageCellFree(SPage *pPage, int cellFree) { - ASSERT(cellFree < 65536); + if (cellFree >= 65536) { + tdbError("tdb/page-set-cell-free: invalid cellFree: %d.", cellFree); + return; + } ((SPageHdr *)(pPage->pPageHdr))[0].cellFree = (u16)cellFree; } // nFree static inline int getPageNFree(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].nFree; } static inline void setPageNFree(SPage *pPage, int nFree) { - ASSERT(nFree < 65536); + if (nFree >= 65536) { + tdbError("tdb/page-set-nfree: invalid nFree: %d.", nFree); + return; + } ((SPageHdr *)(pPage->pPageHdr))[0].nFree = (u16)nFree; } // cell offset static inline int getPageCellOffset(SPage *pPage, int idx) { - ASSERT(idx >= 0 && idx < getPageCellNum(pPage)); + int cellNum = getPageCellNum(pPage); + if (idx < 0 || idx >= cellNum) { + tdbError("tdb/page-cell-offset: idx: %d out of range[%d, %d).", idx, 0, cellNum); + return -1; + } + return ((u16 *)pPage->pCellIdx)[idx]; } static inline void setPageCellOffset(SPage *pPage, int idx, int offset) { - ASSERT(offset < 65536); + if (offset >= 65536) { + tdbError("tdb/page-set-cell-offset: invalid offset: %d.", offset); + return; + } ((u16 *)pPage->pCellIdx)[idx] = (u16)offset; } @@ -590,7 +705,12 @@ static inline void setLPageNFree(SPage *pPage, int nFree) { // cell offset static inline int getLPageCellOffset(SPage *pPage, int idx) { - ASSERT(idx >= 0 && idx < getLPageCellNum(pPage)); + int cellNum = getLPageCellNum(pPage); + if (idx < 0 || idx >= cellNum) { + tdbError("tdb/lpage-cell-offset: idx: %d out of range[%d, %d).", idx, 0, cellNum); + return -1; + } + return TDB_GET_U24(pPage->pCellIdx + 3 * idx); } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 8d9933b160bd9b31ee97031e672e092bcfab5ecc..ced867e938fc66574e70774e0e99000be2bf2fa5 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -14,7 +14,7 @@ */ #include "tdbInt.h" - +/* #pragma pack(push, 1) typedef struct { u8 hdrString[16]; @@ -26,7 +26,7 @@ typedef struct { #pragma pack(pop) TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct"); - +*/ struct hashset_st { size_t nbits; size_t mask; @@ -234,7 +234,6 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { int ret; SPage **ppPage; - // ASSERT(pPager->inTran); if (pPage->isDirty) return 0; // ref page one more time so the page will not be release @@ -243,23 +242,8 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { // Set page as dirty pPage->isDirty = 1; - /* - // Add page to dirty list(TODO: NOT use O(n^2) algorithm) - for (ppPage = &pPager->pDirty; (*ppPage) && TDB_PAGE_PGNO(*ppPage) < TDB_PAGE_PGNO(pPage); - ppPage = &((*ppPage)->pDirtyNext)) { - } - - if (*ppPage && TDB_PAGE_PGNO(*ppPage) == TDB_PAGE_PGNO(pPage)) { - tdbUnrefPage(pPage); - - return 0; - } - ASSERT(*ppPage == NULL || TDB_PAGE_PGNO(*ppPage) > TDB_PAGE_PGNO(pPage)); - pPage->pDirtyNext = *ppPage; - *ppPage = pPage; - */ - tdbTrace("put page: %p %d to dirty tree: %p", pPage, TDB_PAGE_PGNO(pPage), &pPager->rbt); + tdbTrace("tdb/pager-write: put page: %p %d to dirty tree: %p", pPage, TDB_PAGE_PGNO(pPage), &pPager->rbt); tRBTreePut(&pPager->rbt, (SRBTreeNode *)pPage); // Write page to journal if neccessary @@ -327,7 +311,11 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { while ((pNode = tRBTreeIterNext(&iter)) != NULL) { pPage = (SPage *)pNode; - ASSERT(pPage->nOverflow == 0); + if (pPage->nOverflow != 0) { + tdbError("tdb/pager-commit: %p, pPage: %p, ovfl: %d, commit page failed.", pPager, pPage, pPage->nOverflow); + return -1; + } + ret = tdbPagerPWritePageToDB(pPager, pPage); if (ret < 0) { tdbError("failed to write page to db since %s", tstrerror(terrno)); @@ -652,12 +640,15 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa loadPage = 0; ret = tdbPagerAllocPage(pPager, &pgno); if (ret < 0) { - ASSERT(0); + tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno); return -1; } } - ASSERT(pgno > 0); + if (pgno == 0) { + tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno); + return -1; + } // fetch a page container memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN); @@ -671,7 +662,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa if (!TDB_PAGE_INITIALIZED(pPage)) { ret = tdbPagerInitPage(pPager, pPage, initPage, arg, loadPage); if (ret < 0) { - ASSERT(0); + tdbError("tdb/pager: %p, pPage: %p, init page failed.", pPager, pPage); return -1; } } @@ -679,8 +670,14 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa // printf("thread %" PRId64 " pager fetch page %d pgno %d ppage %p\n", taosGetSelfPthreadId(), pPage->id, // TDB_PAGE_PGNO(pPage), pPage); - ASSERT(TDB_PAGE_INITIALIZED(pPage)); - ASSERT(pPage->pPager == pPager); + if (!TDB_PAGE_INITIALIZED(pPage)) { + tdbError("tdb/pager: %p, pPage: %p, fetch page uninited.", pPager, pPage); + return -1; + } + if (pPage->pPager != pPager) { + tdbError("tdb/pager: %p/%p, fetch page failed.", pPager, pPage->pPager); + return -1; + } *ppgno = pgno; *ppPage = pPage; @@ -722,8 +719,10 @@ int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno) { return -1; } - ASSERT(*ppgno != 0); - + if (*ppgno == 0) { + tdbError("tdb/pager:%p, alloc new page failed.", pPager); + return -1; + } return 0; } @@ -732,8 +731,8 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage int ret; int lcode; int nLoops; - i64 nRead; - SPgno pgno; + i64 nRead = 0; + SPgno pgno = 0; int init = 0; lcode = TDB_TRY_LOCK_PAGE(pPage); @@ -752,7 +751,6 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage nRead = tdbOsPRead(pPager->fd, pPage->pData, pPage->pageSize, ((i64)pPage->pageSize) * (pgno - 1)); tdbTrace("tdb/pager:%p, pgno:%d, nRead:%" PRId64, pPager, pgno, nRead); if (nRead < pPage->pageSize) { - ASSERT(0); tdbError("tdb/pager:%p, pgno:%d, nRead:%" PRId64 "pgSize:%" PRId32, pPager, pgno, nRead, pPage->pageSize); TDB_UNLOCK_PAGE(pPage); return -1; @@ -763,7 +761,8 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage ret = (*initPage)(pPage, arg, init); if (ret < 0) { - ASSERT(0); + tdbError("tdb/pager:%p, pgno:%d, nRead:%" PRId64 "pgSize:%" PRId32 " init page failed.", pPager, pgno, nRead, + pPage->pageSize); TDB_UNLOCK_PAGE(pPage); return -1; } @@ -782,7 +781,8 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage } } } else { - ASSERT(0); + tdbError("tdb/pager:%p, pgno:%d, nRead:%" PRId64 "pgSize:%" PRId32 " lock page failed.", pPager, pgno, nRead, + pPage->pageSize); return -1; } @@ -869,11 +869,19 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) { return -1; } + if (tdbOsLSeek(jfd, 0L, SEEK_SET) < 0) { + tdbError("failed to lseek jfd due to %s. file:%s, offset:0", strerror(errno), pPager->dbFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + pageBuf = tdbOsCalloc(1, pPager->pageSize); if (pageBuf == NULL) { return -1; } + tdbDebug("pager/restore: %p, %d/%d, txnId:%s", pPager, pPager->dbOrigSize, pPager->dbFileSize, jFileName); + for (int pgIndex = 0; pgIndex < journalSize; ++pgIndex) { // read pgno & the page from journal SPgno pgno; @@ -884,6 +892,8 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) { return -1; } + tdbTrace("pager/restore: restore pgno:%d,", pgno); + ret = tdbOsRead(jfd, pageBuf, pPager->pageSize); if (ret < 0) { tdbOsFree(pageBuf); @@ -923,7 +933,7 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) { return -1; } - if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) { + if (tdbOsRemove(jFileName) < 0 && errno != ENOENT) { tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -943,7 +953,12 @@ int tdbPagerRestoreJournals(SPager *pPager) { while ((pDirEntry = tdbReadDir(pDir)) != NULL) { char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry)); if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) { - if (tdbPagerRestore(pPager, name) < 0) { + char jname[TD_PATH_MAX] = {0}; + int dirLen = strlen(pPager->pEnv->dbName); + memcpy(jname, pPager->pEnv->dbName, dirLen); + jname[dirLen] = '/'; + memcpy(jname + dirLen + 1, name, strlen(name)); + if (tdbPagerRestore(pPager, jname) < 0) { tdbCloseDir(&pDir); tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), name); @@ -969,7 +984,12 @@ int tdbPagerRollback(SPager *pPager) { char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry)); if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) { - if (tdbOsRemove(name) < 0 && errno != ENOENT) { + char jname[TD_PATH_MAX] = {0}; + int dirLen = strlen(pPager->pEnv->dbName); + memcpy(jname, pPager->pEnv->dbName, dirLen); + jname[dirLen] = '/'; + memcpy(jname + dirLen + 1, name, strlen(name)); + if (tdbOsRemove(jname) < 0 && errno != ENOENT) { tdbCloseDir(&pDir); tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), name); diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index 295016997945a0e22d1b135914a035e47da2cc1d..18d14fa474f51104442cf689bd52649598b9c396 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -105,16 +105,14 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF #endif - ASSERT(pPager != NULL); - if (rollback) { - tdbPagerRollback(pPager); - } else { ret = tdbPagerRestoreJournals(pPager); if (ret < 0) { tdbOsFree(pTb); return -1; } + } else { + tdbPagerRollback(pPager); } // pTb->pBt diff --git a/source/libs/tdb/src/db/tdbTxn.c b/source/libs/tdb/src/db/tdbTxn.c index 055d9c7f984b05003a7a69d996e1eb54a51bbe8c..0aeed3c140f8317c6abcec16a9fd049f6be89e1a 100644 --- a/source/libs/tdb/src/db/tdbTxn.c +++ b/source/libs/tdb/src/db/tdbTxn.c @@ -18,7 +18,10 @@ int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg, int flags) { // not support read-committed version at the moment - ASSERT(flags == 0 || flags == (TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)); + if (flags != 0 && flags != (TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)) { + tdbError("tdb/txn: invalid txn flags: %" PRId32, flags); + return -1; + } pTxn->flags = flags; pTxn->txnId = txnid; @@ -28,13 +31,18 @@ int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void return 0; } -int tdbTxnClose(TXN *pTxn) { +int tdbTxnCloseImpl(TXN *pTxn) { if (pTxn) { if (pTxn->jPageSet) { hashset_destroy(pTxn->jPageSet); pTxn->jPageSet = NULL; } + if (pTxn->jfd) { + tdbOsClose(pTxn->jfd); + ASSERT(pTxn->jfd == NULL); + } + tdbOsFree(pTxn); } diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 943611ee27c6f1d6bfa51dfff211bf723764f88f..9a71fc7f30ffebb5cf864d30faea3b8690438112 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -284,7 +284,9 @@ int32_t tfsMkdir(STfs *pTfs, const char *rname) { } int32_t tfsRmdir(STfs *pTfs, const char *rname) { - ASSERT(rname[0] != 0); + if (rname[0] == 0) { + return 0; + } char aname[TMPNAME_LEN] = "\0"; diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index bf9a6c005103c76a6e8201d4833870527d107f36..5f964f6b1ab0e7145f61b25e2e51a1a1d1c86a9e 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -100,14 +100,7 @@ typedef void* queue[2]; #define TRANS_READ_TIMEOUT 3000 // read timeout (ms) #define TRANS_PACKET_LIMIT 1024 * 1024 * 512 -#define TRANS_MAGIC_NUM 0x5f375a86 - -#define TRANS_NOVALID_PACKET(src) ((src) != TRANS_MAGIC_NUM ? 1 : 0) - -#define TRANS_PACKET_LIMIT 1024 * 1024 * 512 - -#define TRANS_MAGIC_NUM 0x5f375a86 - +#define TRANS_MAGIC_NUM 0x5f375a86 #define TRANS_NOVALID_PACKET(src) ((src) != TRANS_MAGIC_NUM ? 1 : 0) typedef SRpcMsg STransMsg; diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 00854b5ee596dd8af75b3bc6abe5e1f59b5aa1ed..cd508f6fe9e71ef90fc5c156685e95f5972931e3 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -35,6 +35,7 @@ typedef struct SHttpModule { typedef struct SHttpMsg { queue q; char* server; + char* uri; int32_t port; char* cont; int32_t len; @@ -63,26 +64,26 @@ static void httpHandleReq(SHttpMsg* msg); static void httpHandleQuit(SHttpMsg* msg); static int32_t httpSendQuit(); -static int32_t taosSendHttpReportImpl(const char* server, uint16_t port, char* pCont, int32_t contLen, +static int32_t taosSendHttpReportImpl(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag); -static int32_t taosBuildHttpHeader(const char* server, int32_t contLen, char* pHead, int32_t headLen, +static int32_t taosBuildHttpHeader(const char* server, const char* uri, int32_t contLen, char* pHead, int32_t headLen, EHttpCompFlag flag) { if (flag == HTTP_FLAT) { return snprintf(pHead, headLen, - "POST /report HTTP/1.1\n" + "POST %s HTTP/1.1\n" "Host: %s\n" "Content-Type: application/json\n" "Content-Length: %d\n\n", - server, contLen); + uri, server, contLen); } else if (flag == HTTP_GZIP) { return snprintf(pHead, headLen, - "POST /report HTTP/1.1\n" + "POST %s HTTP/1.1\n" "Host: %s\n" "Content-Type: application/json\n" "Content-Encoding: gzip\n" "Content-Length: %d\n\n", - server, contLen); + uri, server, contLen); } else { terrno = TSDB_CODE_INVALID_CFG; return -1; @@ -181,6 +182,7 @@ static void httpDestroyMsg(SHttpMsg* msg) { if (msg == NULL) return; taosMemoryFree(msg->server); + taosMemoryFree(msg->uri); taosMemoryFree(msg->cont); taosMemoryFree(msg); } @@ -293,10 +295,11 @@ int32_t httpSendQuit() { return 0; } -static int32_t taosSendHttpReportImpl(const char* server, uint16_t port, char* pCont, int32_t contLen, +static int32_t taosSendHttpReportImpl(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { SHttpMsg* msg = taosMemoryMalloc(sizeof(SHttpMsg)); msg->server = strdup(server); + msg->uri = strdup(uri); msg->port = port; msg->cont = taosMemoryMalloc(contLen); memcpy(msg->cont, pCont, contLen); @@ -309,12 +312,10 @@ static int32_t taosSendHttpReportImpl(const char* server, uint16_t port, char* p httpDestroyMsg(msg); tError("http-report already released"); return -1; - } else { - msg->http = load; - transAsyncSend(load->asyncPool, &(msg->q)); } - - return 0; + + msg->http = load; + return transAsyncSend(load->asyncPool, &(msg->q)); } static void httpDestroyClientCb(uv_handle_t* handle) { @@ -360,7 +361,7 @@ static void httpHandleReq(SHttpMsg* msg) { int32_t len = 2048; char* header = taosMemoryCalloc(1, len); - int32_t headLen = taosBuildHttpHeader(msg->server, msg->len, header, len, msg->flag); + int32_t headLen = taosBuildHttpHeader(msg->server, msg->uri, msg->len, header, len, msg->flag); if (headLen < 0) { taosMemoryFree(header); goto END; @@ -380,6 +381,7 @@ static void httpHandleReq(SHttpMsg* msg) { cli->port = msg->port; cli->dest = dest; + taosMemoryFree(msg->uri); taosMemoryFree(msg); uv_tcp_init(http->loop, &cli->tcp); @@ -406,9 +408,9 @@ END: httpDestroyMsg(msg); } -int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { +int32_t taosSendHttpReport(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { taosThreadOnce(&transHttpInit, transHttpEnvInit); - return taosSendHttpReportImpl(server, port, pCont, contLen, flag); + return taosSendHttpReportImpl(server, uri, port, pCont, contLen, flag); } static void transHttpEnvInit() { diff --git a/source/libs/transport/src/tmsgcb.c b/source/libs/transport/src/tmsgcb.c index 95bc532994d7581eb538e9ec22d3d93395f16424..eea0d876840a7fe44d6a14ecb6723061b3d2ee75 100644 --- a/source/libs/transport/src/tmsgcb.c +++ b/source/libs/transport/src/tmsgcb.c @@ -24,7 +24,6 @@ static SMsgCb defaultMsgCb; void tmsgSetDefault(const SMsgCb* msgcb) { defaultMsgCb = *msgcb; } int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg) { - ASSERT(msgcb != NULL); int32_t code = (*msgcb->putToQueueFp)(msgcb->mgmt, qtype, pMsg); if (code != 0) { rpcFreeCont(pMsg->pCont); @@ -59,3 +58,7 @@ void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg) { (*defaultMsgCb.registerBrokenLin void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type) { (*defaultMsgCb.releaseHandleFp)(pHandle, type); } void tmsgReportStartup(const char* name, const char* desc) { (*defaultMsgCb.reportStartupFp)(name, desc); } + +void tmsgUpdateDnodeInfo(int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port) { + (*defaultMsgCb.updateDnodeInfoFp)(defaultMsgCb.data, dnodeId, clusterId, fqdn, port); +} diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 55c3c61b05ebb0728858b8151f5fc33425fc7a65..47b1ac5ca7c8a46d435ea7c412b2c7adf25d5081 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -160,21 +160,12 @@ int rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) { int rpcSendResponse(const SRpcMsg* pMsg) { return transSendResponse(pMsg); } -void rpcRefHandle(void* handle, int8_t type) { - assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT); - (*taosRefHandle[type])(handle); -} +void rpcRefHandle(void* handle, int8_t type) { (*taosRefHandle[type])(handle); } -void rpcUnrefHandle(void* handle, int8_t type) { - assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT); - (*taosUnRefHandle[type])(handle); -} +void rpcUnrefHandle(void* handle, int8_t type) { (*taosUnRefHandle[type])(handle); } int rpcRegisterBrokenLinkArg(SRpcMsg* msg) { return transRegisterMsg(msg); } -int rpcReleaseHandle(void* handle, int8_t type) { - assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT); - return (*transReleaseHandle[type])(handle); -} +int rpcReleaseHandle(void* handle, int8_t type) { return (*transReleaseHandle[type])(handle); } int rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) { // later diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index d144a76eb035522272bb13b8db630e0042485519..1a99db5f992d51f5e01ba5120e2099f1d53a307f 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -651,7 +651,6 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { return; } - assert(nread <= 0); if (nread == 0) { // ref http://docs.libuv.org/en/v1.x/stream.html?highlight=uv_read_start#c.uv_read_cb // nread might be 0, which does not indicate an error or EOF. This is equivalent to EAGAIN or EWOULDBLOCK under @@ -801,7 +800,11 @@ static void cliSendCb(uv_write_t* req, int status) { } void cliSend(SCliConn* pConn) { - assert(!transQueueEmpty(&pConn->cliMsgs)); + bool empty = transQueueEmpty(&pConn->cliMsgs); + ASSERTS(empty == false, "trans-cli get invalid msg"); + if (empty == true) { + return; + } SCliMsg* pCliMsg = NULL; CONN_GET_NEXT_SENDMSG(pConn); @@ -933,7 +936,6 @@ void cliConnCb(uv_connect_t* req, int status) { transSockInfo2Str(&sockname, pConn->src); tTrace("%s conn %p connect to server successfully", CONN_GET_INST_LABEL(pConn), pConn); - assert(pConn->stream == req->handle); cliSend(pConn); } @@ -1124,7 +1126,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { int ret = uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, cliConnCb); if (ret != 0) { - tGTrace("%s conn %p failed to connect to %s:%d, reason:%s", pTransInst->label, conn, conn->ip, conn->port, + tGError("%s conn %p failed to connect to %s:%d, reason:%s", pTransInst->label, conn, conn->ip, conn->port, uv_err_name(ret)); uv_timer_stop(conn->timer); @@ -1237,7 +1239,7 @@ bool cliRecvReleaseReq(SCliConn* conn, STransMsgHead* pHead) { for (int i = 0; ahandle == 0 && i < transQueueSize(&conn->cliMsgs); i++) { SCliMsg* cliMsg = transQueueGet(&conn->cliMsgs, i); if (cliMsg->type == Release) { - assert(pMsg == NULL); + ASSERTS(pMsg == NULL, "trans-cli recv invaid release-req"); return true; } } @@ -1665,11 +1667,21 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { if (pCtx->retryCode != TSDB_CODE_SUCCESS) { int32_t code = pResp->code; // return internal code app - if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK) { + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || + code == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED) { pResp->code = pCtx->retryCode; } } + // check whole vnodes is offline on this vgroup + if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps || pCtx->retryStep > 0) { + if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + pResp->code = TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED; + } else if (pResp->code == TSDB_CODE_RPC_BROKEN_LINK) { + pResp->code = TSDB_CODE_RPC_SOMENODE_BROKEN_LINK; + } + } + STraceId* trace = &pResp->info.traceId; bool hasEpSet = cliTryExtractEpSet(pResp, &pCtx->epSet); if (hasEpSet) { diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 09f9a78ab88ecce3b95441d9bb0727be6c18e2fd..1161ed7c00408a9559146ba532c74055813f5c5d 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -134,7 +134,9 @@ int transDumpFromBuffer(SConnBuffer* connBuf, char** buf) { if (total >= HEADSIZE && !p->invalid) { *buf = taosMemoryCalloc(1, total); memcpy(*buf, p->buf, total); - transResetBuffer(connBuf); + if (transResetBuffer(connBuf) < 0) { + return -1; + } } else { total = -1; } @@ -154,7 +156,8 @@ int transResetBuffer(SConnBuffer* connBuf) { p->total = 0; p->len = 0; } else { - assert(0); + ASSERTS(0, "invalid read from sock buf"); + return -1; } return 0; } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 2b1f68d5f64952cbd933e3c20c1c507c2f2be966..fa8929f7d954fd42a04f3b8bade2cb084d79c981 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -267,7 +267,10 @@ static bool uvHandleReq(SSvrConn* pConn) { tGTrace("%s handle %p conn:%p translated to app, refId:%" PRIu64, transLabel(pTransInst), transMsg.info.handle, pConn, pConn->refId); - assert(transMsg.info.handle != NULL); + ASSERTS(transMsg.info.handle != NULL, "trans-svr failed to alloc handle to msg"); + if (transMsg.info.handle == NULL) { + return false; + } if (pHead->noResp == 1) { transMsg.info.refId = -1; @@ -718,8 +721,8 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { return; } // free memory allocated by - assert(nread == strlen(notify)); - assert(buf->base[0] == notify[0]); + ASSERTS(nread == strlen(notify), "trans-svr mem corrupted"); + ASSERTS(buf->base[0] == notify[0], "trans-svr mem corrupted"); taosMemoryFree(buf->base); SWorkThrd* pThrd = q->data; @@ -731,7 +734,6 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { } uv_handle_type pending = uv_pipe_pending_type(pipe); - assert(pending == UV_TCP); SSvrConn* pConn = createConn(pThrd); @@ -971,19 +973,24 @@ static void uvPipeListenCb(uv_stream_t* handle, int status) { uv_pipe_t* pipe = &(srv->pipe[srv->numOfWorkerReady][0]); int ret = uv_pipe_init(srv->loop, pipe, 1); - assert(ret == 0); + ASSERTS(ret == 0, "trans-svr failed to init pipe"); + if (ret != 0) return; ret = uv_accept((uv_stream_t*)&srv->pipeListen, (uv_stream_t*)pipe); - assert(ret == 0); + ASSERTS(ret == 0, "trans-svr failed to accept pipe msg"); + if (ret != 0) return; ret = uv_is_readable((uv_stream_t*)pipe); - assert(ret == 1); + ASSERTS(ret == 1, "trans-svr pipe status corrupted"); + if (ret != 1) return; ret = uv_is_writable((uv_stream_t*)pipe); - assert(ret == 1); + ASSERTS(ret == 1, "trans-svr pipe status corrupted"); + if (ret != 1) return; ret = uv_is_closing((uv_handle_t*)pipe); - assert(ret == 0); + ASSERTS(ret == 0, "trans-svr pipe status corrupted"); + if (ret != 0) return; srv->numOfWorkerReady++; } @@ -1001,6 +1008,13 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, uv_loop_init(srv->loop); char pipeName[PATH_MAX]; + + if (false == taosValidIpAndPort(srv->ip, srv->port)) { + terrno = TAOS_SYSTEM_ERROR(errno); + tError("invalid ip/port, %d:%d, reason:%s", srv->ip, srv->port, terrstr()); + goto End; + } + #if defined(WINDOWS) || defined(DARWIN) int ret = uv_pipe_init(srv->loop, &srv->pipeListen, 0); if (ret != 0) { @@ -1087,12 +1101,6 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, } #endif - if (false == taosValidIpAndPort(srv->ip, srv->port)) { - terrno = TAOS_SYSTEM_ERROR(errno); - tError("invalid ip/port, %d:%d, reason:%s", srv->ip, srv->port, terrstr()); - goto End; - } - if (false == addHandleToAcceptloop(srv)) { goto End; } @@ -1185,8 +1193,8 @@ void transCloseServer(void* arg) { // impl later SServerObj* srv = arg; - tDebug("send quit msg to accept thread"); if (srv->inited) { + tDebug("send quit msg to accept thread"); uv_async_send(srv->pAcceptAsync); taosThreadJoin(srv->thread, NULL); SRV_RELEASE_UV(srv->loop); @@ -1271,7 +1279,6 @@ int transSendResponse(const STransMsg* msg) { SExHandle* exh = msg->info.handle; int64_t refId = msg->info.refId; ASYNC_CHECK_HANDLE(exh, refId); - assert(refId != 0); STransMsg tmsg = *msg; tmsg.info.refId = refId; diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index da8cf24627a8f672e101856451210ed5faf975b9..4a0570bee34cafb95f2ce389fa48dbde831383c0 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -289,19 +289,10 @@ void walAlignVersions(SWal* pWal) { } pWal->vers.lastVer = pWal->vers.snapshotVer; } - if (pWal->vers.commitVer < pWal->vers.snapshotVer) { - wWarn("vgId:%d, commitVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId, - pWal->vers.commitVer, pWal->vers.snapshotVer); - pWal->vers.commitVer = pWal->vers.snapshotVer; - } - if (pWal->vers.appliedVer < pWal->vers.snapshotVer) { - wWarn("vgId:%d, appliedVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId, - pWal->vers.appliedVer, pWal->vers.snapshotVer); - pWal->vers.appliedVer = pWal->vers.snapshotVer; - } - - pWal->vers.commitVer = TMIN(pWal->vers.lastVer, pWal->vers.commitVer); - pWal->vers.appliedVer = TMIN(pWal->vers.commitVer, pWal->vers.appliedVer); + // reset commitVer and appliedVer + pWal->vers.commitVer = pWal->vers.snapshotVer; + pWal->vers.appliedVer = pWal->vers.snapshotVer; + wInfo("vgId:%d, reset commitVer to %" PRId64, pWal->cfg.vgId, pWal->vers.commitVer); } bool walLogEntriesComplete(const SWal* pWal) { diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index be536dec3e9c232dd9e1288de08ee71cfcabcd93..d068ede52d217a389cce32bc8ff01b5a813bd47c 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -267,54 +267,55 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { return 0; } -static int32_t walFetchBodyNew(SWalReader *pRead) { - SWalCont *pReadHead = &pRead->pHead->head; +static int32_t walFetchBodyNew(SWalReader *pReader) { + SWalCont *pReadHead = &pReader->pHead->head; int64_t ver = pReadHead->version; - wDebug("vgId:%d, wal starts to fetch body, index:%" PRId64, pRead->pWal->cfg.vgId, ver); + wDebug("vgId:%d, wal starts to fetch body, ver:%" PRId64 " ,len:%d", pReader->pWal->cfg.vgId, ver, + pReadHead->bodyLen); - if (pRead->capacity < pReadHead->bodyLen) { - SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen); + if (pReader->capacity < pReadHead->bodyLen) { + SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen); if (ptr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pRead->pHead = ptr; - pReadHead = &pRead->pHead->head; - pRead->capacity = pReadHead->bodyLen; + pReader->pHead = ptr; + pReadHead = &pReader->pHead->head; + pReader->capacity = pReadHead->bodyLen; } - if (pReadHead->bodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, pReadHead->bodyLen)) { + if (pReadHead->bodyLen != taosReadFile(pReader->pLogFile, pReadHead->body, pReadHead->bodyLen)) { if (pReadHead->bodyLen < 0) { terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s", - pRead->pWal->cfg.vgId, pRead->pHead->head.version, ver, tstrerror(terrno)); + pReader->pWal->cfg.vgId, pReader->pHead->head.version, ver, tstrerror(terrno)); } else { wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since file corrupted", - pRead->pWal->cfg.vgId, pRead->pHead->head.version, ver); + pReader->pWal->cfg.vgId, pReader->pHead->head.version, ver); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } - pRead->curInvalid = 1; + pReader->curInvalid = 1; return -1; } if (pReadHead->version != ver) { - wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64, pRead->pWal->cfg.vgId, - pRead->pHead->head.version, ver); - pRead->curInvalid = 1; + wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64, pReader->pWal->cfg.vgId, + pReader->pHead->head.version, ver); + pReader->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } - if (walValidBodyCksum(pRead->pHead) != 0) { - wError("vgId:%d, wal fetch body error:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, ver); - pRead->curInvalid = 1; + if (walValidBodyCksum(pReader->pHead) != 0) { + wError("vgId:%d, wal fetch body error:%" PRId64 ", since body checksum not passed", pReader->pWal->cfg.vgId, ver); + pReader->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } - wDebug("vgId:%d, index:%" PRId64 " is fetched, cursor advance", pRead->pWal->cfg.vgId, ver); - pRead->curVersion = ver + 1; + wDebug("vgId:%d, index:%" PRId64 " is fetched, cursor advance", pReader->pWal->cfg.vgId, ver); + pReader->curVersion = ver + 1; return 0; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 4233c089a4753f2adf3c1ec0418e3262914610df..0562dc10cee9b4aef88c76a2257cb728207b6d14 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -103,7 +103,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { wInfo("vgId:%d, wal rollback for version %" PRId64, pWal->cfg.vgId, ver); int64_t code; char fnameStr[WAL_FILE_LEN]; - if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer || ver <= pWal->vers.snapshotVer) { + if (ver > pWal->vers.lastVer || ver <= pWal->vers.commitVer || ver <= pWal->vers.snapshotVer) { terrno = TSDB_CODE_WAL_INVALID_VER; taosThreadMutexUnlock(&pWal->mutex); return -1; @@ -613,16 +613,13 @@ int32_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in } void walFsync(SWal *pWal, bool forceFsync) { + taosThreadMutexLock(&pWal->mutex); if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { - wTrace("vgId:%d, fileId:%" PRId64 ".idx, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); - if (taosFsyncFile(pWal->pIdxFile) < 0) { - wError("vgId:%d, file:%" PRId64 ".idx, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), - strerror(errno)); - } wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); if (taosFsyncFile(pWal->pLogFile) < 0) { wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), strerror(errno)); } } + taosThreadMutexUnlock(&pWal->mutex); } diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index b7cb20896bb79d5cee6eba6fed74dc96d87173f6..3aac5e97751295be13956283c2e8dae46f4c453e 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -41,9 +41,13 @@ target_link_libraries( ) if(TD_WINDOWS) target_link_libraries( - os PUBLIC ws2_32 iconv msvcregex wcwidth winmm crashdump + os PUBLIC ws2_32 iconv msvcregex wcwidth winmm crashdump dbghelp ) elseif(TD_DARWIN_64) + find_library(CORE_FOUNDATION_FRAMEWORK CoreFoundation) + target_link_libraries(os PUBLIC ${CORE_FOUNDATION_FRAMEWORK}) + find_library(SYSTEM_CONFIGURATION_FRAMEWORK SystemConfiguration) + target_link_libraries(os PUBLIC ${SYSTEM_CONFIGURATION_FRAMEWORK}) target_link_libraries( os PUBLIC dl m iconv ) @@ -55,4 +59,8 @@ endif() IF (JEMALLOC_ENABLED) target_link_libraries(os PUBLIC -ljemalloc) -ENDIF () \ No newline at end of file +ENDIF () + +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index d8cccc83ed7ce0942576873859b27b33cc4621ec..71b3125de8ff42f4a3ea1ef19690c6053a00e707 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -39,14 +39,6 @@ #define _SEND_FILE_STEP_ 1000 #endif -#if defined(WINDOWS) -typedef int32_t FileFd; -typedef int32_t SocketFd; -#else -typedef int32_t FileFd; -typedef int32_t SocketFd; -#endif - typedef int32_t FileFd; typedef struct TdFile { @@ -54,19 +46,10 @@ typedef struct TdFile { int refId; FileFd fd; FILE *fp; -} * TdFilePtr, TdFile; +} TdFile; #define FILE_WITH_LOCK 1 -typedef struct AutoDelFile *AutoDelFilePtr; -typedef struct AutoDelFile { - char *name; - AutoDelFilePtr lastAutoDelFilePtr; -} AutoDelFile; -static TdThreadMutex autoDelFileLock; -static AutoDelFilePtr nowAutoDelFilePtr = NULL; -static TdThreadOnce autoDelFileInit = PTHREAD_ONCE_INIT; - void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) { #ifdef WINDOWS const char *tdengineTmpFileNamePrefix = "tdengine-"; @@ -208,7 +191,7 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { int32_t code = _stati64(path, &fileStat); #else struct stat fileStat; - int32_t code = stat(path, &fileStat); + int32_t code = stat(path, &fileStat); #endif if (code < 0) { return code; @@ -225,10 +208,9 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { return 0; } int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { - if (pFile == NULL) { - return 0; + if (pFile == NULL || pFile->fd < 0) { + return -1; } - assert(pFile->fd >= 0); // Please check if you have closed the file. #ifdef WINDOWS @@ -250,7 +232,7 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { #else struct stat fileStat; - int32_t code = fstat(pFile->fd, &fileStat); + int32_t code = fstat(pFile->fd, &fileStat); if (code < 0) { printf("taosFStatFile run fstat fail."); return code; @@ -268,34 +250,6 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { return 0; } -void autoDelFileList() { - taosThreadMutexLock(&autoDelFileLock); - while (nowAutoDelFilePtr != NULL) { - taosRemoveFile(nowAutoDelFilePtr->name); - AutoDelFilePtr tmp = nowAutoDelFilePtr->lastAutoDelFilePtr; - taosMemoryFree(nowAutoDelFilePtr->name); - taosMemoryFree(nowAutoDelFilePtr); - nowAutoDelFilePtr = tmp; - } - taosThreadMutexUnlock(&autoDelFileLock); - taosThreadMutexDestroy(&autoDelFileLock); -} - -void autoDelFileListInit() { - taosThreadMutexInit(&autoDelFileLock, NULL); - atexit(autoDelFileList); -} - -void autoDelFileListAdd(const char *path) { - taosThreadOnce(&autoDelFileInit, autoDelFileListInit); - taosThreadMutexLock(&autoDelFileLock); - AutoDelFilePtr tmp = taosMemoryMalloc(sizeof(AutoDelFile)); - tmp->lastAutoDelFilePtr = nowAutoDelFilePtr; - tmp->name = taosMemoryStrDup(path); - nowAutoDelFilePtr = tmp; - taosThreadMutexUnlock(&autoDelFileLock); -} - TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { int fd = -1; FILE *fp = NULL; @@ -310,10 +264,12 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { } else { mode = (tdFileOptions & TD_FILE_TEXT) ? "rt+" : "rb+"; } - assert(!(tdFileOptions & TD_FILE_EXCL)); + ASSERT(!(tdFileOptions & TD_FILE_EXCL)); + if (tdFileOptions & TD_FILE_EXCL) { + return NULL; + } fp = fopen(path, mode); if (fp == NULL) { - // terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } } else { @@ -331,32 +287,44 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { access |= (tdFileOptions & TD_FILE_TEXT) ? O_TEXT : 0; access |= (tdFileOptions & TD_FILE_EXCL) ? O_EXCL : 0; #ifdef WINDOWS - fd = _open(path, access, _S_IREAD | _S_IWRITE); + int32_t pmode = _S_IREAD | _S_IWRITE; + if (tdFileOptions & TD_FILE_AUTO_DEL) { + pmode |= _O_TEMPORARY; + } + fd = _open(path, access, pmode); #else fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO); #endif if (fd == -1) { - // terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } } TdFilePtr pFile = (TdFilePtr)taosMemoryMalloc(sizeof(TdFile)); if (pFile == NULL) { - // terrno = TSDB_CODE_OUT_OF_MEMORY; if (fd >= 0) close(fd); if (fp != NULL) fclose(fp); return NULL; } + #if FILE_WITH_LOCK taosThreadRwlockInit(&(pFile->rwlock), NULL); #endif pFile->fd = fd; pFile->fp = fp; pFile->refId = 0; + if (tdFileOptions & TD_FILE_AUTO_DEL) { - autoDelFileListAdd(path); +#ifdef WINDOWS + // do nothing, since the property of pmode is set with _O_TEMPORARY; the OS will recycle + // the file handle, as well as the space on disk. +#else + // Remove it instantly, so when the program exits normally/abnormally, the file + // will be automatically remove by OS. + unlink(path); +#endif } + return pFile; } @@ -398,7 +366,13 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + ASSERT(pFile->fd >= 0); // Please check if you have closed the file. + if (pFile->fd < 0) { +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif + return -1; + } int64_t leftbytes = count; int64_t readbytes; char *tbuf = (char *)buf; @@ -442,7 +416,13 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + ASSERT(pFile->fd >= 0); // Please check if you have closed the file. + if (pFile->fd < 0) { +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif + return -1; + } #ifdef WINDOWS size_t pos = _lseeki64(pFile->fd, 0, SEEK_CUR); _lseeki64(pFile->fd, offset, SEEK_SET); @@ -465,6 +445,9 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { taosThreadRwlockWrlock(&(pFile->rwlock)); #endif if (pFile->fd < 0) { +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif return 0; } @@ -500,7 +483,13 @@ int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t #if FILE_WITH_LOCK taosThreadRwlockWrlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + ASSERT(pFile->fd >= 0); // Please check if you have closed the file. + if (pFile->fd < 0) { +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif + return 0; + } #ifdef WINDOWS size_t pos = _lseeki64(pFile->fd, 0, SEEK_CUR); _lseeki64(pFile->fd, offset, SEEK_SET); @@ -519,7 +508,10 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + ASSERT(pFile->fd >= 0); // Please check if you have closed the file. + if (pFile->fd < 0) { + return -1; + } #ifdef WINDOWS int64_t ret = _lseeki64(pFile->fd, offset, whence); #else @@ -535,7 +527,10 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); // Please check if you have closed the file. + ASSERT(pFile->fd >= 0); // Please check if you have closed the file. + if (pFile->fd < 0) { + return -1; + } struct stat fileStat; #ifdef WINDOWS @@ -559,32 +554,61 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { } int32_t taosLockFile(TdFilePtr pFile) { + ASSERT(pFile->fd >= 0); // Please check if you have closed the file. + if (pFile->fd < 0) { + return -1; + } #ifdef WINDOWS + BOOL fSuccess = FALSE; + LARGE_INTEGER fileSize; + OVERLAPPED overlapped = {0}; + + HANDLE hFile = (HANDLE)_get_osfhandle(pFile->fd); + + fSuccess = LockFileEx(hFile, LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY, + 0, // reserved + ~0, // number of bytes to lock low + ~0, // number of bytes to lock high + &overlapped // overlapped structure + ); + if (!fSuccess) { + return GetLastError(); + } return 0; #else - assert(pFile->fd >= 0); // Please check if you have closed the file. - return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB); #endif } int32_t taosUnLockFile(TdFilePtr pFile) { + ASSERT(pFile->fd >= 0); + if (pFile->fd < 0) { + return 0; + } #ifdef WINDOWS + BOOL fSuccess = FALSE; + OVERLAPPED overlapped = {0}; + HANDLE hFile = (HANDLE)_get_osfhandle(pFile->fd); + + fSuccess = UnlockFileEx(hFile, 0, ~0, ~0, &overlapped); + if (!fSuccess) { + return GetLastError(); + } return 0; #else - assert(pFile->fd >= 0); // Please check if you have closed the file. - return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB); #endif } int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { -#ifdef WINDOWS + if (pFile == NULL) { + return 0; + } if (pFile->fd < 0) { - errno = EBADF; printf("Ftruncate file error, fd arg was negative\n"); return -1; } +#ifdef WINDOWS HANDLE h = (HANDLE)_get_osfhandle(pFile->fd); @@ -629,11 +653,6 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { return 0; #else - if (pFile == NULL) { - return 0; - } - assert(pFile->fd >= 0); // Please check if you have closed the file. - return ftruncate(pFile->fd, l_size); #endif } @@ -661,7 +680,10 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in if (pFileOut == NULL || pFileIn == NULL) { return 0; } - assert(pFileIn->fd >= 0 && pFileOut->fd >= 0); + ASSERT(pFileIn->fd >= 0 && pFileOut->fd >= 0); + if (pFileIn->fd < 0 || pFileOut->fd < 0) { + return 0; + } #ifdef WINDOWS @@ -754,11 +776,9 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in } void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { - if (pFile == NULL) { + if (pFile == NULL || pFile->fp == NULL) { return; } - assert(pFile->fp != NULL); - va_list ap; va_start(ap, format); vfprintf(pFile->fp, format, ap); @@ -783,7 +803,10 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { if (*ptrBuf != NULL) { taosMemoryFreeClear(*ptrBuf); } - assert(pFile->fp != NULL); + ASSERT(pFile->fp != NULL); + if (pFile->fp == NULL) { + return -1; + } #ifdef WINDOWS *ptrBuf = taosMemoryMalloc(1024); if (*ptrBuf == NULL) return -1; @@ -803,7 +826,10 @@ int64_t taosGetsFile(TdFilePtr pFile, int32_t maxSize, char *__restrict buf) { if (pFile == NULL || buf == NULL) { return -1; } - assert(pFile->fp != NULL); + ASSERT(pFile->fp != NULL); + if (pFile->fp == NULL) { + return -1; + } if (fgets(buf, maxSize, pFile->fp) == NULL) { return -1; } @@ -812,9 +838,12 @@ int64_t taosGetsFile(TdFilePtr pFile, int32_t maxSize, char *__restrict buf) { int32_t taosEOFFile(TdFilePtr pFile) { if (pFile == NULL) { - return 0; + return -1; + } + ASSERT(pFile->fp != NULL); + if (pFile->fp == NULL) { + return -1; } - assert(pFile->fp != NULL); return feof(pFile->fp); } diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c index 7319181a777cb8140396f592009507a151d2620b..b4a2845e961aa8a4c8d993304f869f8c66f3e9fb 100644 --- a/source/os/src/osLocale.c +++ b/source/os/src/osLocale.c @@ -71,7 +71,7 @@ char *taosCharsetReplace(char *charsetstr) { * seems does not response as expected. * * In some Linux systems, setLocale(LC_CTYPE, "") may return NULL, in which case the launch of - * both the TDengine Server and the Client may be interrupted. + * both the Server and the Client may be interrupted. * * In case that the setLocale failed to be executed, the right charset needs to be set. */ diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 2e68bd5ccd3ad937dbcfce6988fcb7ef6060f31a..3825b8e074b73cff3b2caec533671405916009e1 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -266,7 +266,10 @@ void *taosMemoryRealloc(void *ptr, int64_t size) { if (ptr == NULL) return taosMemoryMalloc(size); TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char *)ptr - sizeof(TdMemoryInfo)); - assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); + ASSERT(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); + if (tpTdMemoryInfo->symbol != TD_MEMORY_SYMBOL) { ++ return NULL; ++ } TdMemoryInfo tdMemoryInfo; memcpy(&tdMemoryInfo, pTdMemoryInfo, sizeof(TdMemoryInfo)); @@ -288,8 +291,10 @@ void *taosMemoryStrDup(const char *ptr) { if (ptr == NULL) return NULL; TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char *)ptr - sizeof(TdMemoryInfo)); - assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); - + ASSERT(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); + if (pTdMemoryInfo->symbol != TD_MEMORY_SYMBOL) { ++ return NULL; ++ } void *tmp = tstrdup(pTdMemoryInfo); if (tmp == NULL) return NULL; @@ -323,7 +328,10 @@ int64_t taosMemorySize(void *ptr) { #ifdef USE_TD_MEMORY TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char *)ptr - sizeof(TdMemoryInfo)); - assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); + ASSERT(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); + if (pTdMemoryInfo->symbol != TD_MEMORY_SYMBOL) { ++ return NULL; ++ } return pTdMemoryInfo->memorySize; #else @@ -348,7 +356,7 @@ void taosMemoryTrim(int32_t size) { void* taosMemoryMallocAlign(uint32_t alignment, int64_t size) { #ifdef USE_TD_MEMORY - assert(0); + ASSERT(0); #else #if defined(LINUX) void* p = memalign(alignment, size); diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 2f947d325263d45025a7cff835a715ac108674e7..cc018bf842f7e8a6835999211f08bbca2882db34 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -86,8 +86,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t ms) { while ((rc = sem_timedwait(sem, &ts)) == -1 && errno == EINTR) continue; return rc; /* This should have timed out */ - // assert(errno == ETIMEDOUT); - // assert(rc != 0); + // ASSERT(errno == ETIMEDOUT); + // ASSERT(rc != 0); // GetSystemTimeAsFileTime(&ft_after); // // We specified a non-zero wait. Time must advance. // if (ft_before.dwLowDateTime == ft_after.dwLowDateTime && ft_before.dwHighDateTime == ft_after.dwHighDateTime) diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index db2a9937b5861db87b17a21cc0efbd8a16683bf1..715c2632e0ba0cf299d1180038e319571604d81f 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -298,7 +298,7 @@ int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void return -1; } #ifdef WINDOWS - assert(0); + ASSERT(0); return 0; #else return getsockopt(pSocket->fd, level, optname, optval, (int *)optlen); @@ -662,7 +662,7 @@ int32_t taosKeepTcpAlive(TdSocketPtr pSocket) { int taosGetLocalIp(const char *eth, char *ip) { #if defined(WINDOWS) // DO NOTHAING - assert(0); + ASSERT(0); return 0; #else int fd; @@ -689,7 +689,7 @@ int taosGetLocalIp(const char *eth, char *ip) { int taosValidIp(uint32_t ip) { #if defined(WINDOWS) // DO NOTHAING - assert(0); + ASSERT(0); return 0; #else int ret = -1; @@ -924,7 +924,7 @@ uint32_t ip2uint(const char *const ip_addr) { void taosBlockSIGPIPE() { #ifdef WINDOWS - // assert(0); + // ASSERT(0); #else sigset_t signal_mask; sigemptyset(&signal_mask); @@ -988,40 +988,38 @@ int32_t taosGetFqdn(char *fqdn) { #endif char hostname[1024]; hostname[1023] = '\0'; - if (gethostname(hostname, 1023) == -1) { + if (taosGetlocalhostname(hostname, 1023) == -1) { #ifdef WINDOWS printf("failed to get hostname, reason:%s\n", strerror(WSAGetLastError())); #else printf("failed to get hostname, reason:%s\n", strerror(errno)); #endif - assert(0); + ASSERT(0); return -1; } - struct addrinfo hints = {0}; - struct addrinfo *result = NULL; #ifdef __APPLE__ // on macosx, hostname -f has the form of xxx.local // which will block getaddrinfo for a few seconds if AI_CANONNAME is set // thus, we choose AF_INET (ipv4 for the moment) to make getaddrinfo return // immediately - hints.ai_family = AF_INET; + // hints.ai_family = AF_INET; + strcpy(fqdn, hostname); + strcpy(fqdn+strlen(hostname), ".local"); #else // __APPLE__ + struct addrinfo hints = {0}; + struct addrinfo *result = NULL; hints.ai_flags = AI_CANONNAME; -#endif // __APPLE__ + int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { fprintf(stderr, "failed to get fqdn, code:%d, reason:%s\n", ret, gai_strerror(ret)); return -1; } - -#ifdef __APPLE__ - // refer to comments above - strcpy(fqdn, hostname); -#else // __APPLE__ strcpy(fqdn, result->ai_canonname); -#endif // __APPLE__ freeaddrinfo(result); +#endif // __APPLE__ + return 0; } @@ -1033,7 +1031,7 @@ void taosIgnSIGPIPE() { signal(SIGPIPE, SIG_IGN); } void taosSetMaskSIGPIPE() { #ifdef WINDOWS - // assert(0); + // ASSERT(0); #else sigset_t signal_mask; sigemptyset(&signal_mask); diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 5419da1c0ce49fe8e61cbf030ad540918835f465..b9f537ad34a15c89a003390c2173f70b227beb2b 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -112,7 +112,7 @@ int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) { } TdUcs4 *tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) { - assert(taosMemorySize(target_ucs4) >= len_ucs4 * sizeof(TdUcs4)); + ASSERT(taosMemorySize(target_ucs4) >= len_ucs4 * sizeof(TdUcs4)); return memcpy(target_ucs4, source_ucs4, len_ucs4 * sizeof(TdUcs4)); } @@ -217,7 +217,7 @@ void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type) { bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len) { #ifdef DISALLOW_NCHAR_WITHOUT_ICONV - printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n"); + printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n"); return -1; #else memset(ucs4, 0, ucs4_max_len); @@ -245,7 +245,7 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4 int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) { #ifdef DISALLOW_NCHAR_WITHOUT_ICONV - printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n"); + printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n"); return -1; #else @@ -263,7 +263,7 @@ int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) { } bool taosValidateEncodec(const char *encodec) { #ifdef DISALLOW_NCHAR_WITHOUT_ICONV - printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n"); + printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n"); return true; #else iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC); @@ -355,8 +355,8 @@ int64_t taosStr2Int64(const char *str, char **pEnd, int32_t radix) { if (errno == EINVAL) errno = 0; #endif #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); #endif return tmp; } @@ -367,8 +367,8 @@ uint64_t taosStr2UInt64(const char *str, char **pEnd, int32_t radix) { if (errno == EINVAL) errno = 0; #endif #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); #endif return tmp; } @@ -379,8 +379,8 @@ int32_t taosStr2Int32(const char *str, char **pEnd, int32_t radix) { if (errno == EINVAL) errno = 0; #endif #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); #endif return tmp; } @@ -391,8 +391,8 @@ uint32_t taosStr2UInt32(const char *str, char **pEnd, int32_t radix) { if (errno == EINVAL) errno = 0; #endif #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); #endif return tmp; } @@ -403,10 +403,10 @@ int16_t taosStr2Int16(const char *str, char **pEnd, int32_t radix) { if (errno == EINVAL) errno = 0; #endif #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); - assert(tmp >= SHRT_MIN); - assert(tmp <= SHRT_MAX); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); + ASSERT(tmp >= SHRT_MIN); + ASSERT(tmp <= SHRT_MAX); #endif return (int16_t)tmp; } @@ -417,9 +417,9 @@ uint16_t taosStr2UInt16(const char *str, char **pEnd, int32_t radix) { if (errno == EINVAL) errno = 0; #endif #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); - assert(tmp <= USHRT_MAX); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); + ASSERT(tmp <= USHRT_MAX); #endif return (uint16_t)tmp; } @@ -427,10 +427,10 @@ uint16_t taosStr2UInt16(const char *str, char **pEnd, int32_t radix) { int8_t taosStr2Int8(const char *str, char **pEnd, int32_t radix) { int32_t tmp = strtol(str, pEnd, radix); #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); - assert(tmp >= SCHAR_MIN); - assert(tmp <= SCHAR_MAX); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); + ASSERT(tmp >= SCHAR_MIN); + ASSERT(tmp <= SCHAR_MAX); #endif return tmp; } @@ -441,9 +441,9 @@ uint8_t taosStr2UInt8(const char *str, char **pEnd, int32_t radix) { if (errno == EINVAL) errno = 0; #endif #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); - assert(tmp <= UCHAR_MAX); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); + ASSERT(tmp <= UCHAR_MAX); #endif return tmp; } @@ -451,9 +451,9 @@ uint8_t taosStr2UInt8(const char *str, char **pEnd, int32_t radix) { double taosStr2Double(const char *str, char **pEnd) { double tmp = strtod(str, pEnd); #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); - assert(tmp != HUGE_VAL); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); + ASSERT(tmp != HUGE_VAL); #endif return tmp; } @@ -461,10 +461,10 @@ double taosStr2Double(const char *str, char **pEnd) { float taosStr2Float(const char *str, char **pEnd) { float tmp = strtof(str, pEnd); #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); - assert(tmp != HUGE_VALF); - assert(tmp != NAN); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); + ASSERT(tmp != HUGE_VALF); + ASSERT(tmp != NAN); #endif return tmp; } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index e1abe8484188106b0a53238881ef55feb1fd2876..35e76f0e67779189ecec915dee0231b5afe63bf4 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -98,6 +98,9 @@ LONG WINAPI exceptionHandler(LPEXCEPTION_POINTERS exception); #include #include #include +#include +#include +#include #else @@ -246,7 +249,7 @@ void taosGetSystemInfo() { int32_t taosGetEmail(char *email, int32_t maxLen) { #ifdef WINDOWS - // assert(0); + // ASSERT(0); #elif defined(_TD_DARWIN_64) const char *filepath = "/usr/local/taos/email"; @@ -860,7 +863,7 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { char *taosGetCmdlineByPID(int pid) { #ifdef WINDOWS - assert(0); + ASSERT(0); return ""; #elif defined(_TD_DARWIN_64) static char cmdline[1024]; @@ -1007,6 +1010,11 @@ SysNameInfo taosGetSysNameInfo() { tstrncpy(info.machine, uts.machine, sizeof(info.machine)); } + char localHostName[512]; + taosGetlocalhostname(localHostName, 512); + TdCmdPtr pCmd = taosOpenCmd("scutil --get LocalHostName"); + tstrncpy(info.nodename, localHostName, sizeof(info.nodename)); + return info; #else SysNameInfo info = {0}; @@ -1042,3 +1050,46 @@ bool taosCheckCurrentInDll() { return false; #endif } + +#ifdef _TD_DARWIN_64 +int taosGetMaclocalhostnameByCommand(char *hostname, size_t maxLen) { + TdCmdPtr pCmd = taosOpenCmd("scutil --get LocalHostName"); + if (pCmd != NULL) { + if (taosGetsCmd(pCmd, maxLen - 1, hostname) > 0) { + int len = strlen(hostname); + if (hostname[len - 1] == '\n') { + hostname[len - 1] = '\0'; + } + return 0; + } + taosCloseCmd(&pCmd); + } + return -1; +} + +int getMacLocalHostNameBySCD(char *hostname, size_t maxLen) { + SCDynamicStoreRef store = SCDynamicStoreCreate(NULL, CFSTR(""), NULL, NULL); + CFStringRef hostname_cfstr = SCDynamicStoreCopyLocalHostName(store); + if (hostname_cfstr != NULL) { + CFStringGetCString(hostname_cfstr, hostname, maxLen - 1, kCFStringEncodingMacRoman); + CFRelease(hostname_cfstr); + } else { + return -1; + } + CFRelease(store); + return 0; +} +#endif + +int taosGetlocalhostname(char *hostname, size_t maxLen) { +#ifdef _TD_DARWIN_64 + int res = getMacLocalHostNameBySCD(hostname, maxLen); + if (res != 0) { + return taosGetMaclocalhostnameByCommand(hostname, maxLen); + } else { + return 0; + } +#else + return gethostname(hostname, maxLen); +#endif +} diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index c972aebbcaf8e89b7104e9c9241b5f7d11b6449d..7b6c77f7bc7ec969438689068a521ed20c123940 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -90,7 +90,7 @@ typedef struct FILE TdCmd; void* taosLoadDll(const char* filename) { #if defined(WINDOWS) - assert(0); + ASSERT(0); return NULL; #elif defined(_TD_DARWIN_64) return NULL; @@ -109,7 +109,7 @@ void* taosLoadDll(const char* filename) { void* taosLoadSym(void* handle, char* name) { #if defined(WINDOWS) - assert(0); + ASSERT(0); return NULL; #elif defined(_TD_DARWIN_64) return NULL; @@ -130,7 +130,7 @@ void* taosLoadSym(void* handle, char* name) { void taosCloseDll(void* handle) { #if defined(WINDOWS) - assert(0); + ASSERT(0); return; #elif defined(_TD_DARWIN_64) return; diff --git a/source/os/src/osThread.c b/source/os/src/osThread.c index 32c58695cfdac108e38112ccbfc73f35faf5ed42..39ba92fdc5e93d45facd85a2015461ad1fd68d8c 100644 --- a/source/os/src/osThread.c +++ b/source/os/src/osThread.c @@ -233,7 +233,8 @@ int32_t taosThreadSpinDestroy(TdThreadSpinlock *lock) { int32_t taosThreadSpinInit(TdThreadSpinlock *lock, int32_t pshared) { #ifdef TD_USE_SPINLOCK_AS_MUTEX - assert(pshared == 0); + ASSERT(pshared == 0); + if (pshared != 0) return -1; return pthread_mutex_init((pthread_mutex_t *)lock, NULL); #else return pthread_spin_init((pthread_spinlock_t *)lock, pshared); diff --git a/source/os/test/CMakeLists.txt b/source/os/test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..21fb2ee630d2d0058ca48f94a1f39c10ce7abe40 --- /dev/null +++ b/source/os/test/CMakeLists.txt @@ -0,0 +1,24 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) +PROJECT(TDengine) + +FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) +FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib /usr/lib64) +FIND_LIBRARY(LIB_GTEST_SHARED_DIR libgtest.so /usr/lib/ /usr/local/lib /usr/lib64) + +IF (HEADER_GTEST_INCLUDE_DIR AND (LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR)) + MESSAGE(STATUS "gTest library found, build os test") + + INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) + AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ENDIF() + +INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/src/util/inc) + +# osTests +add_executable(osTests "osTests.cpp") +target_link_libraries(osTests os util gtest_main) +add_test( + NAME osTests + COMMAND osTests +) \ No newline at end of file diff --git a/source/os/test/osTests.cpp b/source/os/test/osTests.cpp index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..2e24bb05269ba9a4529b16726ca8c59d6de7ac68 100644 --- a/source/os/test/osTests.cpp +++ b/source/os/test/osTests.cpp @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + +#include "os.h" +#include "tlog.h" + +TEST(osTest, osSystem) { + const char *flags = "UTL FATAL "; + ELogLevel level = DEBUG_FATAL; + int32_t dflag = 255; // tsLogEmbedded ? 255 : uDebugFlag + taosPrintTrace(flags, level, dflag, 0); +} + +void fileOperateOnFree(void *param) { + char * fname = (char *)param; + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE); + printf("On free thread open file\n"); + ASSERT_NE(pFile, nullptr); + + int ret = taosLockFile(pFile); + printf("On free thread lock file ret:%d\n", ret); + ASSERT_EQ(ret, 0); + + ret = taosUnLockFile(pFile); + printf("On free thread unlock file ret:%d\n", ret); + ASSERT_EQ(ret, 0); + + ret = taosCloseFile(&pFile); + ASSERT_EQ(ret, 0); + printf("On free thread close file ret:%d\n", ret); +} +void *fileOperateOnFreeThread(void *param) { + fileOperateOnFree(param); + return NULL; +} +void fileOperateOnBusy(void *param) { + char * fname = (char *)param; + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE); + printf("On busy thread open file\n"); + ASSERT_NE(pFile, nullptr); + + int ret = taosLockFile(pFile); + printf("On busy thread lock file ret:%d\n", ret); + ASSERT_NE(ret, 0); + + ret = taosUnLockFile(pFile); + printf("On busy thread unlock file ret:%d\n", ret); +#ifdef _TD_DARWIN_64 + ASSERT_EQ(ret, 0); +#else + ASSERT_NE(ret, 0); +#endif + + ret = taosCloseFile(&pFile); + printf("On busy thread close file ret:%d\n", ret); + ASSERT_EQ(ret, 0); +} +void *fileOperateOnBusyThread(void *param) { + fileOperateOnBusy(param); + return NULL; +} + +TEST(osTest, osFile) { + char *fname = "./osfiletest1.txt"; + + TdFilePtr pOutFD = taosCreateFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + ASSERT_NE(pOutFD, nullptr); + printf("create file success\n"); + + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE); + printf("open file\n"); + ASSERT_NE(pFile, nullptr); + + int ret = taosLockFile(pFile); + printf("lock file ret:%d\n", ret); + ASSERT_EQ(ret, 0); + + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + + TdThread thread1, thread2; + taosThreadCreate(&(thread1), &thattr, fileOperateOnBusyThread, (void *)fname); + taosThreadAttrDestroy(&thattr); + + taosThreadJoin(thread1, NULL); + taosThreadClear(&thread1); + + ret = taosUnLockFile(pFile); + printf("unlock file ret:%d\n", ret); + ASSERT_EQ(ret, 0); + + ret = taosCloseFile(&pFile); + printf("close file ret:%d\n", ret); + ASSERT_EQ(ret, 0); + + taosThreadCreate(&(thread2), &thattr, fileOperateOnFreeThread, (void *)fname); + taosThreadAttrDestroy(&thattr); + + taosThreadJoin(thread2, NULL); + taosThreadClear(&thread2); + + //int ret = taosRemoveFile(fname); + //ASSERT_EQ(ret, 0); + //printf("remove file success"); +} + +#pragma GCC diagnostic pop diff --git a/source/util/src/talgo.c b/source/util/src/talgo.c index 057d3a620c714b22e0dea9643f672782c93424ed..d9319485b7c3bbed717c054f6d63f91ca2220063 100644 --- a/source/util/src/talgo.c +++ b/source/util/src/talgo.c @@ -39,7 +39,7 @@ static void median(void *src, int64_t size, int64_t s, int64_t e, const void *pa doswap(elePtrAt(src, size, s), elePtrAt(src, size, e), size, buf); } - assert(comparFn(elePtrAt(src, size, mid), elePtrAt(src, size, s), param) <= 0 && + ASSERT(comparFn(elePtrAt(src, size, mid), elePtrAt(src, size, s), param) <= 0 && comparFn(elePtrAt(src, size, s), elePtrAt(src, size, e), param) <= 0); #ifdef _DEBUG_VIEW diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 4bd829442372305501d1d571bdec910b2182fcfe..6920925e5f82b16ba6d213ffbe03d6fa40d21d06 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -191,6 +191,8 @@ void* taosArrayReserve(SArray* pArray, int32_t num) { void* dst = TARRAY_GET_ELEM(pArray, pArray->size); pArray->size += num; + memset(dst, 0, num * pArray->elemSize); + return dst; } @@ -226,7 +228,7 @@ size_t taosArrayGetSize(const SArray* pArray) { if (pArray == NULL) { return 0; } - return pArray->size; + return TARRAY_SIZE(pArray); } void taosArraySetSize(SArray* pArray, size_t size) { @@ -295,6 +297,20 @@ void taosArrayRemove(SArray* pArray, size_t index) { pArray->size -= 1; } +void taosArrayRemoveBatch(SArray* pArray, size_t index, size_t num, FDelete fp) { + ASSERT(index + num <= pArray->size); + + if (fp) { + for (int32_t i = 0; i < num; i++) { + fp(taosArrayGet(pArray, index + i)); + } + } + + memmove((char*)pArray->pData + index * pArray->elemSize, (char*)pArray->pData + (index + num) * pArray->elemSize, + (pArray->size - index - num) * pArray->elemSize); + pArray->size -= num; +} + SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize) { assert(src != NULL && elemSize > 0); SArray* pDst = taosArrayInit(size, elemSize); diff --git a/source/util/src/tbuffer.c b/source/util/src/tbuffer.c deleted file mode 100644 index d2fac72c77dc55a1c9e7c087d1af8f78dc466409..0000000000000000000000000000000000000000 --- a/source/util/src/tbuffer.c +++ /dev/null @@ -1,424 +0,0 @@ -/* - * Copyright (c) 2020 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "tbuffer.h" -#include "texception.h" - -typedef union Un4B { - uint32_t ui; - float f; -} Un4B; -#if __STDC_VERSION__ >= 201112LL -static_assert(sizeof(Un4B) == sizeof(uint32_t), "sizeof(Un4B) must equal to sizeof(uint32_t)"); -static_assert(sizeof(Un4B) == sizeof(float), "sizeof(Un4B) must equal to sizeof(float)"); -#endif - -typedef union Un8B { - uint64_t ull; - double d; -} Un8B; -#if __STDC_VERSION__ >= 201112LL -static_assert(sizeof(Un8B) == sizeof(uint64_t), "sizeof(Un8B) must equal to sizeof(uint64_t)"); -static_assert(sizeof(Un8B) == sizeof(double), "sizeof(Un8B) must equal to sizeof(double)"); -#endif - -//////////////////////////////////////////////////////////////////////////////// -// reader functions - -size_t tbufSkip(SBufferReader* buf, size_t size) { - if ((buf->pos + size) > buf->size) { - THROW(-1); - } - size_t old = buf->pos; - buf->pos += size; - return old; -} - -const char* tbufRead(SBufferReader* buf, size_t size) { - const char* ret = buf->data + buf->pos; - tbufSkip(buf, size); - return ret; -} - -void tbufReadToBuffer(SBufferReader* buf, void* dst, size_t size) { - assert(dst != NULL); - // always using memcpy, leave optimization to compiler - memcpy(dst, tbufRead(buf, size), size); -} - -static size_t tbufReadLength(SBufferReader* buf) { - // maximum length is 65535, if larger length is required - // this function and the corresponding write function need to be - // revised. - uint16_t l = tbufReadUint16(buf); - return l; -} - -const char* tbufReadString(SBufferReader* buf, size_t* len) { - size_t l = tbufReadLength(buf); - const char* ret = buf->data + buf->pos; - tbufSkip(buf, l + 1); - if (ret[l] != 0) { - THROW(-1); - } - if (len != NULL) { - *len = l; - } - return ret; -} - -size_t tbufReadToString(SBufferReader* buf, char* dst, size_t size) { - assert(dst != NULL); - size_t len; - const char* str = tbufReadString(buf, &len); - if (len >= size) { - len = size - 1; - } - memcpy(dst, str, len); - dst[len] = 0; - return len; -} - -const char* tbufReadBinary(SBufferReader* buf, size_t* len) { - size_t l = tbufReadLength(buf); - const char* ret = buf->data + buf->pos; - tbufSkip(buf, l); - if (len != NULL) { - *len = l; - } - return ret; -} - -size_t tbufReadToBinary(SBufferReader* buf, void* dst, size_t size) { - assert(dst != NULL); - size_t len; - const char* data = tbufReadBinary(buf, &len); - if (len >= size) { - len = size; - } - memcpy(dst, data, len); - return len; -} - -bool tbufReadBool(SBufferReader* buf) { - bool ret; - tbufReadToBuffer(buf, &ret, sizeof(ret)); - return ret; -} - -char tbufReadChar(SBufferReader* buf) { - char ret; - tbufReadToBuffer(buf, &ret, sizeof(ret)); - return ret; -} - -int8_t tbufReadInt8(SBufferReader* buf) { - int8_t ret; - tbufReadToBuffer(buf, &ret, sizeof(ret)); - return ret; -} - -uint8_t tbufReadUint8(SBufferReader* buf) { - uint8_t ret; - tbufReadToBuffer(buf, &ret, sizeof(ret)); - return ret; -} - -int16_t tbufReadInt16(SBufferReader* buf) { - int16_t ret; - tbufReadToBuffer(buf, &ret, sizeof(ret)); - if (buf->endian) { - return (int16_t)ntohs(ret); - } - return ret; -} - -uint16_t tbufReadUint16(SBufferReader* buf) { - uint16_t ret; - tbufReadToBuffer(buf, &ret, sizeof(ret)); - if (buf->endian) { - return ntohs(ret); - } - return ret; -} - -int32_t tbufReadInt32(SBufferReader* buf) { - int32_t ret; - tbufReadToBuffer(buf, &ret, sizeof(ret)); - if (buf->endian) { - return (int32_t)ntohl(ret); - } - return ret; -} - -uint32_t tbufReadUint32(SBufferReader* buf) { - uint32_t ret; - tbufReadToBuffer(buf, &ret, sizeof(ret)); - if (buf->endian) { - return ntohl(ret); - } - return ret; -} - -int64_t tbufReadInt64(SBufferReader* buf) { - int64_t ret; - tbufReadToBuffer(buf, &ret, sizeof(ret)); - if (buf->endian) { - return (int64_t)htobe64(ret); // TODO: ntohll - } - return ret; -} - -uint64_t tbufReadUint64(SBufferReader* buf) { - uint64_t ret; - tbufReadToBuffer(buf, &ret, sizeof(ret)); - if (buf->endian) { - return htobe64(ret); // TODO: ntohll - } - return ret; -} - -float tbufReadFloat(SBufferReader* buf) { - Un4B _un; - tbufReadToBuffer(buf, &_un, sizeof(_un)); - if (buf->endian) { - _un.ui = ntohl(_un.ui); - } - return _un.f; -} - -double tbufReadDouble(SBufferReader* buf) { - Un8B _un; - tbufReadToBuffer(buf, &_un, sizeof(_un)); - if (buf->endian) { - _un.ull = htobe64(_un.ull); - } - return _un.d; -} - -//////////////////////////////////////////////////////////////////////////////// -// writer functions - -void tbufCloseWriter(SBufferWriter* buf) { - taosMemoryFreeClear(buf->data); - // (*buf->allocator)( buf->data, 0 ); // potential memory leak. - buf->data = NULL; - buf->pos = 0; - buf->size = 0; -} - -void tbufEnsureCapacity(SBufferWriter* buf, size_t size) { - size += buf->pos; - if (size > buf->size) { - size_t nsize = size + buf->size; - char* data = (*buf->allocator)(buf->data, nsize); - // TODO: the exception should be thrown by the allocator function - if (data == NULL) { - THROW(-1); - } - buf->data = data; - buf->size = nsize; - } -} - -size_t tbufReserve(SBufferWriter* buf, size_t size) { - tbufEnsureCapacity(buf, size); - size_t old = buf->pos; - buf->pos += size; - return old; -} - -char* tbufGetData(SBufferWriter* buf, bool takeOver) { - char* ret = buf->data; - if (takeOver) { - buf->pos = 0; - buf->size = 0; - buf->data = NULL; - } - return ret; -} - -void tbufWrite(SBufferWriter* buf, const void* data, size_t size) { - assert(data != NULL); - tbufEnsureCapacity(buf, size); - memcpy(buf->data + buf->pos, data, size); - buf->pos += size; -} - -void tbufWriteAt(SBufferWriter* buf, size_t pos, const void* data, size_t size) { - assert(data != NULL); - // this function can only be called to fill the gap on previous writes, - // so 'pos + size <= buf->pos' must be true - assert(pos + size <= buf->pos); - memcpy(buf->data + pos, data, size); -} - -static void tbufWriteLength(SBufferWriter* buf, size_t len) { - // maximum length is 65535, if larger length is required - // this function and the corresponding read function need to be - // revised. - assert(len <= 0xffff); - tbufWriteUint16(buf, (uint16_t)len); -} - -void tbufWriteStringLen(SBufferWriter* buf, const char* str, size_t len) { - tbufWriteLength(buf, len); - tbufWrite(buf, str, len); - tbufWriteChar(buf, '\0'); -} - -void tbufWriteString(SBufferWriter* buf, const char* str) { tbufWriteStringLen(buf, str, strlen(str)); } - -void tbufWriteBinary(SBufferWriter* buf, const void* data, size_t len) { - tbufWriteLength(buf, len); - tbufWrite(buf, data, len); -} - -void tbufWriteBool(SBufferWriter* buf, bool data) { tbufWrite(buf, &data, sizeof(data)); } - -void tbufWriteBoolAt(SBufferWriter* buf, size_t pos, bool data) { tbufWriteAt(buf, pos, &data, sizeof(data)); } - -void tbufWriteChar(SBufferWriter* buf, char data) { tbufWrite(buf, &data, sizeof(data)); } - -void tbufWriteCharAt(SBufferWriter* buf, size_t pos, char data) { tbufWriteAt(buf, pos, &data, sizeof(data)); } - -void tbufWriteInt8(SBufferWriter* buf, int8_t data) { tbufWrite(buf, &data, sizeof(data)); } - -void tbufWriteInt8At(SBufferWriter* buf, size_t pos, int8_t data) { tbufWriteAt(buf, pos, &data, sizeof(data)); } - -void tbufWriteUint8(SBufferWriter* buf, uint8_t data) { tbufWrite(buf, &data, sizeof(data)); } - -void tbufWriteUint8At(SBufferWriter* buf, size_t pos, uint8_t data) { tbufWriteAt(buf, pos, &data, sizeof(data)); } - -void tbufWriteInt16(SBufferWriter* buf, int16_t data) { - if (buf->endian) { - data = (int16_t)htons(data); - } - tbufWrite(buf, &data, sizeof(data)); -} - -void tbufWriteInt16At(SBufferWriter* buf, size_t pos, int16_t data) { - if (buf->endian) { - data = (int16_t)htons(data); - } - tbufWriteAt(buf, pos, &data, sizeof(data)); -} - -void tbufWriteUint16(SBufferWriter* buf, uint16_t data) { - if (buf->endian) { - data = htons(data); - } - tbufWrite(buf, &data, sizeof(data)); -} - -void tbufWriteUint16At(SBufferWriter* buf, size_t pos, uint16_t data) { - if (buf->endian) { - data = htons(data); - } - tbufWriteAt(buf, pos, &data, sizeof(data)); -} - -void tbufWriteInt32(SBufferWriter* buf, int32_t data) { - if (buf->endian) { - data = (int32_t)htonl(data); - } - tbufWrite(buf, &data, sizeof(data)); -} - -void tbufWriteInt32At(SBufferWriter* buf, size_t pos, int32_t data) { - if (buf->endian) { - data = (int32_t)htonl(data); - } - tbufWriteAt(buf, pos, &data, sizeof(data)); -} - -void tbufWriteUint32(SBufferWriter* buf, uint32_t data) { - if (buf->endian) { - data = htonl(data); - } - tbufWrite(buf, &data, sizeof(data)); -} - -void tbufWriteUint32At(SBufferWriter* buf, size_t pos, uint32_t data) { - if (buf->endian) { - data = htonl(data); - } - tbufWriteAt(buf, pos, &data, sizeof(data)); -} - -void tbufWriteInt64(SBufferWriter* buf, int64_t data) { - if (buf->endian) { - data = (int64_t)htobe64(data); - } - tbufWrite(buf, &data, sizeof(data)); -} - -void tbufWriteInt64At(SBufferWriter* buf, size_t pos, int64_t data) { - if (buf->endian) { - data = (int64_t)htobe64(data); - } - tbufWriteAt(buf, pos, &data, sizeof(data)); -} - -void tbufWriteUint64(SBufferWriter* buf, uint64_t data) { - if (buf->endian) { - data = htobe64(data); - } - tbufWrite(buf, &data, sizeof(data)); -} - -void tbufWriteUint64At(SBufferWriter* buf, size_t pos, uint64_t data) { - if (buf->endian) { - data = htobe64(data); - } - tbufWriteAt(buf, pos, &data, sizeof(data)); -} - -void tbufWriteFloat(SBufferWriter* buf, float data) { - Un4B _un; - _un.f = data; - if (buf->endian) { - _un.ui = htonl(_un.ui); - } - tbufWrite(buf, &_un, sizeof(_un)); -} - -void tbufWriteFloatAt(SBufferWriter* buf, size_t pos, float data) { - Un4B _un; - _un.f = data; - if (buf->endian) { - _un.ui = htonl(_un.ui); - } - tbufWriteAt(buf, pos, &_un, sizeof(_un)); -} - -void tbufWriteDouble(SBufferWriter* buf, double data) { - Un8B _un; - _un.d = data; - if (buf->endian) { - _un.ull = htobe64(_un.ull); - } - tbufWrite(buf, &_un, sizeof(_un)); -} - -void tbufWriteDoubleAt(SBufferWriter* buf, size_t pos, double data) { - Un8B _un; - _un.d = data; - if (buf->endian) { - _un.ull = htobe64(_un.ull); - } - tbufWriteAt(buf, pos, &_un, sizeof(_un)); -} diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index d84a3d25c604d0089e1abc5ca2778a255a13c859..75d6a43b249fd56f8b6274b0f68db27627182d08 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -21,7 +21,9 @@ #include "tdef.h" #include "thash.h" #include "tlog.h" +#include "tutil.h" #include "types.h" +#include "osString.h" int32_t setChkInBytes1(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; @@ -207,16 +209,16 @@ int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) { int32_t len1 = varDataLen(pLeft); int32_t len2 = varDataLen(pRight); - if (len1 != len2) { - return len1 > len2 ? 1 : -1; - } else { - int32_t ret = memcmp((TdUcs4 *)pLeft, (TdUcs4 *)pRight, len1); - if (ret == 0) { + int32_t ret = tasoUcs4Compare((TdUcs4 *)varDataVal(pLeft), (TdUcs4 *)varDataVal(pRight), len1>len2 ? len2:len1); + if (ret == 0) { + if (len1 > len2) + return 1; + else if(len1 < len2) + return -1; + else return 0; - } else { - return ret > 0 ? 1 : -1; - } } + return (ret < 0) ? -1 : 1; } int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight) { @@ -243,7 +245,7 @@ int32_t compareJsonVal(const void *pLeft, const void *pRight) { } else if (leftType == TSDB_DATA_TYPE_NULL) { return 0; } else { - assert(0); + ASSERTS(0, "data type unexpected"); return 0; } } @@ -997,6 +999,7 @@ int32_t compareUint64Uint32(const void *pLeft, const void *pRight) { } int32_t compareJsonValDesc(const void *pLeft, const void *pRight) { return compareJsonVal(pRight, pLeft); } + /* * Compare two strings * TSDB_MATCH: Match @@ -1007,59 +1010,63 @@ int32_t compareJsonValDesc(const void *pLeft, const void *pRight) { return compa * '_': Matches one character * */ -int32_t patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) { +int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t ssize, + const SPatternCompareInfo *pInfo) { char c, c1; int32_t i = 0; int32_t j = 0; - int32_t o = 0; - int32_t m = 0; + int32_t nMatchChar = 0; - while ((c = patterStr[i++]) != 0) { + while ((i < psize) && ((c = pattern[i++]) != 0)) { if (c == pInfo->matchAll) { /* Match "*" */ - while ((c = patterStr[i++]) == pInfo->matchAll || c == pInfo->matchOne) { + while ((i < psize) && ((c = pattern[i++]) == pInfo->matchAll || c == pInfo->matchOne)) { if (c == pInfo->matchOne) { - if (j > size || str[j++] == 0) { - // empty string, return not match + if (j >= ssize || str[j++] == 0) { // empty string, return not match return TSDB_PATTERN_NOWILDCARDMATCH; } else { - ++o; + ++nMatchChar; } } } - if (c == 0) { + if (i >= psize && (c == pInfo->umatchOne || c == pInfo->umatchAll)) { return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */ } - char next[3] = {toupper(c), tolower(c), 0}; - m = o; + char rejectList[2] = {toupper(c), tolower(c)}; + + str += nMatchChar; + int32_t remain = ssize - nMatchChar; while (1) { - size_t n = strcspn(str + m, next); - str += m + n; + size_t n = tstrncspn(str, remain, rejectList, 2); + + str += n; + remain -= n; - if (str[0] == 0 || (n >= size)) { + if ((remain <= 0) || str[0] == 0) { break; } - int32_t ret = patternMatch(&patterStr[i], ++str, size - n - 1, pInfo); + int32_t ret = patternMatch(&pattern[i], psize - i, ++str, --remain, pInfo); if (ret != TSDB_PATTERN_NOMATCH) { return ret; } - m = 0; } + return TSDB_PATTERN_NOWILDCARDMATCH; } - c1 = str[j++]; - ++o; + if (j < ssize) { + c1 = str[j++]; + ++nMatchChar; - if (j <= size) { - if (c == '\\' && patterStr[i] == '_' && c1 == '_') { + if (c == '\\' && pattern[i] == '_' && c1 == '_') { i++; continue; } + if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) { continue; } @@ -1068,39 +1075,49 @@ int32_t patternMatch(const char *patterStr, const char *str, size_t size, const return TSDB_PATTERN_NOMATCH; } - return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; + return (j >= ssize || str[j] == 0) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; } -int32_t WCSPatternMatch(const TdUcs4 *patterStr, const TdUcs4 *str, size_t size, const SPatternCompareInfo *pInfo) { +int32_t wcsPatternMatch(const TdUcs4 *pattern, size_t psize, const TdUcs4 *str, size_t ssize, + const SPatternCompareInfo *pInfo) { TdUcs4 c, c1; - TdUcs4 matchOne = L'_'; // "_" - TdUcs4 matchAll = L'%'; // "%" int32_t i = 0; int32_t j = 0; + int32_t nMatchChar = 0; - while ((c = patterStr[i++]) != 0) { - if (c == matchAll) { /* Match "%" */ + while ((i < psize) && ((c = pattern[i++]) != 0)) { + if (c == pInfo->umatchAll) { /* Match "%" */ - while ((c = patterStr[i++]) == matchAll || c == matchOne) { - if (c == matchOne && (j >= size || str[j++] == 0)) { - return TSDB_PATTERN_NOWILDCARDMATCH; + while ((i < psize) && ((c = pattern[i++]) == pInfo->umatchAll || c == pInfo->umatchOne)) { + if (c == pInfo->umatchOne) { + if (j >= ssize || str[j++] == 0) { + return TSDB_PATTERN_NOWILDCARDMATCH; + } else { + ++nMatchChar; + } } } - if (c == 0) { + + if (i >= psize && (c == pInfo->umatchOne || c == pInfo->umatchAll)) { return TSDB_PATTERN_MATCH; } - TdUcs4 accept[3] = {towupper(c), towlower(c), 0}; + TdUcs4 rejectList[2] = {towupper(c), towlower(c)}; + + str += nMatchChar; + int32_t remain = ssize - nMatchChar; while (1) { - size_t n = wcscspn(str, accept); + size_t n = twcsncspn(str, remain, rejectList, 2); str += n; - if (str[0] == 0 || (n >= size)) { + remain -= n; + + if ((remain <= 0) || str[0] == 0) { break; } - int32_t ret = WCSPatternMatch(&patterStr[i], ++str, size - n - 1, pInfo); + int32_t ret = wcsPatternMatch(&pattern[i], psize - i, ++str, --remain, pInfo); if (ret != TSDB_PATTERN_NOMATCH) { return ret; } @@ -1109,10 +1126,16 @@ int32_t WCSPatternMatch(const TdUcs4 *patterStr, const TdUcs4 *str, size_t size, return TSDB_PATTERN_NOWILDCARDMATCH; } - c1 = str[j++]; + if (j < ssize) { + c1 = str[j++]; + nMatchChar++; - if (j <= size) { - if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) { + if (c == L'\\' && pattern[i] == L'_' && c1 == L'_') { + i++; + continue; + } + + if (c == c1 || towlower(c) == towlower(c1) || (c == pInfo->umatchOne && c1 != 0)) { continue; } } @@ -1120,16 +1143,39 @@ int32_t WCSPatternMatch(const TdUcs4 *patterStr, const TdUcs4 *str, size_t size, return TSDB_PATTERN_NOMATCH; } - return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; + return (j >= ssize || str[j] == 0) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; } -int32_t compareStrRegexCompMatch(const void *pLeft, const void *pRight) { return compareStrRegexComp(pLeft, pRight); } +int32_t comparestrRegexNMatch(const void *pLeft, const void *pRight) { + return comparestrRegexMatch(pLeft, pRight) ? 0 : 1; +} + +static int32_t doExecRegexMatch(const char *pString, const char *pPattern) { + int32_t ret = 0; + regex_t regex; + char msgbuf[256] = {0}; + + int32_t cflags = REG_EXTENDED; + if ((ret = regcomp(®ex, pPattern, cflags)) != 0) { + regerror(ret, ®ex, msgbuf, tListLen(msgbuf)); + + uError("Failed to compile regex pattern %s. reason %s", pPattern, msgbuf); + regfree(®ex); + return 1; + } + + regmatch_t pmatch[1]; + ret = regexec(®ex, pString, 1, pmatch, 0); + if (ret != 0 && ret != REG_NOMATCH) { + regerror(ret, ®ex, msgbuf, sizeof(msgbuf)); + uDebug("Failed to match %s with pattern %s, reason %s", pString, pPattern, msgbuf) + } -int32_t compareStrRegexCompNMatch(const void *pLeft, const void *pRight) { - return compareStrRegexComp(pLeft, pRight) ? 0 : 1; + regfree(®ex); + return (ret == 0) ? 0 : 1; } -int32_t compareStrRegexComp(const void *pLeft, const void *pRight) { +int32_t comparestrRegexMatch(const void *pLeft, const void *pRight) { size_t sz = varDataLen(pRight); char *pattern = taosMemoryMalloc(sz + 1); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); @@ -1140,30 +1186,49 @@ int32_t compareStrRegexComp(const void *pLeft, const void *pRight) { memcpy(str, varDataVal(pLeft), sz); str[sz] = 0; - int32_t errCode = 0; - regex_t regex; - char msgbuf[256] = {0}; + int32_t ret = doExecRegexMatch(str, pattern); - int32_t cflags = REG_EXTENDED; - if ((errCode = regcomp(®ex, pattern, cflags)) != 0) { - regerror(errCode, ®ex, msgbuf, sizeof(msgbuf)); - uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf); - regfree(®ex); - taosMemoryFree(str); + taosMemoryFree(str); + taosMemoryFree(pattern); + + return (ret == 0) ? 0 : 1; + ; +} + +int32_t comparewcsRegexMatch(const void *pString, const void *pPattern) { + size_t len = varDataLen(pPattern); + char *pattern = taosMemoryMalloc(len + 1); + + int convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pPattern), len, pattern); + if (convertLen < 0) { taosMemoryFree(pattern); - return 1; + return TSDB_CODE_APP_ERROR; } - errCode = regexec(®ex, str, 0, NULL, 0); - if (errCode != 0 && errCode != REG_NOMATCH) { - regerror(errCode, ®ex, msgbuf, sizeof(msgbuf)); - uDebug("Failed to match %s with pattern %s, reason %s", str, pattern, msgbuf) + pattern[convertLen] = 0; + + len = varDataLen(pString); + char *str = taosMemoryMalloc(len + 1); + convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pString), len, str); + if (convertLen < 0) { + taosMemoryFree(str); + taosMemoryFree(pattern); + + return TSDB_CODE_APP_ERROR; } - int32_t result = (errCode == 0) ? 0 : 1; - regfree(®ex); + + str[convertLen] = 0; + + int32_t ret = doExecRegexMatch(str, pattern); + taosMemoryFree(str); taosMemoryFree(pattern); - return result; + + return (ret == 0) ? 0 : 1; +} + +int32_t comparewcsRegexNMatch(const void *pLeft, const void *pRight) { + return comparewcsRegexMatch(pLeft, pRight) ? 0 : 1; } int32_t taosArrayCompareString(const void *a, const void *b) { @@ -1173,46 +1238,35 @@ int32_t taosArrayCompareString(const void *a, const void *b) { return compareLenPrefixedStr(x, y); } -int32_t compareStrPatternMatch(const void *pLeft, const void *pRight) { - SPatternCompareInfo pInfo = {'%', '_'}; - - assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN); - char *pattern = taosMemoryCalloc(varDataLen(pRight) + 1, sizeof(char)); - memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); +int32_t comparestrPatternMatch(const void *pLeft, const void *pRight) { + SPatternCompareInfo pInfo = PATTERN_COMPARE_INFO_INITIALIZER; + ASSERT(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN); + size_t pLen = varDataLen(pRight); size_t sz = varDataLen(pLeft); - char *buf = taosMemoryMalloc(sz + 1); - memcpy(buf, varDataVal(pLeft), sz); - buf[sz] = 0; - int32_t ret = patternMatch(pattern, buf, sz, &pInfo); - taosMemoryFree(buf); - taosMemoryFree(pattern); + int32_t ret = patternMatch(varDataVal(pRight), pLen, varDataVal(pLeft), sz, &pInfo); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } -int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight) { - return compareStrPatternMatch(pLeft, pRight) ? 0 : 1; +int32_t comparestrPatternNMatch(const void *pLeft, const void *pRight) { + return comparestrPatternMatch(pLeft, pRight) ? 0 : 1; } -int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight) { - SPatternCompareInfo pInfo = {'%', '_'}; - - assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE); +int32_t comparewcsPatternMatch(const void *pLeft, const void *pRight) { + SPatternCompareInfo pInfo = PATTERN_COMPARE_INFO_INITIALIZER; - char *pattern = taosMemoryCalloc(varDataLen(pRight) + TSDB_NCHAR_SIZE, 1); - memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); - - int32_t ret = - WCSPatternMatch((TdUcs4 *)pattern, (TdUcs4 *)varDataVal(pLeft), varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo); - taosMemoryFree(pattern); + size_t psize = varDataLen(pRight); + int32_t ret = wcsPatternMatch((TdUcs4 *)varDataVal(pRight), psize / TSDB_NCHAR_SIZE, (TdUcs4 *)varDataVal(pLeft), + varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } -int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight) { - return compareWStrPatternMatch(pLeft, pRight) ? 0 : 1; +int32_t comparewcsPatternNMatch(const void *pLeft, const void *pRight) { + return comparewcsPatternMatch(pLeft, pRight) ? 0 : 1; } + __compar_fn_t getComparFunc(int32_t type, int32_t optr) { __compar_fn_t comparFn = NULL; @@ -1235,7 +1289,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { case TSDB_DATA_TYPE_TIMESTAMP: return setChkInBytes8; default: - assert(0); + ASSERTS(0, "data type unexpected"); } } @@ -1258,7 +1312,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { case TSDB_DATA_TYPE_TIMESTAMP: return setChkNotInBytes8; default: - assert(0); + ASSERTS(0, "data type unexpected"); } } @@ -1285,13 +1339,13 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { break; case TSDB_DATA_TYPE_BINARY: { if (optr == OP_TYPE_MATCH) { - comparFn = compareStrRegexCompMatch; + comparFn = comparestrRegexMatch; } else if (optr == OP_TYPE_NMATCH) { - comparFn = compareStrRegexCompNMatch; + comparFn = comparestrRegexNMatch; } else if (optr == OP_TYPE_LIKE) { /* wildcard query using like operator */ - comparFn = compareStrPatternMatch; + comparFn = comparestrPatternMatch; } else if (optr == OP_TYPE_NOT_LIKE) { /* wildcard query using like operator */ - comparFn = compareStrPatternNotMatch; + comparFn = comparestrPatternNMatch; } else if (optr == OP_TYPE_IN) { comparFn = compareChkInString; } else if (optr == OP_TYPE_NOT_IN) { @@ -1305,13 +1359,13 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { case TSDB_DATA_TYPE_NCHAR: { if (optr == OP_TYPE_MATCH) { - comparFn = compareStrRegexCompMatch; + comparFn = comparewcsRegexMatch; } else if (optr == OP_TYPE_NMATCH) { - comparFn = compareStrRegexCompNMatch; + comparFn = comparewcsRegexNMatch; } else if (optr == OP_TYPE_LIKE) { - comparFn = compareWStrPatternMatch; + comparFn = comparewcsPatternMatch; } else if (optr == OP_TYPE_NOT_LIKE) { - comparFn = compareWStrPatternNotMatch; + comparFn = comparewcsPatternNMatch; } else if (optr == OP_TYPE_IN) { comparFn = compareChkInString; } else if (optr == OP_TYPE_NOT_IN) { diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 18305e594b7942019c955523109941b05779618c..64d550e874f8a7f22b54a3f5ac27e59c317f8025 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -273,42 +273,90 @@ int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, cha char bit = bit_per_integer[(int32_t)selector]; // bit = 3 int32_t elems = selector_to_elems[(int32_t)selector]; - for (int32_t i = 0; i < elems; i++) { - uint64_t zigzag_value; + // Optimize the performance, by remove the constantly switch operation. + int32_t v = 0; + uint64_t zigzag_value; + + switch (type) { + case TSDB_DATA_TYPE_BIGINT: { + for (int32_t i = 0; i < elems; i++) { + if (selector == 0 || selector == 1) { + zigzag_value = 0; + } else { + zigzag_value = ((w >> (4 + v)) & INT64MASK(bit)); + } - if (selector == 0 || selector == 1) { - zigzag_value = 0; - } else { - zigzag_value = ((w >> (4 + bit * i)) & INT64MASK(bit)); - } - int64_t diff = ZIGZAG_DECODE(int64_t, zigzag_value); - int64_t curr_value = diff + prev_value; - prev_value = curr_value; + int64_t diff = ZIGZAG_DECODE(int64_t, zigzag_value); + int64_t curr_value = diff + prev_value; + prev_value = curr_value; - switch (type) { - case TSDB_DATA_TYPE_BIGINT: *((int64_t *)output + _pos) = (int64_t)curr_value; _pos++; - break; - case TSDB_DATA_TYPE_INT: + + v += bit; + if ((++count) == nelements) break; + } + } break; + case TSDB_DATA_TYPE_INT: { + for (int32_t i = 0; i < elems; i++) { + if (selector == 0 || selector == 1) { + zigzag_value = 0; + } else { + zigzag_value = ((w >> (4 + v)) & INT64MASK(bit)); + } + + int64_t diff = ZIGZAG_DECODE(int64_t, zigzag_value); + int64_t curr_value = diff + prev_value; + prev_value = curr_value; + *((int32_t *)output + _pos) = (int32_t)curr_value; _pos++; - break; - case TSDB_DATA_TYPE_SMALLINT: + + v += bit; + if ((++count) == nelements) break; + } + } break; + case TSDB_DATA_TYPE_SMALLINT: { + for (int32_t i = 0; i < elems; i++) { + if (selector == 0 || selector == 1) { + zigzag_value = 0; + } else { + zigzag_value = ((w >> (4 + v)) & INT64MASK(bit)); + } + + int64_t diff = ZIGZAG_DECODE(int64_t, zigzag_value); + int64_t curr_value = diff + prev_value; + prev_value = curr_value; + *((int16_t *)output + _pos) = (int16_t)curr_value; _pos++; - break; - case TSDB_DATA_TYPE_TINYINT: + + v += bit; + if ((++count) == nelements) break; + } + } break; + + case TSDB_DATA_TYPE_TINYINT: { + for (int32_t i = 0; i < elems; i++) { + if (selector == 0 || selector == 1) { + zigzag_value = 0; + } else { + zigzag_value = ((w >> (4 + v)) & INT64MASK(bit)); + } + + int64_t diff = ZIGZAG_DECODE(int64_t, zigzag_value); + int64_t curr_value = diff + prev_value; + prev_value = curr_value; + *((int8_t *)output + _pos) = (int8_t)curr_value; _pos++; - break; - default: - perror("Wrong integer types.\n"); - return -1; - } - count++; - if (count == nelements) break; + + v += bit; + if ((++count) == nelements) break; + } + } break; } + ip += LONG_BYTES; } @@ -470,7 +518,7 @@ int32_t tsDecompressStringImp(const char *const input, int32_t compressedSize, c // TODO: Take care here, we assumes little endian encoding. int32_t tsCompressTimestampImp(const char *const input, const int32_t nelements, char *const output) { int32_t _pos = 1; - assert(nelements >= 0); + ASSERTS(nelements >= 0, "nelements is negative"); if (nelements == 0) return 0; @@ -565,7 +613,7 @@ _exit_over: } int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelements, char *const output) { - assert(nelements >= 0); + ASSERTS(nelements >= 0, "nelements is negative"); if (nelements == 0) return 0; if (input[0] == 0) { @@ -629,7 +677,7 @@ int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelement } } else { - assert(0); + ASSERT(0); return -1; } } @@ -758,7 +806,7 @@ int32_t tsDecompressDoubleImp(const char *const input, const int32_t nelements, uint64_t prev_value = 0; for (int32_t i = 0; i < nelements; i++) { - if (i % 2 == 0) { + if ((i & 0x01) == 0) { flags = input[ipos++]; } @@ -2146,7 +2194,7 @@ int32_t tsCompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, in int32_t len = tsCompressTimestampImp(pIn, nEle, pBuf); return tsCompressStringImp(pBuf, len, pOut, nOut); } else { - assert(0); + ASSERTS(0, "compress algo not one or two stage"); return -1; } } @@ -2159,7 +2207,7 @@ int32_t tsDecompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; return tsDecompressTimestampImp(pBuf, nEle, pOut); } else { - assert(0); + ASSERTS(0, "compress algo invalid"); return -1; } } @@ -2180,7 +2228,7 @@ int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_ int32_t len = tsCompressFloatImp(pIn, nEle, pBuf); return tsCompressStringImp(pBuf, len, pOut, nOut); } else { - assert(0); + ASSERTS(0, "compress algo invalid"); return -1; } #ifdef TD_TSZ @@ -2203,7 +2251,7 @@ int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int3 if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; return tsDecompressFloatImp(pBuf, nEle, pOut); } else { - assert(0); + ASSERTS(0, "compress algo invalid"); return -1; } #ifdef TD_TSZ @@ -2227,7 +2275,7 @@ int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32 int32_t len = tsCompressDoubleImp(pIn, nEle, pBuf); return tsCompressStringImp(pBuf, len, pOut, nOut); } else { - assert(0); + ASSERTS(0, "compress algo invalid"); return -1; } #ifdef TD_TSZ @@ -2250,7 +2298,7 @@ int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; return tsDecompressDoubleImp(pBuf, nEle, pOut); } else { - assert(0); + ASSERTS(0, "compress algo invalid"); return -1; } #ifdef TD_TSZ @@ -2281,7 +2329,7 @@ int32_t tsCompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t } return tsCompressStringImp(pBuf, len, pOut, nOut); } else { - assert(0); + ASSERTS(0, "compress algo invalid"); return -1; } } @@ -2294,7 +2342,7 @@ int32_t tsDecompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32 if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; return tsDecompressBoolImp(pBuf, nEle, pOut); } else { - assert(0); + ASSERTS(0, "compress algo invalid"); return -1; } } @@ -2308,7 +2356,7 @@ int32_t tsCompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int3 int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_TINYINT); return tsCompressStringImp(pBuf, len, pOut, nOut); } else { - assert(0); + ASSERTS(0, "compress algo invalid"); return -1; } } @@ -2321,7 +2369,7 @@ int32_t tsDecompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, in if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_TINYINT); } else { - assert(0); + ASSERTS(0, "compress algo invalid"); return -1; } } @@ -2335,7 +2383,7 @@ int32_t tsCompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_SMALLINT); return tsCompressStringImp(pBuf, len, pOut, nOut); } else { - assert(0); + ASSERTS(0, "compress algo invalid"); return -1; } } @@ -2348,7 +2396,7 @@ int32_t tsDecompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, i if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_SMALLINT); } else { - assert(0); + ASSERTS(0, "compress algo invalid"); return -1; } } @@ -2362,7 +2410,7 @@ int32_t tsCompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_INT); return tsCompressStringImp(pBuf, len, pOut, nOut); } else { - assert(0); + ASSERTS(0, "compress algo invalid"); return -1; } } @@ -2375,7 +2423,7 @@ int32_t tsDecompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_ if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_INT); } else { - assert(0); + ASSERTS(0, "compress algo invalid"); return -1; } } @@ -2389,7 +2437,7 @@ int32_t tsCompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32 int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_BIGINT); return tsCompressStringImp(pBuf, len, pOut, nOut); } else { - assert(0); + ASSERTS(0, "compress algo invalid"); return -1; } } @@ -2402,7 +2450,7 @@ int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_BIGINT); } else { - assert(0); + ASSERTS(0, "compress algo invalid"); return -1; } } diff --git a/source/util/src/tdigest.c b/source/util/src/tdigest.c index 337ee803ffbf4fa3a715251daa1c8b850b06e17d..b46a50b2dc03bcf1242eded9028424327121b612 100644 --- a/source/util/src/tdigest.c +++ b/source/util/src/tdigest.c @@ -27,6 +27,7 @@ #include "tdigest.h" #include "os.h" #include "osMath.h" +#include "tlog.h" #define INTERPOLATE(x, x0, x1) (((x) - (x0)) / ((x1) - (x0))) //#define INTEGRATED_LOCATION(compression, q) ((compression) * (asin(2 * (q) - 1) + M_PI / 2) / M_PI) @@ -135,24 +136,24 @@ void tdigestCompress(TDigest *t) { if (a->mean <= b->mean) { mergeCentroid(&args, a); - assert(args.idx < t->size); + ASSERTS(args.idx < t->size, "idx over size"); i++; } else { mergeCentroid(&args, b); - assert(args.idx < t->size); + ASSERTS(args.idx < t->size, "idx over size"); j++; } } while (i < num_unmerged) { mergeCentroid(&args, &unmerged_centroids[i++]); - assert(args.idx < t->size); + ASSERTS(args.idx < t->size, "idx over size"); } taosMemoryFree((void *)unmerged_centroids); while (j < t->num_centroids) { mergeCentroid(&args, &t->centroids[j++]); - assert(args.idx < t->size); + ASSERTS(args.idx < t->size, "idx over size"); } if (t->total_weight > 0) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index abd8a9055a12f4bb7778e5720958294f58f18eb4..bab3edc870bb888bf2ad6030ff102c88daa05be6 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -51,6 +51,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, "Unable to resolve FQD TAOS_DEFINE_ERROR(TSDB_CODE_RPC_PORT_EADDRINUSE, "Port already in use") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_BROKEN_LINK, "Conn is broken") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_TIMEOUT, "Conn read timeout") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED, "some vnode/qnode/mnode(s) out of service") //common & util TAOS_DEFINE_ERROR(TSDB_CODE_TIME_UNSYNCED, "Client and server's time is not synchronized") @@ -136,6 +137,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CLAUSE_ERROR, "not supported stmt cl TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_KILLED, "Query killed") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_EXEC_NODE, "No available execution node in current query policy configuration") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NOT_STABLE_ERROR, "Table is not a super table") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CACHE_ERROR, "Stmt cache error") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INTERNAL_ERROR, "Internal error") // mnode-common TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS, "Insufficient privilege for operation") @@ -275,8 +278,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST, "Subcribe not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_OFFSET_NOT_EXIST, "Offset not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_READY, "Consumer not ready") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_SUBSCRIBED, "Topic subscribed cannot be dropped") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_MUST_BE_DELETED, "Topic must be dropped first") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CGROUP_USED, "Consumer group being used by some consumer") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_MUST_BE_DELETED, "Topic must be dropped first") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SUB_OPTION, "Invalid subscribe option") TAOS_DEFINE_ERROR(TSDB_CODE_MND_IN_REBALANCE, "Topic being rebalanced") // mnode-stream @@ -316,6 +320,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_SUBSCRIBED, "Table column is subsc TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_AVAIL_BUFPOOL, "No availabe buffer pool") TAOS_DEFINE_ERROR(TSDB_CODE_VND_STOPPED, "Vnode stopped") TAOS_DEFINE_ERROR(TSDB_CODE_VND_DUP_REQUEST, "Duplicate write request") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_QUERY_BUSY, "Query busy") // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") @@ -405,6 +410,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_PROPOSE_NOT_READY, "Sync not ready for pr TAOS_DEFINE_ERROR(TSDB_CODE_SYN_STANDBY_NOT_READY, "Sync not ready for standby") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_BATCH_ERROR, "Sync batch error") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_RESTORING, "Sync is restoring") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_SNAPSHOT_MSG, "Sync invalid snapshot msg") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_BUFFER_FULL, "Sync buffer is full") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INTERNAL_ERROR, "Sync internal error") //tq @@ -573,6 +580,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE, "Invalid timestamp p TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DATA, "Invalid data format") TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DB_CONF, "Invalid schemaless db config") TAOS_DEFINE_ERROR(TSDB_CODE_SML_NOT_SAME_TYPE, "Not the same type like before") +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INTERNAL_ERROR, "Internal error") //tsma TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_INIT_FAILED, "Tsma init failed") @@ -644,13 +652,10 @@ const char* tstrerror(int32_t err) { // this is a system errno if ((err & 0x00ff0000) == 0x00ff0000) { int32_t code = err & 0x0000ffff; - if (code >= 0 && code < 36) { - return strerror(code); - } else { - return "unknown err"; - } + // strerror can handle any invalid code + // invalid code return Unknown error + return strerror(code); } - int32_t s = 0; int32_t e = sizeof(errors) / sizeof(errors[0]); diff --git a/source/util/src/texception.c b/source/util/src/texception.c index 33befb694a0a0a71c01042d9c9b5b19f7da12912..4723e349903505923de5790cdbca223a0d5073b5 100644 --- a/source/util/src/texception.c +++ b/source/util/src/texception.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "texception.h" +#include "tlog.h" static threadlocal SExceptionNode* expList; @@ -71,7 +72,7 @@ static wrapper wrappers[] = { }; void cleanupPush_void_ptr_ptr(bool failOnly, void* func, void* arg1, void* arg2) { - assert(expList->numCleanupAction < expList->maxCleanupAction); + ASSERTS(expList->numCleanupAction < expList->maxCleanupAction, "numCleanupAction over maxCleanupAction"); SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++; ca->wrapper = 0; @@ -82,7 +83,7 @@ void cleanupPush_void_ptr_ptr(bool failOnly, void* func, void* arg1, void* arg2) } void cleanupPush_void_ptr_bool(bool failOnly, void* func, void* arg1, bool arg2) { - assert(expList->numCleanupAction < expList->maxCleanupAction); + ASSERTS(expList->numCleanupAction < expList->maxCleanupAction, "numCleanupAction over maxCleanupAction"); SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++; ca->wrapper = 1; @@ -93,7 +94,7 @@ void cleanupPush_void_ptr_bool(bool failOnly, void* func, void* arg1, bool arg2) } void cleanupPush_void_ptr(bool failOnly, void* func, void* arg) { - assert(expList->numCleanupAction < expList->maxCleanupAction); + ASSERTS(expList->numCleanupAction < expList->maxCleanupAction, "numCleanupAction over maxCleanupAction"); SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++; ca->wrapper = 2; @@ -103,7 +104,7 @@ void cleanupPush_void_ptr(bool failOnly, void* func, void* arg) { } void cleanupPush_int_int(bool failOnly, void* func, int32_t arg) { - assert(expList->numCleanupAction < expList->maxCleanupAction); + ASSERTS(expList->numCleanupAction < expList->maxCleanupAction, "numCleanupAction over maxCleanupAction"); SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++; ca->wrapper = 3; @@ -113,7 +114,7 @@ void cleanupPush_int_int(bool failOnly, void* func, int32_t arg) { } void cleanupPush_void(bool failOnly, void* func) { - assert(expList->numCleanupAction < expList->maxCleanupAction); + ASSERTS(expList->numCleanupAction < expList->maxCleanupAction, "numCleanupAction over maxCleanupAction"); SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++; ca->wrapper = 4; @@ -122,7 +123,7 @@ void cleanupPush_void(bool failOnly, void* func) { } void cleanupPush_int_ptr(bool failOnly, void* func, void* arg) { - assert(expList->numCleanupAction < expList->maxCleanupAction); + ASSERTS(expList->numCleanupAction < expList->maxCleanupAction, "numCleanupAction over maxCleanupAction"); SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++; ca->wrapper = 5; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index f6f814d82b8f1de69e28790bb6407c629446590e..34ad9ae6bc39753c897d932993c917848dba64ee 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -18,6 +18,8 @@ #include "os.h" #include "tconfig.h" #include "tutil.h" +#include "tjson.h" +#include "tglobal.h" #define LOG_MAX_LINE_SIZE (1024) #define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 3) @@ -790,7 +792,7 @@ cmp_end: return ret; } -bool taosAssert(bool condition, const char *file, int32_t line, const char *format, ...) { +bool taosAssertDebug(bool condition, const char *file, int32_t line, const char *format, ...) { if (condition) return false; const char *flags = "UTL FATAL "; @@ -808,7 +810,7 @@ bool taosAssert(bool condition, const char *file, int32_t line, const char *form taosPrintLogImp(1, 255, buffer, len); taosPrintLog(flags, level, dflag, "tAssert at file %s:%d exit:%d", file, line, tsAssert); - taosPrintTrace(flags, level, dflag); + taosPrintTrace(flags, level, dflag, -1); if (tsAssert) { // taosCloseLog(); @@ -822,4 +824,234 @@ bool taosAssert(bool condition, const char *file, int32_t line, const char *form } return true; -} \ No newline at end of file +} + +int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime) { + SJson* pJson = tjsonCreateObject(); + if (pJson == NULL) return -1; + char tmp[4096] = {0}; + + tjsonAddDoubleToObject(pJson, "reportVersion", 1); + + tjsonAddIntegerToObject(pJson, "clusterId", clusterId); + tjsonAddIntegerToObject(pJson, "startTime", startTime); + + taosGetFqdn(tmp); + tjsonAddStringToObject(pJson, "fqdn", tmp); + + tjsonAddIntegerToObject(pJson, "pid", taosGetPId()); + + taosGetAppName(tmp, NULL); + tjsonAddStringToObject(pJson, "appName", tmp); + + if (taosGetOsReleaseName(tmp, sizeof(tmp)) == 0) { + tjsonAddStringToObject(pJson, "os", tmp); + } + + float numOfCores = 0; + if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores) == 0) { + tjsonAddStringToObject(pJson, "cpuModel", tmp); + tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores); + } else { + tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores); + } + + snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB); + tjsonAddStringToObject(pJson, "memory", tmp); + + tjsonAddStringToObject(pJson, "version", version); + tjsonAddStringToObject(pJson, "buildInfo", buildinfo); + tjsonAddStringToObject(pJson, "gitInfo", gitinfo); + + tjsonAddIntegerToObject(pJson, "crashSig", signum); + tjsonAddIntegerToObject(pJson, "crashTs", taosGetTimestampUs()); + +#ifdef _TD_DARWIN_64 + taosLogTraceToBuf(tmp, sizeof(tmp), 4); +#elif !defined(WINDOWS) + taosLogTraceToBuf(tmp, sizeof(tmp), 3); +#else + taosLogTraceToBuf(tmp, sizeof(tmp), 8); +#endif + + tjsonAddStringToObject(pJson, "stackInfo", tmp); + + char* pCont = tjsonToString(pJson); + tjsonDelete(pJson); + + *pMsg = pCont; + + return TSDB_CODE_SUCCESS; +} + + +void taosLogCrashInfo(char* nodeType, char* pMsg, int64_t msgLen, int signum, void *sigInfo) { + const char *flags = "UTL FATAL "; + ELogLevel level = DEBUG_FATAL; + int32_t dflag = 255; + char filepath[PATH_MAX] = {0}; + TdFilePtr pFile = NULL; + + if (pMsg && msgLen > 0) { + snprintf(filepath, sizeof(filepath), "%s%s.%sCrashLog", tsLogDir, TD_DIRSEP, nodeType); + + pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pFile == NULL) { + taosPrintLog(flags, level, dflag, "failed to open file:%s since %s", filepath, terrstr()); + goto _return; + } + + taosLockFile(pFile); + + int64_t writeSize = taosWriteFile(pFile, &msgLen, sizeof(msgLen)); + if (sizeof(msgLen) != writeSize) { + taosUnLockFile(pFile); + taosPrintLog(flags, level, dflag, "failed to write len to file:%s,%p wlen:%" PRId64 " tlen:%lu since %s", + filepath, pFile, writeSize, sizeof(msgLen), terrstr()); + goto _return; + } + + writeSize = taosWriteFile(pFile, pMsg, msgLen); + if (msgLen != writeSize) { + taosUnLockFile(pFile); + taosPrintLog(flags, level, dflag, "failed to write file:%s,%p wlen:%" PRId64 " tlen:%" PRId64 " since %s", + filepath, pFile, writeSize, msgLen, terrstr()); + goto _return; + } + + taosUnLockFile(pFile); + } + +_return: + + if (pFile) taosCloseFile(&pFile); + + terrno = TAOS_SYSTEM_ERROR(errno); + taosPrintLog(flags, level, dflag, "crash signal is %d", signum); + +#ifdef _TD_DARWIN_64 + taosPrintTrace(flags, level, dflag, 4); +#elif !defined(WINDOWS) + taosPrintLog(flags, level, dflag, "sender PID:%d cmdline:%s", ((siginfo_t *)sigInfo)->si_pid, + taosGetCmdlineByPID(((siginfo_t *)sigInfo)->si_pid)); + taosPrintTrace(flags, level, dflag, 3); +#else + taosPrintTrace(flags, level, dflag, 8); +#endif + + taosMemoryFree(pMsg); +} + +void taosReadCrashInfo(char* filepath, char** pMsg, int64_t* pMsgLen, TdFilePtr* pFd) { + const char *flags = "UTL FATAL "; + ELogLevel level = DEBUG_FATAL; + int32_t dflag = 255; + TdFilePtr pFile = NULL; + bool truncateFile = false; + char* buf = NULL; + + if (NULL == *pFd) { + int64_t filesize = 0; + if (taosStatFile(filepath, &filesize, NULL) < 0) { + if (ENOENT == errno) { + return; + } + + terrno = TAOS_SYSTEM_ERROR(errno); + taosPrintLog(flags, level, dflag, "failed to stat file:%s since %s", filepath, terrstr()); + return; + } + + if (filesize <= 0) { + return; + } + + pFile = taosOpenFile(filepath, TD_FILE_READ|TD_FILE_WRITE); + if (pFile == NULL) { + if (ENOENT == errno) { + return; + } + + terrno = TAOS_SYSTEM_ERROR(errno); + taosPrintLog(flags, level, dflag, "failed to open file:%s since %s", filepath, terrstr()); + return; + } + + taosLockFile(pFile); + } else { + pFile = *pFd; + } + + int64_t msgLen = 0; + int64_t readSize = taosReadFile(pFile, &msgLen, sizeof(msgLen)); + if (sizeof(msgLen) != readSize) { + truncateFile = true; + if (readSize < 0) { + taosPrintLog(flags, level, dflag, "failed to read len from file:%s,%p wlen:%" PRId64 " tlen:%lu since %s", + filepath, pFile, readSize, sizeof(msgLen), terrstr()); + } + goto _return; + } + + buf = taosMemoryMalloc(msgLen); + if (NULL == buf) { + taosPrintLog(flags, level, dflag, "failed to malloc buf, size:%" PRId64, msgLen); + goto _return; + } + + readSize = taosReadFile(pFile, buf, msgLen); + if (msgLen != readSize) { + truncateFile = true; + taosPrintLog(flags, level, dflag, "failed to read file:%s,%p wlen:%" PRId64 " tlen:%" PRId64 " since %s", + filepath, pFile, readSize, msgLen, terrstr()); + goto _return; + } + + *pMsg = buf; + *pMsgLen = msgLen; + *pFd = pFile; + + return; + +_return: + + if (truncateFile) { + taosFtruncateFile(pFile, 0); + } + taosUnLockFile(pFile); + taosCloseFile(&pFile); + taosMemoryFree(buf); + + *pMsg = NULL; + *pMsgLen = 0; + *pFd = NULL; +} + +void taosReleaseCrashLogFile(TdFilePtr pFile, bool truncateFile) { + if (truncateFile) { + taosFtruncateFile(pFile, 0); + } + + taosUnLockFile(pFile); + taosCloseFile(&pFile); +} + +#ifdef NDEBUG +bool taosAssertRelease(bool condition) { + if (condition) return false; + + const char *flags = "UTL FATAL "; + ELogLevel level = DEBUG_FATAL; + int32_t dflag = 255; // tsLogEmbedded ? 255 : uDebugFlag + + taosPrintLog(flags, level, dflag, "tAssert called in release mode, exit:%d", tsAssert); + taosPrintTrace(flags, level, dflag, 0); + + if (tsAssert) { + taosMsleep(300); + abort(); + } + + return true; +} +#endif diff --git a/source/util/src/tlosertree.c b/source/util/src/tlosertree.c index aeb9ce310b3a8802120fa58e52a3fc88a57b7eec..bf99212b78d4e75d3cbe3eb9823881168e03c129 100644 --- a/source/util/src/tlosertree.c +++ b/source/util/src/tlosertree.c @@ -20,7 +20,7 @@ // Set the initial value of the multiway merge tree. static void tMergeTreeInit(SMultiwayMergeTreeInfo* pTree) { - assert((pTree->totalSources & 0x01) == 0 && (pTree->numOfSources << 1 == pTree->totalSources)); + ASSERT((pTree->totalSources & 0x01) == 0 && (pTree->numOfSources << 1 == pTree->totalSources)); for (int32_t i = 0; i < pTree->totalSources; ++i) { if (i < pTree->numOfSources) { @@ -80,7 +80,7 @@ void tMergeTreeDestroy(SMultiwayMergeTreeInfo* pTree) { } void tMergeTreeAdjust(SMultiwayMergeTreeInfo* pTree, int32_t idx) { - assert(idx <= pTree->totalSources - 1 && idx >= pTree->numOfSources && pTree->totalSources >= 2); + ASSERT(idx <= pTree->totalSources - 1 && idx >= pTree->numOfSources && pTree->totalSources >= 2); if (pTree->totalSources == 2) { pTree->pNode[0].index = 0; @@ -115,7 +115,7 @@ void tMergeTreeAdjust(SMultiwayMergeTreeInfo* pTree, int32_t idx) { } void tMergeTreeRebuild(SMultiwayMergeTreeInfo* pTree) { - assert((pTree->totalSources & 0x1) == 0); + ASSERT((pTree->totalSources & 0x1) == 0); tMergeTreeInit(pTree); for (int32_t i = pTree->totalSources - 1; i >= pTree->numOfSources; i--) { diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index ced5b4f25e3f86e2a360cf60d964b7566b8b94a3..87b44b2d1337ab157e4499f5165abf0868069bc5 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -125,20 +125,20 @@ static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) { return pageSize * @return */ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { - assert(!pg->used && pg->pData != NULL); + ASSERT(!pg->used && pg->pData != NULL); int32_t size = pBuf->pageSize; char* t = NULL; if (pg->offset == -1 || pg->dirty) { void* payload = GET_DATA_PAYLOAD(pg); t = doCompressData(payload, pBuf->pageSize, &size, pBuf); - assert(size >= 0); + ASSERTS(size >= 0, "size is negative"); } // this page is flushed to disk for the first time if (pg->dirty) { if (pg->offset == -1) { - assert(pg->dirty == true); + ASSERTS(pg->dirty == true, "pg->dirty is false"); pg->offset = allocatePositionInFile(pBuf, size); pBuf->nextPos += size; @@ -210,7 +210,7 @@ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { int32_t ret = TSDB_CODE_SUCCESS; - assert(((int64_t)pBuf->numOfPages * pBuf->pageSize) == pBuf->totalBufSize && pBuf->numOfPages >= pBuf->inMemPages); + ASSERT(((int64_t)pBuf->numOfPages * pBuf->pageSize) == pBuf->totalBufSize && pBuf->numOfPages >= pBuf->inMemPages); if (pBuf->pFile == NULL) { if ((ret = createDiskFile(pBuf)) != TSDB_CODE_SUCCESS) { @@ -272,7 +272,7 @@ static SListNode* getEldestUnrefedPage(SDiskbasedBuf* pBuf) { SListNode* pn = NULL; while ((pn = tdListNext(&iter)) != NULL) { SPageInfo* pageInfo = *(SPageInfo**)pn->data; - assert(pageInfo->pageId >= 0 && pageInfo->pn == pn); + ASSERT(pageInfo->pageId >= 0 && pageInfo->pn == pn); if (!pageInfo->used) { // printf("%d is chosen\n", pageInfo->pageId); @@ -303,7 +303,7 @@ static char* evacOneDataPage(SDiskbasedBuf* pBuf) { tdListPopNode(pBuf->lruList, pn); SPageInfo* d = *(SPageInfo**)pn->data; - assert(d->pn == pn); + ASSERTS(d->pn == pn, "d->pn not equal pn"); d->pn = NULL; taosMemoryFreeClear(pn); @@ -353,7 +353,7 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem pPBuf->freePgList = tdListNew(POINTER_BYTES); // at least more than 2 pages must be in memory - assert(inMemBufSize >= pagesize * 2); + ASSERT(inMemBufSize >= pagesize * 2); pPBuf->lruList = tdListNew(POINTER_BYTES); @@ -402,7 +402,7 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId) { } // add to LRU list - assert(listNEles(pBuf->lruList) < pBuf->inMemPages && pBuf->inMemPages > 0); + ASSERT(listNEles(pBuf->lruList) < pBuf->inMemPages && pBuf->inMemPages > 0); lruListPushFront(pBuf->lruList, pi); // allocate buf @@ -421,11 +421,11 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId) { } void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { - assert(pBuf != NULL && id >= 0); + ASSERT(pBuf != NULL && id >= 0); pBuf->statis.getPages += 1; SPageInfo** pi = taosHashGet(pBuf->all, &id, sizeof(int32_t)); - assert(pi != NULL && *pi != NULL); + ASSERT(pi != NULL && *pi != NULL); if ((*pi)->pData != NULL) { // it is in memory // no need to update the LRU list if only one page exists @@ -435,7 +435,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { } SPageInfo** pInfo = (SPageInfo**)((*pi)->pn->data); - assert(*pInfo == *pi); + ASSERT(*pInfo == *pi); lruListMoveToFront(pBuf->lruList, (*pi)); (*pi)->used = true; @@ -444,7 +444,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { #endif return (void*)(GET_DATA_PAYLOAD(*pi)); } else { // not in memory - assert((*pi)->pData == NULL && (*pi)->pn == NULL && + ASSERT((*pi)->pData == NULL && (*pi)->pn == NULL && (((*pi)->length >= 0 && (*pi)->offset >= 0) || ((*pi)->length == -1 && (*pi)->offset == -1))); char* availablePage = NULL; @@ -482,7 +482,9 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { } void releaseBufPage(SDiskbasedBuf* pBuf, void* page) { - assert(pBuf != NULL && page != NULL); + if (ASSERTS(pBuf != NULL && page != NULL, "pBuf or page is NULL")) { + return; + } SPageInfo* ppi = getPageInfoFromPayload(page); releaseBufPageInfo(pBuf, ppi); } @@ -491,8 +493,10 @@ void releaseBufPageInfo(SDiskbasedBuf* pBuf, SPageInfo* pi) { #ifdef BUF_PAGE_DEBUG uDebug("page_releaseBufPageInfo pageId:%d, used:%d, offset:%" PRId64, pi->pageId, pi->used, pi->offset); #endif - // assert(pi->pData != NULL && pi->used == true); - assert(pi->pData != NULL); + if (ASSERTS(pi->pData != NULL, "pi->pData is NULL")) { + return; + } + pi->used = false; pBuf->statis.releasePages += 1; } diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 42b63588930712d2e132fd7e8bf860aef3622ff9..3769da6ccd60a2fe4a6a949ae9b66e8a5a3e1486 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -339,7 +339,6 @@ void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue) { STaosQueue *prev = qset->head; tqueue = qset->head->next; while (tqueue) { - assert(tqueue->qset); if (tqueue == queue) { prev->next = tqueue->next; break; diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index c6aea874704a9508c7a75568fc976226a07402e1..ffae5441aa088f2cc71710b301d5243cb3975235 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -14,6 +14,7 @@ */ #include "trbtree.h" +#include "tlog.h" static void tRBTreeRotateLeft(SRBTree *pTree, SRBTreeNode *x) { SRBTreeNode *y = x->right; @@ -258,7 +259,7 @@ static void rbtree_delete_fixup(rbtree_t *rbtree, rbnode_t *child, rbnode_t *chi child_parent->color = BLACK; return; } - assert(sibling != RBTREE_NULL); + ASSERTS(sibling != RBTREE_NULL, "sibling is NULL"); /* get a new sibling, by rotating at sibling. See which child of sibling is red */ @@ -288,11 +289,11 @@ static void rbtree_delete_fixup(rbtree_t *rbtree, rbnode_t *child, rbnode_t *chi sibling->color = child_parent->color; child_parent->color = BLACK; if (child_parent->right == child) { - assert(sibling->left->color == RED); + ASSERTS(sibling->left->color == RED, "slibing->left->color=%d not equal RED", sibling->left->color); sibling->left->color = BLACK; rbtree_rotate_right(rbtree, child_parent); } else { - assert(sibling->right->color == RED); + ASSERTS(sibling->right->color == RED, "slibing->right->color=%d not equal RED", sibling->right->color); sibling->right->color = BLACK; rbtree_rotate_left(rbtree, child_parent); } @@ -315,18 +316,18 @@ static void swap_np(rbnode_t **x, rbnode_t **y) { /** Update parent pointers of child trees of 'parent' */ static void change_parent_ptr(rbtree_t *rbtree, rbnode_t *parent, rbnode_t *old, rbnode_t *new) { if (parent == RBTREE_NULL) { - assert(rbtree->root == old); + ASSERTS(rbtree->root == old, "root not equal old"); if (rbtree->root == old) rbtree->root = new; return; } - assert(parent->left == old || parent->right == old || parent->left == new || parent->right == new); + ASSERT(parent->left == old || parent->right == old || parent->left == new || parent->right == new); if (parent->left == old) parent->left = new; if (parent->right == old) parent->right = new; } /** Update parent pointer of a node 'child' */ static void change_child_ptr(rbtree_t *rbtree, rbnode_t *child, rbnode_t *old, rbnode_t *new) { if (child == RBTREE_NULL) return; - assert(child->parent == old || child->parent == new); + ASSERT(child->parent == old || child->parent == new); if (child->parent == old) child->parent = new; } @@ -371,7 +372,7 @@ rbnode_t *rbtree_delete(rbtree_t *rbtree, void *key) { /* now delete to_delete (which is at the location where the smright previously was) */ } - assert(to_delete->left == RBTREE_NULL || to_delete->right == RBTREE_NULL); + ASSERT(to_delete->left == RBTREE_NULL || to_delete->right == RBTREE_NULL); if (to_delete->left != RBTREE_NULL) child = to_delete->left; diff --git a/source/util/src/tref.c b/source/util/src/tref.c index 75a16b0ad706668edc1ad2567aa9393a06fc685d..e70e12b37b4366d60bbb87c3eee33fc08bcced73 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -466,7 +466,7 @@ static void taosLockList(int64_t *lockedBy) { static void taosUnlockList(int64_t *lockedBy) { int64_t tid = taosGetSelfPthreadId(); if (atomic_val_compare_exchange_64(lockedBy, tid, 0) != tid) { - assert(false); + ASSERTS(false, "atomic_val_compare_exchange_64 tid failed"); } } diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 51d0a59d001b12cd64775cf43b20f83ee0f1a638..7e99d6a35cf2eafeb8205e768152b5fb0deb0bd7 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -159,8 +159,7 @@ static void lockTimerList(timer_list_t* list) { static void unlockTimerList(timer_list_t* list) { int64_t tid = taosGetSelfPthreadId(); if (atomic_val_compare_exchange_64(&(list->lockedBy), tid, 0) != tid) { - assert(false); - tmrError("%" PRId64 " trying to unlock a timer list not locked by current thread.", tid); + ASSERTS(false, "%" PRId64 " trying to unlock a timer list not locked by current thread.", tid); } } @@ -506,7 +505,7 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* han } } - assert(timer->refCount == 1); + ASSERTS(timer->refCount == 1, "timer refCount=%d not expected 1", timer->refCount); memset(timer, 0, sizeof(*timer)); *pTmrId = (tmr_h)doStartTimer(timer, fp, mseconds, param, ctrl); diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index addb9f55ba9760f8f5d6f915836ca22260f67333..e94f94a00dcdd3b625b9bf705bf19a5febae43a9 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "tutil.h" +#include "tlog.h" void *tmemmem(const char *haystack, int32_t hlen, const char *needle, int32_t nlen) { const char *limit; @@ -117,7 +118,7 @@ char **strsplit(char *z, const char *delim, int32_t *num) { if ((*num) >= size) { size = (size << 1); split = taosMemoryRealloc(split, POINTER_BYTES * size); - assert(NULL != split); + ASSERTS(NULL != split, "realloc memory failed. size=%d", POINTER_BYTES * size); } } @@ -144,11 +145,23 @@ char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote) { return NULL; } +TdUcs4* wcsnchr(const TdUcs4* haystack, TdUcs4 needle, size_t len) { + for(int32_t i = 0; i < len; ++i) { + if (haystack[i] == needle) { + return (TdUcs4*) &haystack[i]; + } + } + + return NULL; +} + char *strtolower(char *dst, const char *src) { int32_t esc = 0; char quote = 0, *p = dst, c; - assert(dst != NULL); + if (ASSERTS(dst != NULL, "dst is NULL")) { + return NULL; + } for (c = *src++; c; c = *src++) { if (esc) { @@ -175,7 +188,10 @@ char *strntolower(char *dst, const char *src, int32_t n) { int32_t esc = 0; char quote = 0, *p = dst, c; - assert(dst != NULL); + if (ASSERTS(dst != NULL, "dst is NULL")) { + return NULL; + } + if (n == 0) { *p = 0; return dst; @@ -204,7 +220,10 @@ char *strntolower(char *dst, const char *src, int32_t n) { char *strntolower_s(char *dst, const char *src, int32_t n) { char *p = dst, c; - assert(dst != NULL); + if (ASSERTS(dst != NULL, "dst is NULL")) { + return NULL; + } + if (n == 0) { return NULL; } @@ -376,3 +395,73 @@ void taosIp2String(uint32_t ip, char *str) { void taosIpPort2String(uint32_t ip, uint16_t port, char *str) { sprintf(str, "%u.%u.%u.%u:%u", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, (uint8_t)(ip >> 24), port); } + +size_t tstrncspn(const char *str, size_t size, const char *reject, size_t rsize) { + if (rsize == 0 || rsize == 1) { + char* p = strnchr(str, reject[0], size, false); + return (p == NULL)? size:(p-str); + } + + /* Use multiple small memsets to enable inlining on most targets. */ + unsigned char table[256]; + unsigned char *p = memset(table, 0, 64); + memset(p + 64, 0, 64); + memset(p + 128, 0, 64); + memset(p + 192, 0, 64); + + unsigned char *s = (unsigned char *)reject; + int32_t index = 0; + do { + p[s[index++]] = 1; + } while (index < rsize); + + s = (unsigned char*) str; + int32_t times = size >> 2; + if (times == 0) { + for(int32_t i = 0; i < size; ++i) { + if (p[s[i]]) { + return i; + } + } + + return size; + } + + index = 0; + uint32_t c0, c1, c2, c3; + for(int32_t i = 0; i < times; ++i, index += 4) { + int32_t j = index; + c0 = p[s[j]]; + c1 = p[s[j + 1]]; + c2 = p[s[j + 2]]; + c3 = p[s[j + 3]]; + + if ((c0 | c1 | c2 | c3) != 0) { + size_t count = ((i + 1) >> 2); + return (c0 | c1) != 0 ? count - c0 + 1 : count - c2 + 3; + } + } + + int32_t offset = times * 4; + for(int32_t i = offset; i < size; ++i) { + if (p[s[i]]) { + return i; + } + } + + return size; +} + +size_t twcsncspn(const TdUcs4 *wcs, size_t size, const TdUcs4 *reject, size_t rsize) { + if (rsize == 0 || rsize == 1) { + TdUcs4* p = wcsnchr(wcs, reject[0], size); + return (p == NULL)? size:(p-wcs); + } + + size_t index = 0; + while ((index < size) && (wcsnchr(reject, wcs[index], rsize) == NULL)) { + ++index; + } + + return index; +} diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 863cee9b08ced50308afc11db48a5f15c49f8162..5581931178486ba05f36640edcf7500477cbc96c 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -36,7 +36,7 @@ int32_t tQWorkerInit(SQWorkerPool *pool) { worker->pool = pool; } - uDebug("worker:%s is initialized, min:%d max:%d", pool->name, pool->min, pool->max); + uInfo("worker:%s is initialized, min:%d max:%d", pool->name, pool->min, pool->max); return 0; } @@ -51,8 +51,10 @@ void tQWorkerCleanup(SQWorkerPool *pool) { for (int32_t i = 0; i < pool->max; ++i) { SQWorker *worker = pool->workers + i; if (taosCheckPthreadValid(worker->thread)) { + uInfo("worker:%s:%d is stopping", pool->name, worker->id); taosThreadJoin(worker->thread, NULL); taosThreadClear(&worker->thread); + uInfo("worker:%s:%d is stopped", pool->name, worker->id); } } @@ -60,7 +62,7 @@ void tQWorkerCleanup(SQWorkerPool *pool) { taosCloseQset(pool->qset); taosThreadMutexDestroy(&pool->mutex); - uDebug("worker:%s is closed", pool->name); + uInfo("worker:%s is closed", pool->name); } static void *tQWorkerThreadFp(SQWorker *worker) { @@ -119,7 +121,7 @@ STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) { taosThreadAttrDestroy(&thAttr); pool->num++; - uDebug("worker:%s:%d is launched, total:%d", pool->name, worker->id, pool->num); + uInfo("worker:%s:%d is launched, total:%d", pool->name, worker->id, pool->num); } while (pool->num < pool->min); } @@ -130,7 +132,135 @@ STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) { } void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue) { - uDebug("worker:%s, queue:%p is freed", pool->name, queue); + uInfo("worker:%s, queue:%p is freed", pool->name, queue); + taosCloseQueue(queue); +} + +int32_t tAutoQWorkerInit(SAutoQWorkerPool *pool) { + pool->qset = taosOpenQset(); + pool->workers = taosArrayInit(2, sizeof(SQWorker *)); + if (pool->workers == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + (void)taosThreadMutexInit(&pool->mutex, NULL); + + uInfo("worker:%s is initialized as auto", pool->name); + return 0; +} + +void tAutoQWorkerCleanup(SAutoQWorkerPool *pool) { + int32_t size = taosArrayGetSize(pool->workers); + for (int32_t i = 0; i < size; ++i) { + SQWorker *worker = taosArrayGetP(pool->workers, i); + if (taosCheckPthreadValid(worker->thread)) { + taosQsetThreadResume(pool->qset); + } + } + + for (int32_t i = 0; i < size; ++i) { + SQWorker *worker = taosArrayGetP(pool->workers, i); + if (taosCheckPthreadValid(worker->thread)) { + uInfo("worker:%s:%d is stopping", pool->name, worker->id); + taosThreadJoin(worker->thread, NULL); + taosThreadClear(&worker->thread); + uInfo("worker:%s:%d is stopped", pool->name, worker->id); + } + taosMemoryFree(worker); + } + + taosArrayDestroy(pool->workers); + taosCloseQset(pool->qset); + taosThreadMutexDestroy(&pool->mutex); + + uInfo("worker:%s is closed", pool->name); +} + +static void *tAutoQWorkerThreadFp(SQWorker *worker) { + SAutoQWorkerPool *pool = worker->pool; + SQueueInfo qinfo = {0}; + void *msg = NULL; + int32_t code = 0; + + taosBlockSIGPIPE(); + setThreadName(pool->name); + worker->pid = taosGetSelfPthreadId(); + uInfo("worker:%s:%d is running, thread:%08" PRId64, pool->name, worker->id, worker->pid); + + while (1) { + if (taosReadQitemFromQset(pool->qset, (void **)&msg, &qinfo) == 0) { + uInfo("worker:%s:%d qset:%p, got no message and exiting, thread:%08" PRId64, pool->name, worker->id, pool->qset, + worker->pid); + break; + } + + if (qinfo.fp != NULL) { + qinfo.workerId = worker->id; + qinfo.threadNum = taosArrayGetSize(pool->workers); + (*((FItem)qinfo.fp))(&qinfo, msg); + } + + taosUpdateItemSize(qinfo.queue, 1); + } + + return NULL; +} + +STaosQueue *tAutoQWorkerAllocQueue(SAutoQWorkerPool *pool, void *ahandle, FItem fp) { + STaosQueue *queue = taosOpenQueue(); + if (queue == NULL) return NULL; + + taosThreadMutexLock(&pool->mutex); + taosSetQueueFp(queue, fp, NULL); + taosAddIntoQset(pool->qset, queue, ahandle); + + int32_t queueNum = taosGetQueueNumber(pool->qset); + int32_t curWorkerNum = taosArrayGetSize(pool->workers); + int32_t dstWorkerNum = ceil(queueNum * pool->ratio); + if (dstWorkerNum < 1) dstWorkerNum = 1; + + // spawn a thread to process queue + while (curWorkerNum < dstWorkerNum) { + SQWorker *worker = taosMemoryCalloc(1, sizeof(SQWorker)); + if (worker == NULL || taosArrayPush(pool->workers, &worker) == NULL) { + uError("worker:%s:%d failed to create", pool->name, curWorkerNum); + taosMemoryFree(worker); + taosCloseQueue(queue); + taosThreadMutexUnlock(&pool->mutex); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + worker->id = curWorkerNum; + worker->pool = pool; + + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (taosThreadCreate(&worker->thread, &thAttr, (ThreadFp)tAutoQWorkerThreadFp, worker) != 0) { + uError("worker:%s:%d failed to create thread, total:%d", pool->name, worker->id, curWorkerNum); + (void)taosArrayPop(pool->workers); + taosMemoryFree(worker); + taosCloseQueue(queue); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + taosThreadAttrDestroy(&thAttr); + uInfo("worker:%s:%d is launched, total:%d", pool->name, worker->id, (int32_t)taosArrayGetSize(pool->workers)); + + curWorkerNum++; + } + + taosThreadMutexUnlock(&pool->mutex); + uInfo("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle); + + return queue; +} + +void tAutoQWorkerFreeQueue(SAutoQWorkerPool *pool, STaosQueue *queue) { + uInfo("worker:%s, queue:%p is freed", pool->name, queue); taosCloseQueue(queue); } @@ -152,7 +282,7 @@ int32_t tWWorkerInit(SWWorkerPool *pool) { worker->pool = pool; } - uDebug("worker:%s is initialized, max:%d", pool->name, pool->max); + uInfo("worker:%s is initialized, max:%d", pool->name, pool->max); return 0; } @@ -169,17 +299,19 @@ void tWWorkerCleanup(SWWorkerPool *pool) { for (int32_t i = 0; i < pool->max; ++i) { SWWorker *worker = pool->workers + i; if (taosCheckPthreadValid(worker->thread)) { + uInfo("worker:%s:%d is stopping", pool->name, worker->id); taosThreadJoin(worker->thread, NULL); taosThreadClear(&worker->thread); taosFreeQall(worker->qall); taosCloseQset(worker->qset); + uInfo("worker:%s:%d is stopped", pool->name, worker->id); } } taosMemoryFreeClear(pool->workers); taosThreadMutexDestroy(&pool->mutex); - uDebug("worker:%s is closed", pool->name); + uInfo("worker:%s is closed", pool->name); } static void *tWWorkerThreadFp(SWWorker *worker) { @@ -235,7 +367,7 @@ STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); if (taosThreadCreate(&worker->thread, &thAttr, (ThreadFp)tWWorkerThreadFp, worker) != 0) goto _OVER; - uDebug("worker:%s:%d is launched, max:%d", pool->name, worker->id, pool->max); + uInfo("worker:%s:%d is launched, max:%d", pool->name, worker->id, pool->max); pool->nextId = (pool->nextId + 1) % pool->max; taosThreadAttrDestroy(&thAttr); @@ -259,13 +391,14 @@ _OVER: } else { while (worker->pid <= 0) taosMsleep(10); queue->threadId = worker->pid; - uInfo("worker:%s, queue:%p is allocated, ahandle:%p thread:%08" PRId64, pool->name, queue, ahandle, queue->threadId); + uInfo("worker:%s, queue:%p is allocated, ahandle:%p thread:%08" PRId64, pool->name, queue, ahandle, + queue->threadId); return queue; } } void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue) { - uDebug("worker:%s, queue:%p is freed", pool->name, queue); + uInfo("worker:%s, queue:%p is freed", pool->name, queue); taosCloseQueue(queue); } diff --git a/source/util/test/utilTests.cpp b/source/util/test/utilTests.cpp index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..c56ef348cc3754499a15a84fad3cd93603901077 100644 --- a/source/util/test/utilTests.cpp +++ b/source/util/test/utilTests.cpp @@ -0,0 +1,297 @@ +#include +#include +#include +#include + +#include "tarray.h" +#include "tcompare.h" + +namespace { +} // namespace + +TEST(utilTest, wchar_pattern_match_test) { + const TdWchar* pattern = L"%1"; + + int32_t ret = 0; + SPatternCompareInfo pInfo = PATTERN_COMPARE_INFO_INITIALIZER; + + const TdWchar* str0 = L"14"; + ret = wcsPatternMatch(reinterpret_cast(pattern), 2, reinterpret_cast(str0), wcslen(str0), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + const TdWchar* str1 = L"11"; + ret = wcsPatternMatch(reinterpret_cast(pattern), 2, reinterpret_cast(str1), wcslen(str1), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const TdWchar* str2 = L"41"; + ret = wcsPatternMatch(reinterpret_cast(pattern), 2, reinterpret_cast(str2), wcslen(str2), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const TdWchar* pattern3 = L"%_"; + const TdWchar* str3 = L"88"; + ret = wcsPatternMatch(reinterpret_cast(pattern3), 2, reinterpret_cast(str3), wcslen(str3), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const TdWchar* pattern4 = L"%___"; + const TdWchar* str4 = L"88"; + ret = wcsPatternMatch(reinterpret_cast(pattern4), 4, reinterpret_cast(str4), wcslen(str4), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + const TdWchar* pattern5 = L"%___"; + const TdWchar* str5 = L"883391"; + ret = wcsPatternMatch(reinterpret_cast(pattern5), 4, reinterpret_cast(str5), wcslen(str5), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const TdWchar* pattern6 = L"%___66"; + const TdWchar* str6 = L"88339166"; + ret = wcsPatternMatch(reinterpret_cast(pattern6), 6, reinterpret_cast(str6), wcslen(str6), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const TdWchar* pattern7 = L"%____66"; + const TdWchar* str7 = L"66166"; + ret = wcsPatternMatch(reinterpret_cast(pattern7), 7, reinterpret_cast(str7), wcslen(str7), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + const TdWchar* pattern8 = L"6%____66"; + const TdWchar* str8 = L"666166"; + ret = wcsPatternMatch(reinterpret_cast(pattern8), 8, reinterpret_cast(str8), wcslen(str8), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + const TdWchar* pattern9 = L"6\\__6"; + const TdWchar* str9 = L"6_66"; + ret = wcsPatternMatch(reinterpret_cast(pattern9), 6, reinterpret_cast(str9), wcslen(str9), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const TdWchar* pattern10 = L"%"; + const TdWchar* str10 = L""; + ret = wcsPatternMatch(reinterpret_cast(pattern10), 1, reinterpret_cast(str10), 0, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const TdWchar* pattern11 = L"china%"; + const TdWchar* str11 = L"CHI "; + ret = wcsPatternMatch(reinterpret_cast(pattern11), 6, reinterpret_cast(str11), 3, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOMATCH); + + const TdWchar* pattern12 = L"abc%"; + const TdWchar* str12 = L""; + ret = wcsPatternMatch(reinterpret_cast(pattern12), 4, reinterpret_cast(str12), 0, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOMATCH); +} + +TEST(utilTest, wchar_pattern_match_no_terminated) { + const TdWchar* pattern = L"%1 "; + + int32_t ret = 0; + SPatternCompareInfo pInfo = PATTERN_COMPARE_INFO_INITIALIZER; + + const TdWchar* str0 = L"14 "; + ret = wcsPatternMatch(reinterpret_cast(pattern), 2, reinterpret_cast(str0), 2, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + const TdWchar* str1 = L"11 "; + ret = wcsPatternMatch(reinterpret_cast(pattern), 2, reinterpret_cast(str1), 2, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const TdWchar* str2 = L"41 "; + ret = wcsPatternMatch(reinterpret_cast(pattern), 2, reinterpret_cast(str2), 2, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const TdWchar* pattern3 = L"%_ "; + const TdWchar* str3 = L"88 "; + ret = wcsPatternMatch(reinterpret_cast(pattern3), 2, reinterpret_cast(str3), 2, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const TdWchar* pattern4 = L"%___ "; + const TdWchar* str4 = L"88 "; + ret = wcsPatternMatch(reinterpret_cast(pattern4), 4, reinterpret_cast(str4), 2, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + const TdWchar* pattern5 = L"%___ "; + const TdWchar* str5 = L"883391 "; + ret = wcsPatternMatch(reinterpret_cast(pattern5), 4, reinterpret_cast(str5), 6, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const TdWchar* pattern6 = L"%___66 "; + const TdWchar* str6 = L"88339166 "; + ret = wcsPatternMatch(reinterpret_cast(pattern6), 6, reinterpret_cast(str6), 8, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const TdWchar* pattern7 = L"%____66 "; + const TdWchar* str7 = L"66166 "; + ret = wcsPatternMatch(reinterpret_cast(pattern7), 7, reinterpret_cast(str7), 5, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + const TdWchar* pattern8 = L"6%____66 "; + const TdWchar* str8 = L"666166 "; + ret = wcsPatternMatch(reinterpret_cast(pattern8), 8, reinterpret_cast(str8), 6, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + const TdWchar* pattern9 = L"6\\_6 "; + const TdWchar* str9 = L"6_6 "; + ret = wcsPatternMatch(reinterpret_cast(pattern9), 4, reinterpret_cast(str9), 3, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const TdWchar* pattern10 = L"% "; + const TdWchar* str10 = L"6_6 "; + ret = wcsPatternMatch(reinterpret_cast(pattern10), 1, reinterpret_cast(str10), 3, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); +} + +TEST(utilTest, char_pattern_match_test) { + const char* pattern = "%1"; + + int32_t ret = 0; + SPatternCompareInfo pInfo = PATTERN_COMPARE_INFO_INITIALIZER; + + const char* str0 = "14"; + ret = patternMatch(pattern, 2, str0, strlen(str0), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + const char* str1 = "11"; + ret = patternMatch(pattern, 2, str1, strlen(str1), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const char* str2 = "41"; + ret = patternMatch(pattern, 2, str2, strlen(str2), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const char* pattern3 = "%_"; + const char* str3 = "88"; + ret = patternMatch(pattern3, 2, str3, strlen(str3), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const char* pattern4 = "%___"; + const char* str4 = "88"; + ret = patternMatch(pattern4, 4, str4, strlen(str4), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + const char* pattern5 = "%___"; + const char* str5 = "883391"; + ret = patternMatch(pattern5, 4, str5, strlen(str5), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const char* pattern6 = "%___66"; + const char* str6 = "88339166"; + ret = patternMatch(pattern6, 6, str6, strlen(str6), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const char* pattern7 = "%____66"; + const char* str7 = "66166"; + ret = patternMatch(pattern7, 7, str7, strlen(str7), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + const char* pattern8 = "6%____66"; + const char* str8 = "666166"; + ret = patternMatch(pattern8, 8, str8, strlen(str8), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + const char* pattern9 = "6\\_6"; + const char* str9 = "6_6"; + ret = patternMatch(pattern9, 5, str9, strlen(str9), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const char* pattern10 = "%"; + const char* str10 = " "; + ret = patternMatch(pattern10, 1, str10, 0, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const char* pattern11 = "china%"; + const char* str11 = "abc "; + ret = patternMatch(pattern11, 6, str11, 3, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOMATCH); + + const char* pattern12 = "abc%"; + const char* str12 = NULL; + ret = patternMatch(pattern12, 4, str12, 0, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOMATCH); +} + +TEST(utilTest, char_pattern_match_no_terminated) { + const char* pattern = "%1 "; + + int32_t ret = 0; + SPatternCompareInfo pInfo = PATTERN_COMPARE_INFO_INITIALIZER; + + const char* str0 = "14"; + ret = patternMatch(pattern, 2, str0, strlen(str0), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + const char* str1 = "11"; + ret = patternMatch(pattern, 2, str1, strlen(str1), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const char* str2 = "41"; + ret = patternMatch(pattern, 2, str2, strlen(str2), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const char* pattern3 = "%_ "; + const char* str3 = "88"; + ret = patternMatch(pattern3, 2, str3, strlen(str3), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const char* pattern4 = "%___ "; + const char* str4 = "88"; + ret = patternMatch(pattern4, 4, str4, strlen(str4), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + const char* pattern5 = "%___ "; + const char* str5 = "883391"; + ret = patternMatch(pattern5, 4, str5, strlen(str5), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const char* pattern6 = "%___66 "; + const char* str6 = "88339166"; + ret = patternMatch(pattern6, 6, str6, strlen(str6), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const char* pattern7 = "%____66 "; + const char* str7 = "66166"; + ret = patternMatch(pattern7, 7, str7, strlen(str7), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + const char* pattern8 = "6%____66 "; + const char* str8 = "666166"; + ret = patternMatch(pattern8, 8, str8, strlen(str8), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + const char* pattern9 = "6\\_6 "; + const char* str9 = "6_6"; + ret = patternMatch(pattern9, 4, str9, strlen(str9), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const char* pattern10 = "% "; + const char* str10 = "6_6"; + ret = patternMatch(pattern10, 1, str10, strlen(str10), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); +} + +TEST(utilTest, tstrncspn) { + const char* p1 = "abc"; + const char* reject = "d"; + size_t v = tstrncspn(p1, strlen(p1), reject, 1); + ASSERT_EQ(v, 3); + + const char* reject1 = "a"; + v = tstrncspn(p1, strlen(p1), reject1, 1); + ASSERT_EQ(v, 0); + + const char* reject2 = "de"; + v = tstrncspn(p1, strlen(p1), reject2, 2); + ASSERT_EQ(v, 3); + + const char* p2 = "abcdefghijklmn"; + v = tstrncspn(p2, strlen(p2), reject2, 2); + ASSERT_EQ(v, 3); + + const char* reject3 = "12345n"; + v = tstrncspn(p2, strlen(p2), reject3, 6); + ASSERT_EQ(v, 13); + + const char* reject4 = ""; + v = tstrncspn(p2, strlen(p2), reject4, 0); + ASSERT_EQ(v, 14); + + const char* reject5 = "911"; + v = tstrncspn(p2, strlen(p2), reject5, 0); + ASSERT_EQ(v, 14); +} \ No newline at end of file diff --git a/tests/develop-test/2-query/table_count_scan.py b/tests/develop-test/2-query/table_count_scan.py new file mode 100644 index 0000000000000000000000000000000000000000..1ef65bfc67083719afbdbe499ab9a2d6ec3d3ead --- /dev/null +++ b/tests/develop-test/2-query/table_count_scan.py @@ -0,0 +1,238 @@ +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from math import inf + +class TDTestCase: + def caseDescription(self): + ''' + case1: [TD-21890] table count scan test case + ''' + return + + def init(self, conn, logSql, replicaVer=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), False) + self._conn = conn + + def restartTaosd(self, index=1, dbname="db"): + tdDnodes.stop(index) + tdDnodes.startWithoutSleep(index) + tdSql.execute(f"use tbl_count") + + def run(self): + print("running {}".format(__file__)) + tdSql.execute("drop database if exists tbl_count") + tdSql.execute("create database if not exists tbl_count") + tdSql.execute('use tbl_count') + tdSql.execute('create table stb1 (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(10), c9 nchar(10), c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned) TAGS(t1 int, t2 binary(10), t3 double);') + + tdSql.execute("create table tb1 using stb1 tags(1,'1',1.0);") + + tdSql.execute("create table tb2 using stb1 tags(2,'2',2.0);") + + tdSql.execute("create table tb3 using stb1 tags(3,'3',3.0);") + + tdSql.execute('insert into tb1 values (\'2021-11-11 09:00:00\',true,1,1,1,1,1,1,"123","1234",1,1,1,1);') + + tdSql.execute("insert into tb1 values ('2021-11-11 09:00:01',true,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);") + + tdSql.execute('insert into tb1 values (\'2021-11-11 09:00:02\',true,2,NULL,2,NULL,2,NULL,"234",NULL,2,NULL,2,NULL);') + + tdSql.execute('insert into tb1 values (\'2021-11-11 09:00:03\',false,NULL,3,NULL,3,NULL,3,NULL,"3456",NULL,3,NULL,3);') + + tdSql.execute('insert into tb1 values (\'2021-11-11 09:00:04\',true,4,4,4,4,4,4,"456","4567",4,4,4,4);') + + tdSql.execute('insert into tb1 values (\'2021-11-11 09:00:05\',true,127,32767,2147483647,9223372036854775807,3.402823466e+38,1.79769e+308,"567","5678",254,65534,4294967294,9223372036854775807);') + + tdSql.execute('insert into tb1 values (\'2021-11-11 09:00:06\',true,-127,-32767,-2147483647,-9223372036854775807,-3.402823466e+38,-1.79769e+308,"678","6789",0,0,0,0);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:00\',true,1,1,1,1,1,1,"111","1111",1,1,1,1);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:01\',true,2,2,2,2,2,2,"222","2222",2,2,2,2);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:02\',true,3,3,2,3,3,3,"333","3333",3,3,3,3);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:03\',false,4,4,4,4,4,4,"444","4444",4,4,4,4);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:04\',true,5,5,5,5,5,5,"555","5555",5,5,5,5);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:05\',true,6,6,6,6,6,6,"666","6666",6,6,6,6);') + + tdSql.execute('insert into tb2 values (\'2021-11-11 09:00:06\',true,7,7,7,7,7,7,"777","7777",7,7,7,7);') + + tdSql.query('select count(*),db_name, stable_name from information_schema.ins_tables group by db_name, stable_name;') + tdSql.checkRows(3) + tdSql.checkData(0, 0, 23) + tdSql.checkData(0, 1, 'information_schema') + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 3) + tdSql.checkData(1, 1, 'tbl_count') + tdSql.checkData(1, 2, 'stb1') + tdSql.checkData(2, 0, 5) + tdSql.checkData(2, 1, 'performance_schema') + tdSql.checkData(2, 2, None) + + tdSql.query('select count(1),db_name, stable_name from information_schema.ins_tables group by db_name, stable_name;') + tdSql.checkRows(3) + tdSql.checkData(0, 0, 23) + tdSql.checkData(0, 1, 'information_schema') + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 5) + tdSql.checkData(1, 1, 'performance_schema') + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 0, 3) + tdSql.checkData(2, 1, 'tbl_count') + tdSql.checkData(2, 2, 'stb1') + + tdSql.query('select count(1),db_name from information_schema.ins_tables group by db_name') + tdSql.checkRows(3) + tdSql.checkData(0, 0, 5) + tdSql.checkData(0, 1, 'performance_schema') + tdSql.checkData(1, 0, 3) + tdSql.checkData(1, 1, 'tbl_count') + tdSql.checkData(2, 0, 23) + tdSql.checkData(2, 1, 'information_schema') + + tdSql.query("select count(*) from information_schema.ins_tables where db_name='tbl_count'") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 3) + + tdSql.query('select count(*) from information_schema.ins_tables where db_name=\'tbl_count\' and stable_name="stb1";') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 3) + + tdSql.query('select count(*) from information_schema.ins_tables') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 31) + + + tdSql.execute('create table stba (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(10), c9 nchar(10), c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned) TAGS(t1 int, t2 binary(10), t3 double);') + + tdSql.execute("create table tba1 using stba tags(1,'1',1.0);") + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:00\',true, 1,1,1,1,1,1,"111","1111",1,1,1,1);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:01\',true, 2,2,2,2,2,2,"222","2222",2,2,2,2);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:02\',true, 3,3,2,3,3,3,"333","3333",3,3,3,3);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:03\',false,4,4,4,4,4,4,"444","4444",4,4,4,4);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:04\',true, 5,5,5,5,5,5,"555","5555",5,5,5,5);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:05\',true, 6,6,6,6,6,6,"666","6666",6,6,6,6);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:06\',true, 7,7,7,7,7,7,"777","7777",7,7,7,7);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:07\',true, 8,8,8,8,8,8,"888","8888",8,8,8,8);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:08\',true, 9,9,9,9,9,9,"999","9999",9,9,9,9);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:09\',true, 0,0,0,0,0,0,"000","0000",0,0,0,0);') + + self.restartTaosd(1, dbname='tbl_count') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:10\',true, 1,1,1,1,1,1,"111","1111",1,1,1,1);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:11\',true, 2,2,2,2,2,2,"222","2222",2,2,2,2);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:12\',true, 3,3,2,3,3,3,"333","3333",3,3,3,3);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:13\',false,4,4,4,4,4,4,"444","4444",4,4,4,4);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:14\',true, 5,5,5,5,5,5,"555","5555",5,5,5,5);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:15\',true, 6,6,6,6,6,6,"666","6666",6,6,6,6);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:16\',true, 7,7,7,7,7,7,"777","7777",7,7,7,7);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:17\',true, 8,8,8,8,8,8,"888","8888",8,8,8,8);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:18\',true, 9,9,9,9,9,9,"999","9999",9,9,9,9);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:19\',true, 0,0,0,0,0,0,"000","0000",0,0,0,0);') + + self.restartTaosd(1, dbname='tbl_count') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:20\',true, 1,1,1,1,1,1,"111","1111",1,1,1,1);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:21\',true, 2,2,2,2,2,2,"222","2222",2,2,2,2);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:22\',true, 3,3,2,3,3,3,"333","3333",3,3,3,3);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:23\',false,4,4,4,4,4,4,"444","4444",4,4,4,4);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:24\',true, 5,5,5,5,5,5,"555","5555",5,5,5,5);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:25\',true, 6,6,6,6,6,6,"666","6666",6,6,6,6);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:26\',true, 7,7,7,7,7,7,"777","7777",7,7,7,7);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:27\',true, 8,8,8,8,8,8,"888","8888",8,8,8,8);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:28\',true, 9,9,9,9,9,9,"999","9999",9,9,9,9);') + + tdSql.execute('insert into tba1 values (\'2021-11-11 09:00:29\',true, 0,0,0,0,0,0,"000","0000",0,0,0,0);') + + tdSql.query('select count(*),db_name, stable_name from information_schema.ins_tables group by db_name, stable_name;') + tdSql.checkRows(4) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 'tbl_count') + tdSql.checkData(0, 2, 'stba') + tdSql.checkData(1, 0, 23) + tdSql.checkData(1, 1, 'information_schema') + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 0, 3) + tdSql.checkData(2, 1, 'tbl_count') + tdSql.checkData(2, 2, 'stb1') + tdSql.checkData(3, 0, 5) + tdSql.checkData(3, 1, 'performance_schema') + tdSql.checkData(3, 2, None) + + tdSql.query('select count(1),db_name, stable_name from information_schema.ins_tables group by db_name, stable_name;') + tdSql.checkRows(4) + tdSql.checkData(0, 0, 23) + tdSql.checkData(0, 1, 'information_schema') + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 5) + tdSql.checkData(1, 1, 'performance_schema') + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 0, 1) + tdSql.checkData(2, 1, 'tbl_count') + tdSql.checkData(2, 2, 'stba') + tdSql.checkData(3, 0, 3) + tdSql.checkData(3, 1, 'tbl_count') + tdSql.checkData(3, 2, 'stb1') + + tdSql.query('select count(1),db_name from information_schema.ins_tables group by db_name') + tdSql.checkRows(3) + tdSql.checkData(0, 0, 5) + tdSql.checkData(0, 1, 'performance_schema') + tdSql.checkData(1, 0, 4) + tdSql.checkData(1, 1, 'tbl_count') + tdSql.checkData(2, 0, 23) + tdSql.checkData(2, 1, 'information_schema') + + tdSql.query("select count(*) from information_schema.ins_tables where db_name='tbl_count'") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 4) + + tdSql.query('select count(*) from information_schema.ins_tables where db_name=\'tbl_count\' and stable_name="stb1";') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 3) + + tdSql.query('select count(*) from information_schema.ins_tables') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 32) + + + tdSql.execute('drop database tbl_count') + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 8f25dfa0642754821584a6c715a99876c1ac0ea7..d77f6f170344fc3a5ac67c05250889eeb8da4cda 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -82,6 +82,7 @@ ,,y,script,./test.sh -f tsim/insert/tcp.sim ,,y,script,./test.sh -f tsim/insert/update0.sim ,,y,script,./test.sh -f tsim/insert/update1_sort_merge.sim +,,y,script,./test.sh -f tsim/insert/update2.sim ,,y,script,./test.sh -f tsim/parser/alter__for_community_version.sim ,,y,script,./test.sh -f tsim/parser/alter_column.sim ,,y,script,./test.sh -f tsim/parser/alter_stable.sim @@ -175,6 +176,7 @@ ,,y,script,./test.sh -f tsim/query/session.sim ,,y,script,./test.sh -f tsim/query/udf.sim ,,y,script,./test.sh -f tsim/query/udf_with_const.sim +,,y,script,./test.sh -f tsim/query/sys_tbname.sim ,,y,script,./test.sh -f tsim/query/groupby.sim ,,y,script,./test.sh -f tsim/qnode/basic1.sim ,,y,script,./test.sh -f tsim/snode/basic1.sim @@ -245,6 +247,8 @@ ,,y,script,./test.sh -f tsim/stream/fillIntervalPartitionBy.sim ,,y,script,./test.sh -f tsim/stream/fillIntervalPrevNext.sim ,,y,script,./test.sh -f tsim/stream/fillIntervalValue.sim +,,y,script,./test.sh -f tsim/stream/udTableAndTag0.sim +,,y,script,./test.sh -f tsim/stream/udTableAndTag1.sim ,,y,script,./test.sh -f tsim/trans/lossdata1.sim ,,y,script,./test.sh -f tsim/trans/create_db.sim ,,y,script,./test.sh -f tsim/tmq/basic1.sim @@ -408,6 +412,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShellError.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShellNetChk.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/telemetry.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/backquote_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdMonitor.py ,,n,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 ,,n,system-test,python3 ./test.py -f 0-others/udfTest.py @@ -443,6 +448,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/database_pre_suf.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/InsertFuturets.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/information_schema.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/and_or_for_byte.py @@ -616,6 +622,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/update_data.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/tb_100w_data_order.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_stable.py @@ -668,7 +676,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 +#,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -N 5 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 @@ -681,8 +689,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/create_wrong_topic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/basic5.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb0.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb0.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb2.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb3.py @@ -702,8 +710,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumerGroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqAlterSchema.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb.py @@ -718,11 +726,11 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDnodeRestart.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDnodeRestart1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStbCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py @@ -733,9 +741,10 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-19201.py +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 @@ -834,6 +843,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 3 @@ -929,6 +940,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 4 @@ -1034,9 +1047,18 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/odbc.py +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py #develop test +,,n,develop-test,python3 ./test.py -f 2-query/table_count_scan.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py diff --git a/tests/pytest/auto_crash_gen.py b/tests/pytest/auto_crash_gen.py index 9c134e6d64a343dfa3a20afdf3ceec3d6e8d6855..56629ede131606a917582a885ecaa137938adce1 100755 --- a/tests/pytest/auto_crash_gen.py +++ b/tests/pytest/auto_crash_gen.py @@ -324,7 +324,7 @@ def main(): print( " crash_gen.sh is not exists ") sys.exit(1) - git_commit = subprocess.Popen("cd %s && git log | head -n1"%crash_gen_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")[8:16] + git_commit = subprocess.Popen("cd %s && git log | head -n1"%crash_gen_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")[7:16] # crash_cmds = get_cmds() @@ -348,15 +348,12 @@ def main(): else: print('======== crash_gen run sucess and exit as expected ========') - - if status!=0 : - - try: - text = f"crash_gen instance exit status of docker [ {hostname} ] is : {msg_dict[status]}\n " + f" and git commit : {git_commit}" - send_msg(get_msg(text)) - except Exception as e: - print("exception:", e) - exit(status) + try: + text = f"crash_gen instance exit status of docker [ {hostname} ] is : {msg_dict[status]}\n " + f" and git commit : {git_commit}" + send_msg(get_msg(text)) + except Exception as e: + print("exception:", e) + exit(status) if __name__ == '__main__': diff --git a/tests/pytest/auto_crash_gen_valgrind.py b/tests/pytest/auto_crash_gen_valgrind.py index ce87fec684bdfaa4941a9ffcc01e984143aab559..22fc5a480f2facfe6eff0e00460292bdd5aa1996 100755 --- a/tests/pytest/auto_crash_gen_valgrind.py +++ b/tests/pytest/auto_crash_gen_valgrind.py @@ -357,7 +357,7 @@ def main(): print( " crash_gen.sh is not exists ") sys.exit(1) - git_commit = subprocess.Popen("cd %s && git log | head -n1"%crash_gen_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")[8:16] + git_commit = subprocess.Popen("cd %s && git log | head -n1"%crash_gen_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")[7:16] # crash_cmds = get_cmds() @@ -383,14 +383,12 @@ def main(): else: print('======== crash_gen run sucess and exit as expected ========') - if status!=0 : - - try: - text = f"crash_gen instance exit status of docker [ {hostname} ] is : {msg_dict[status]}\n " + f" and git commit : {git_commit}" - send_msg(get_msg(text)) - except Exception as e: - print("exception:", e) - exit(status) + try: + text = f"crash_gen instance exit status of docker [ {hostname} ] is : {msg_dict[status]}\n " + f" and git commit : {git_commit}" + send_msg(get_msg(text)) + except Exception as e: + print("exception:", e) + exit(status) if __name__ == '__main__': diff --git a/tests/pytest/auto_crash_gen_valgrind_cluster.py b/tests/pytest/auto_crash_gen_valgrind_cluster.py index f4afa80afe999a30d46619b7bba6905a43e820c4..547de9af479c51bac9f608eb01bc6e6b95b6fe48 100755 --- a/tests/pytest/auto_crash_gen_valgrind_cluster.py +++ b/tests/pytest/auto_crash_gen_valgrind_cluster.py @@ -357,7 +357,7 @@ def main(): print( " crash_gen.sh is not exists ") sys.exit(1) - git_commit = subprocess.Popen("cd %s && git log | head -n1"%crash_gen_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")[8:16] + git_commit = subprocess.Popen("cd %s && git log | head -n1"%crash_gen_path, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")[7:16] # crash_cmds = get_cmds() @@ -383,14 +383,12 @@ def main(): else: print('======== crash_gen run sucess and exit as expected ========') - if status!=0 : - - try: - text = f"crash_gen instance exit status of docker [ {hostname} ] is : {msg_dict[status]}\n " + f" and git commit : {git_commit}" - send_msg(get_msg(text)) - except Exception as e: - print("exception:", e) - exit(status) + try: + text = f"crash_gen instance exit status of docker [ {hostname} ] is : {msg_dict[status]}\n " + f" and git commit : {git_commit}" + send_msg(get_msg(text)) + except Exception as e: + print("exception:", e) + exit(status) if __name__ == '__main__': diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 9cfd1d368edfa319aa4e80b307cbcc09020245ff..bdf3f20e1546e029dfbf7b41b280e857fb732020 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -261,6 +261,70 @@ class TDSql: tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + # return true or false replace exit, no print out + def checkRowColNoExit(self, row, col): + caller = inspect.getframeinfo(inspect.stack()[2][0]) + if row < 0: + args = (caller.filename, caller.lineno, self.sql, row) + return False + if col < 0: + args = (caller.filename, caller.lineno, self.sql, row) + return False + if row > self.queryRows: + args = (caller.filename, caller.lineno, self.sql, row, self.queryRows) + return False + if col > self.queryCols: + args = (caller.filename, caller.lineno, self.sql, col, self.queryCols) + return False + + return True + + + # return true or false replace exit, no print out + def checkDataNoExit(self, row, col, data): + if self.checkRowColNoExit(row, col) == False: + return False + if self.queryResult[row][col] != data: + if self.cursor.istype(col, "TIMESTAMP"): + # suppose user want to check nanosecond timestamp if a longer data passed + if (len(data) >= 28): + if pd.to_datetime(self.queryResult[row][col]) == pd.to_datetime(data): + return True + else: + if self.queryResult[row][col] == _parse_datetime(data): + return True + return False + + if str(self.queryResult[row][col]) == str(data): + return True + elif isinstance(data, float): + if abs(data) >= 1 and abs((self.queryResult[row][col] - data) / data) <= 0.000001: + return True + elif abs(data) < 1 and abs(self.queryResult[row][col] - data) <= 0.000001: + return True + else: + return False + else: + return False + + return True + + + # loop execute sql then sleep(waitTime) , if checkData ok break loop + def checkDataLoop(self, row, col, data, sql, loopCount, waitTime): + # loop check util checkData return true + for i in range(loopCount): + self.query(sql) + if self.checkDataNoExit(row, col, data) : + self.checkData(row, col, data) + return + time.sleep(waitTime) + + # last check + self.query(sql) + self.checkData(row, col, data) + + def getData(self, row, col): self.checkRowCol(row, col) return self.queryResult[row][col] diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 60df188d7b84c2f202b6ec11665013da143393e9..88dada44accf588357314843d2a7d09964c896c0 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -232,8 +232,8 @@ CaseCtrl gCaseCtrl = { .printCreateTblSql = true, .printQuerySql = true, .printStmtSql = true, - .printVerbose = false, - .printRes = false, + .printVerbose = true, + .printRes = true, .autoCreateTbl = false, .numericParam = false, .rowNum = 0, diff --git a/tests/script/api/dbTableRoute.c b/tests/script/api/dbTableRoute.c index 2cf721875a23c121d8dfd3131b3758f8dadd2192..bbf17c41e4a2126e5b8314840a438313cf08e2ad 100644 --- a/tests/script/api/dbTableRoute.c +++ b/tests/script/api/dbTableRoute.c @@ -26,7 +26,10 @@ #include #include "taos.h" -int rtTables = 20; +#define RT_TABLE_NUM 100 + +int rtTables = RT_TABLE_NUM; +int rtTableUs[RT_TABLE_NUM] = {0}; char hostName[128]; static void rtExecSQL(TAOS *taos, char *command) { @@ -101,6 +104,22 @@ int rtPrepare(TAOS ** p, int prefix, int suffix) { return 0; } +int32_t rtGetTimeOfDay(struct timeval *tv) { + return gettimeofday(tv, NULL); +} +static int64_t rtGetTimestampMs() { + struct timeval systemTime; + rtGetTimeOfDay(&systemTime); + return (int64_t)systemTime.tv_sec * 1000LL + (int64_t)systemTime.tv_usec/1000; +} + +static int64_t rtGetTimestampUs() { + struct timeval systemTime; + rtGetTimeOfDay(&systemTime); + return (int64_t)systemTime.tv_sec * 1000000LL + (int64_t)systemTime.tv_usec; +} + + int rtGetDbRouteInfo(TAOS * taos) { TAOS_DB_ROUTE_INFO dbInfo; int code = taos_get_db_route_info(taos, "db1", &dbInfo); @@ -126,7 +145,10 @@ int rtGetTableRouteInfo(TAOS * taos) { char sql[1024] = {0}; for (int32_t i = 0; i < rtTables; ++i) { sprintf(table, "tb%d", i); + int64_t startTs = rtGetTimestampUs(); int code = taos_get_table_vgId(taos, "db1", table, &vgId1); + int64_t endTs = rtGetTimestampUs(); + rtTableUs[i] = (int)(endTs - startTs); if (code) { rtExit("taos_get_table_vgId", taos_errstr(NULL)); } @@ -142,9 +164,61 @@ int rtGetTableRouteInfo(TAOS * taos) { } } + printf("table vgId use us:"); + + for (int32_t i = 0; i < rtTables; ++i) { + printf("%d ", rtTableUs[i]); + } + + printf("\n"); + + return 0; +} + +int rtGetTablesRouteInfo(TAOS * taos) { + char *table = {0}; + int *vgId1 = malloc(rtTables * sizeof(int)); + int vgId2 = 0; + char sql[1024] = {0}; + const char *tbs[RT_TABLE_NUM] = {0}; + + for (int32_t i = 0; i < rtTables; ++i) { + table = malloc(10); + sprintf(table, "tb%d", i); + tbs[i] = table; + } + + int64_t startTs = rtGetTimestampUs(); + int code = taos_get_tables_vgId(taos, "db1", tbs, rtTables, vgId1); + int64_t endTs = rtGetTimestampUs(); + rtTableUs[0] = (int)(endTs - startTs); + if (code) { + rtExit("taos_get_tables_vgId", taos_errstr(NULL)); + } + + for (int32_t i = 0; i < rtTables; ++i) { + sprintf(sql, "select vgroup_id from information_schema.ins_tables where table_name=\"tb%d\"", i); + + rtFetchVgId(taos, sql, &vgId2); + if (vgId1[i] != vgId2) { + fprintf(stderr, "!!!! table tb%d vgId mis-match, vgId(api):%d, vgId(sys):%d\n", i, vgId1[i], vgId2); + exit(1); + } else { + printf("table tb%d vgId %d\n", i, vgId1[i]); + } + } + + printf("tables vgId use us:%d\n", rtTableUs[0]); + + for (int32_t i = 0; i < rtTables; ++i) { + free((void*)tbs[i]); + } + free(vgId1); + return 0; } + void rtClose(TAOS * taos) { taos_close(taos); } @@ -170,6 +244,16 @@ int rtRunCase2(void) { return 0; } +int rtRunCase3(void) { + TAOS *taos = NULL; + rtPrepare(&taos, 0, 0); + rtGetTablesRouteInfo(taos); + rtClose(taos); + + return 0; +} + + int main(int argc, char *argv[]) { if (argc != 2) { printf("usage: %s server-ip\n", argv[0]); @@ -182,6 +266,7 @@ int main(int argc, char *argv[]) { rtRunCase1(); rtRunCase2(); + rtRunCase3(); int32_t l = 5; while (l) { diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 7df17b22da1c8a8c3b7cf95315a3aea417d56eae..72257227916785de3d174349d4c83d3302064295 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -39,7 +39,7 @@ python_error=`cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l` # /root/TDengine/source/libs/scalar/src/sclvector.c:1075:66: runtime error: signed integer overflow: 9223372034707292160 + 1668838476672 cannot be represented in type 'long int' # /root/TDengine/source/common/src/tdataformat.c:1876:7: runtime error: signed integer overflow: 8252423483843671206 + 2406154664059062870 cannot be represented in type 'long int' -runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type"| grep -v "signed integer overflow" | wc -l` +runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type"| grep -v "signed integer overflow" |grep -v "strerror.c"| grep -v "asan_malloc_linux.cc" |wc -l` echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m" echo -e "\033[44;32;1m"asan memory_leak: $memory_leak"\033[0m" diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 662c4a1a6c1fb5dba105f4996203885f77e61247..217bd66ef61d937a575145e36c645db37b20f046 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -134,6 +134,7 @@ echo "mDebugFlag 143" >> $TAOS_CFG echo "wDebugFlag 143" >> $TAOS_CFG echo "sDebugFlag 143" >> $TAOS_CFG echo "tsdbDebugFlag 143" >> $TAOS_CFG +echo "tdbDebugFlag 143" >> $TAOS_CFG echo "tqDebugFlag 143" >> $TAOS_CFG echo "fsDebugFlag 143" >> $TAOS_CFG echo "idxDebugFlag 143" >> $TAOS_CFG diff --git a/tests/script/test.sh b/tests/script/test.sh index a7a5d34fbe98dee38e30dec3749db82cf069222d..19180382fd30de26667fef44a5ebc54d19d5b6a7 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -10,13 +10,11 @@ set +e #set -x FILE_NAME= -RELEASE=0 -ASYNC=0 VALGRIND=0 -UNIQUE=0 +TEST=0 UNAME_BIN=`which uname` OS_TYPE=`$UNAME_BIN` -while getopts "f:agvum" arg +while getopts "f:tgv" arg do case $arg in f) @@ -25,8 +23,8 @@ do v) VALGRIND=1 ;; - u) - UNIQUE=1 + t) + TEST=1 ;; g) VALGRIND=2 @@ -140,6 +138,11 @@ if [ -n "$FILE_NAME" ]; then result=$? echo "Execute result:" $result + if [ $TEST -eq 1 ]; then + echo "Exit without check asan errors" + exit 1 + fi + if [ $result -eq 0 ]; then $CODE_DIR/sh/sigint_stop_dnodes.sh $CODE_DIR/sh/checkAsan.sh diff --git a/tests/script/tmp/data.sim b/tests/script/tmp/data.sim index e3bfe23c3d0d41f78e87ba0d9c78871d9a70d6ea..e2ddd60c8c0efbb8b24bf236beac518b763ab080 100644 --- a/tests/script/tmp/data.sim +++ b/tests/script/tmp/data.sim @@ -53,17 +53,25 @@ endi return print =============== step2: create database -sql create database db vgroups 33 replica 3 +sql create database db vgroups 1 replica 3 sql use db; sql create table stb (ts timestamp, c int) tags (t int); sql create table t0 using stb tags (0); -sql insert into t0 values(now, 1); +$x = 0 +while $x < 28 + sql insert into t0 values(now, 1); + $x = $x + 1 +endw + sql select * from information_schema.ins_stables where db_name = 'db'; sql select * from information_schema.ins_tables where db_name = 'db'; sql show db.vgroups; +system sh/exec.sh -n dnode1 -s stop system sh/exec.sh -n dnode2 -s stop +system sh/exec.sh -n dnode3 -s stop +system sh/exec.sh -n dnode4 -s stop return print ======== start back diff --git a/tests/script/tsim/dnode/balance1.sim b/tests/script/tsim/dnode/balance1.sim index 2b0154c8e5fdb975ec993b959d365a04ec7a7c5d..3bb9c4f3ebd6cefab513f0c6fc35cbdc2d7aee89 100644 --- a/tests/script/tsim/dnode/balance1.sim +++ b/tests/script/tsim/dnode/balance1.sim @@ -74,10 +74,10 @@ sql insert into d2.t2 values(now+5s, 21) sql select * from information_schema.ins_dnodes print dnode1 openVnodes $data(1)[2] print dnode2 openVnodes $data(2)[2] -if $data(1)[2] != 0 then +if $data(1)[2] != 1 then return -1 endi -if $data(2)[2] != 2 then +if $data(2)[2] != 1 then return -1 endi diff --git a/tests/script/tsim/dnode/balance2.sim b/tests/script/tsim/dnode/balance2.sim index 3f5a42d4d33b1c29acc054d816dc213c573108d0..9eeb7e125158eafd27bfe31d39586bf2a116d9ee 100644 --- a/tests/script/tsim/dnode/balance2.sim +++ b/tests/script/tsim/dnode/balance2.sim @@ -161,13 +161,13 @@ print dnode1 openVnodes $data(1)[2] print dnode3 openVnodes $data(3)[2] print dnode4 openVnodes $data(4)[2] print dnode5 openVnodes $data(5)[2] -if $data(1)[2] != 2 then +if $data(1)[2] != 3 then return -1 endi if $data(3)[2] != 3 then return -1 endi -if $data(4)[2] != 4 then +if $data(4)[2] != 3 then return -1 endi if $data(5)[2] != 3 then diff --git a/tests/script/tsim/dnode/balance3.sim b/tests/script/tsim/dnode/balance3.sim index ce328f10bd6561fb4277decdb804ef235f51cb09..2fb284b46614e37ee6ac27bd13a16eb7429e9d86 100644 --- a/tests/script/tsim/dnode/balance3.sim +++ b/tests/script/tsim/dnode/balance3.sim @@ -127,10 +127,10 @@ print dnode1 openVnodes $data(1)[2] print dnode2 openVnodes $data(2)[2] print dnode3 openVnodes $data(3)[2] print dnode4 openVnodes $data(4)[2] -if $data(1)[2] != 0 then +if $data(1)[2] != 1 then return -1 endi -if $data(2)[2] != 2 then +if $data(2)[2] != 1 then return -1 endi if $data(3)[2] != 2 then @@ -228,10 +228,10 @@ print dnode1 openVnodes $data(1)[2] print dnode3 openVnodes $data(3)[2] print dnode4 openVnodes $data(4)[2] print dnode5 openVnodes $data(5)[2] -if $data(1)[2] != 1 then +if $data(1)[2] != 2 then return -1 endi -if $data(3)[2] != 3 then +if $data(3)[2] != 2 then return -1 endi if $data(4)[2] != 3 then diff --git a/tests/script/tsim/dnode/balancex.sim b/tests/script/tsim/dnode/balancex.sim index 6b16e8b5696e8279337605bea44c04e001bce531..0cfc64a954f10457ab720990c758cfe4032e900a 100644 --- a/tests/script/tsim/dnode/balancex.sim +++ b/tests/script/tsim/dnode/balancex.sim @@ -142,10 +142,10 @@ print dnode1 openVnodes $data(1)[2] print dnode2 openVnodes $data(2)[2] print dnode2 openVnodes $data(3)[2] print dnode2 openVnodes $data(4)[2] -if $data(1)[2] != 0 then +if $data(1)[2] != 1 then return -1 endi -if $data(2)[2] != 2 then +if $data(2)[2] != 1 then return -1 endi if $data(3)[2] != 2 then diff --git a/tests/script/tsim/dnode/vnode_clean.sim b/tests/script/tsim/dnode/vnode_clean.sim index 112e5f28a45c367dc3df301df01efc356185fc9c..ba1d083c68e069fdbfecbd13144f31750a81eb81 100644 --- a/tests/script/tsim/dnode/vnode_clean.sim +++ b/tests/script/tsim/dnode/vnode_clean.sim @@ -71,10 +71,10 @@ sql insert into d2.t2 values(now+5s, 21) sql select * from information_schema.ins_dnodes print dnode1 openVnodes $data(1)[2] print dnode2 openVnodes $data(2)[2] -if $data(1)[2] != 0 then +if $data(1)[2] != 1 then return -1 endi -if $data(2)[2] != 2 then +if $data(2)[2] != 1 then return -1 endi @@ -181,10 +181,10 @@ sql select * from information_schema.ins_dnodes print dnode1 openVnodes $data(1)[2] print dnode2 openVnodes $data(3)[2] print dnode2 openVnodes $data(4)[2] -if $data(1)[2] != 0 then +if $data(1)[2] != 1 then return -1 endi -if $data(3)[2] != 2 then +if $data(3)[2] != 1 then return -1 endi if $data(4)[2] != 1 then @@ -204,10 +204,10 @@ sql select * from information_schema.ins_dnodes print dnode1 openVnodes $data(1)[2] print dnode2 openVnodes $data(3)[2] print dnode2 openVnodes $data(4)[2] -if $data(1)[2] != 0 then +if $data(1)[2] != 1 then return -1 endi -if $data(3)[2] != 2 then +if $data(3)[2] != 1 then return -1 endi if $data(4)[2] != 2 then @@ -220,13 +220,13 @@ sql select * from information_schema.ins_dnodes print dnode1 openVnodes $data(1)[2] print dnode2 openVnodes $data(3)[2] print dnode2 openVnodes $data(4)[2] -if $data(1)[2] != 1 then +if $data(1)[2] != 2 then return -1 endi if $data(3)[2] != null then return -1 endi -if $data(4)[2] != 3 then +if $data(4)[2] != 2 then return -1 endi diff --git a/tests/script/tsim/insert/insert_select.sim b/tests/script/tsim/insert/insert_select.sim index e3374ee277ae982d2df476b981ca7fc58f76199d..333964b1d66f72ec663fc0f807b4e84bb2ea7a69 100644 --- a/tests/script/tsim/insert/insert_select.sim +++ b/tests/script/tsim/insert/insert_select.sim @@ -42,4 +42,24 @@ if $rows != 2 then return -1 endi +print ======== step2 +sql drop database if exists db1; +sql create database db1 vgroups 1; +sql use db1; +sql create table t1(ts timestamp, a int, b int ); +sql create table t2(ts timestamp, a int, b int ); +sql insert into t1 values(1648791211000,1,2); +sql insert into t2 (ts, b, a) select ts, a, b from t1; +sql select * from t2; +if $rows != 1 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data02 != 1 then + return -1 +endi + + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/insert/update2.sim b/tests/script/tsim/insert/update2.sim new file mode 100644 index 0000000000000000000000000000000000000000..d2b5b052678a93499157dea219571df6ed67c6b7 --- /dev/null +++ b/tests/script/tsim/insert/update2.sim @@ -0,0 +1,221 @@ +################################################################################################ +# migrate from 2.0 insert_update2.sim +################################################################################################ + +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/exec.sh -n dnode1 -s start + +sleep 2000 +sql connect + +### 4096*0.8 - 1 = 3275 +$rowSuperBlk = 3275 +### 4096 - 3275 -1 - 1 = 819 +$rowSubBlk = 819 +$ts0 = 1672372800000 +$ts1 = 1672372900000 +$ts2 = 1672686800000 + +$i = 0 +$db = db0 +$stb1 = stb1 +$tb1 = tb1 +$stb2 = stb2 +$tb2 = tb2 + +print ====== create database +sql drop database if exists $db +sql create database $db keep 1000 duration 10 +print ====== create tables +sql use $db +sql create table $stb1 (ts timestamp, c1 bigint, c2 bigint, c3 bigint) tags(t1 int) +sql create table $stb2 (ts timestamp, c1 bigint, c2 bigint, c3 bigint, c4 bigint, c5 bigint, c6 bigint, c7 bigint, c8 bigint, c9 bigint, c10 bigint, c11 bigint, c12 bigint, c13 bigint, c14 bigint, c15 bigint, c16 bigint, c17 bigint, c18 bigint, c19 bigint, c20 bigint, c21 bigint, c22 bigint, c23 bigint, c24 bigint, c25 bigint, c26 bigint, c27 bigint, c28 bigint, c29 bigint, c30 bigint) tags(t1 int) +sql create table $tb1 using $stb1 tags(1) +sql create table $tb2 using $stb2 tags(2) +print ====== tables created + + +print ========== step 1: merge dataRow in mem + +$i = 0 +while $i < $rowSuperBlk + $xs = $i * 10 + $ts = $ts2 + $xs + sql insert into $tb1 (ts,c1) values ( $ts , $i ) + $i = $i + 1 +endw + +sql insert into $tb1 values ( $ts0 , 1,NULL,0) +sql insert into $tb1 (ts,c2,c3) values ( $ts0 , 1,1) + +sql select * from $tb1 where ts = $ts0 +if $rows != 1 then + return -1 +endi +if $data01 != 1 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data03 != 1 then + return -1 +endi + +print ========== step 2: merge kvRow in mem +$i = 0 +while $i < $rowSuperBlk + $xs = $i * 10 + $ts = $ts2 + $xs + sql insert into $tb2 (ts,c1) values ( $ts , $i ) + $i = $i + 1 +endw + +sql insert into $tb2 (ts,c3,c8,c10) values ( $ts0 , 1,NULL,0) +sql insert into $tb2 (ts,c8,c10) values ( $ts0 , 1,1) + +sql select ts,c1,c3,c8,c10 from $tb2 where ts = $ts0 +if $rows != 1 then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data03 != 1 then + return -1 +endi +if $data04 != 1 then + return -1 +endi + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 2000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sleep 2000 + +print ========== step 3: merge dataRow in file +sql insert into $tb1 (ts,c1) values ( $ts0 , 2) +print ========== step 4: merge kvRow in file +sql insert into $tb2 (ts,c3) values ( $ts0 , 2) + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 2000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sleep 2000 + +sql select * from $tb1 where ts = $ts0 +if $rows != 1 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data03 != 1 then + return -1 +endi + +sql select ts,c1,c3,c8,c10 from $tb2 where ts = $ts0 +if $rows != 1 then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data02 != 2 then + return -1 +endi +if $data03 != 1 then + return -1 +endi +if $data04 != 1 then + return -1 +endi + + +print ========== step 5: merge dataRow in file/mem +$i = 0 +while $i < $rowSubBlk + $xs = $i * 1 + $ts = $ts1 + $xs + sql insert into $tb1 (ts,c1) values ( $ts , $i ) + $i = $i + 1 +endw +print ========== step 6: merge kvRow in file/mem +$i = 0 +while $i < $rowSubBlk + $xs = $i * 1 + $ts = $ts1 + $xs + sql insert into $tb2 (ts,c1) values ( $ts , $i ) + $i = $i + 1 +endw + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 2000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sleep 2000 + +sql insert into $tb1 (ts,c3) values ( $ts0 , 3) +sql insert into $tb2 (ts,c3) values ( $ts0 , 3) +$tsN = $ts0 + 1 +sql insert into $tb1 (ts,c1,c3) values ( $tsN , 1,0) +sql insert into $tb2 (ts,c3,c8) values ( $tsN , 100,200) +$tsN = $ts0 + 2 +sql insert into $tb1 (ts,c1,c3) values ( $tsN , 1,0) +sql insert into $tb2 (ts,c3,c8) values ( $tsN , 100,200) + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 2000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sleep 2000 + + +sql select * from $tb1 where ts = $ts0 +if $rows != 1 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data03 != 3 then + return -1 +endi + +sql select ts,c1,c3,c8,c10 from $tb2 where ts = $ts0 +if $rows != 1 then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data02 != 3 then + return -1 +endi +if $data03 != 1 then + return -1 +endi + +if $data04 != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/alter1.sim b/tests/script/tsim/parser/alter1.sim index 369419dcd9cd91688f39c27dbd54c33ee0699ae8..cf9da46fba276911627f6b5d025176084b473854 100644 --- a/tests/script/tsim/parser/alter1.sim +++ b/tests/script/tsim/parser/alter1.sim @@ -88,6 +88,7 @@ sql insert into car1 values (now, 1, 1,1 ) (now +1s, 2,2,2) car2 values (now, 1, sql select c1+speed from stb where c1 > 0 if $rows != 3 then + print $rows , expect 3 return -1 endi diff --git a/tests/script/tsim/parser/regressiontest.sim b/tests/script/tsim/parser/regressiontest.sim new file mode 100644 index 0000000000000000000000000000000000000000..1b127155cbed8f9eaf26898c27bd87e46ed1d7e4 --- /dev/null +++ b/tests/script/tsim/parser/regressiontest.sim @@ -0,0 +1,66 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +$dbPrefix = reg_db +$tb = tb +$rowNum = 8200 + +$ts0 = 1537146000000 +$delta = 100 +print ========== reg.sim +$i = 0 +$db = $dbPrefix . $i + +sql drop database if exists $db -x step1 +step1: +sql create database $db vgroups 1; + +sql use $db +sql create table $tb (ts timestamp, c1 int) + +$i = 0 +$ts = $ts0 + +$x = 0 +while $x < $rowNum +$xs = $x * $delta +$ts = $ts0 + $xs +sql insert into $tb values ( $ts , $x ) +$x = $x + 1 +endw + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sql connect + +sql use $db +sql delete from $tb where ts=1537146000000 +sql delete from $tb where ts=1537146409500 + +print =========================> TS-2410 +sql select * from $tb limit 20 offset 4090 +print $data00 +print $data10 +print $data20 +print $data30 +print $data40 +print $data50 +print $data60 +print $data70 +print $data80 +print $data90 + +if $data40 != @18-09-17 09:06:49.600@ then + return -1 +endi + +sql select * from $tb order by ts desc; +if $rows != 8198 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/sys_tbname.sim b/tests/script/tsim/query/sys_tbname.sim new file mode 100644 index 0000000000000000000000000000000000000000..9b16d982026b1648da938d9f74b4c0ee52f989c4 --- /dev/null +++ b/tests/script/tsim/query/sys_tbname.sim @@ -0,0 +1,89 @@ + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database sys_tbname; +sql use sys_tbname; +sql create stable st(ts timestamp, f int) tags(t int); +sql create table ct1 using st tags(1); +sql create table ct2 using st tags(2); + +sql create table t (ts timestamp, f int); +sql insert into t values(now, 1)(now+1s, 2); + + +sql create table t2 (ts timestamp, f1 int, f2 int); +sql insert into t2 values(now, 0, 0)(now+1s, 1, 1); + +sql select tbname from information_schema.ins_databases; +print $rows $data00 +if $rows != 3 then + return -1 +endi +if $data00 != @ins_databases@ then + return -1 +endi +sql select distinct tbname from information_schema.ins_databases; +print $rows $data00 +if $rows != 1 then + return -1 +endi +if $data00 != @ins_databases@ then + return -1 +endi + +sql select tbname from information_schema.ins_stables; +print $rows $data00 +if $rows != 1 then + return -1 +endi +if $data00 != @ins_stables@ then + return -1 +endi +sql select distinct tbname from information_schema.ins_stables; +print $rows $data00 +if $rows != 1 then + return -1 +endi +if $data00 != @ins_stables@ then + return -1 +endi + +sql select tbname from information_schema.ins_tables; +print $rows $data00 +if $rows != 33 then + return -1 +endi +if $data00 != @ins_tables@ then + return -1 +endi + +sql select distinct tbname from information_schema.ins_tables; +print $rows $data00 +if $rows != 1 then + return -1 +endi +if $data00 != @ins_tables@ then + return -1 +endi + +sql select tbname from information_schema.ins_tags; +print $rows $data00 +if $rows != 2 then + return -1 +endi +if $data00 != @ins_tags@ then + return -1 +endi +sql select distinct tbname from information_schema.ins_tags; +print $rows $data00 +if $rows != 1 then + return -1 +endi +if $data00 != @ins_tags@ then + return -1 +endi + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/sma/rsmaCreateInsertQuery.sim b/tests/script/tsim/sma/rsmaCreateInsertQuery.sim index 04cf09715c7def3d73800e9a5cbc6f2e0fc78ddd..508e6f88c14f09d4e55c8bd3d40ea78426e5fa43 100644 --- a/tests/script/tsim/sma/rsmaCreateInsertQuery.sim +++ b/tests/script/tsim/sma/rsmaCreateInsertQuery.sim @@ -82,8 +82,8 @@ endi #=================================================================== - #==================== reboot to trigger commit data to file +sql flush database d0; system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/tsim/sma/rsmaPersistenceRecovery.sim b/tests/script/tsim/sma/rsmaPersistenceRecovery.sim index faff48b61c1216e744d349a301f39fabd6dc578a..4117a2403dcc7156a8b9ea3adb840fe25b26b376 100644 --- a/tests/script/tsim/sma/rsmaPersistenceRecovery.sim +++ b/tests/script/tsim/sma/rsmaPersistenceRecovery.sim @@ -85,6 +85,7 @@ endi #==================== reboot to trigger commit data to file +sql flush database d0; system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/tsim/stream/basic3.sim b/tests/script/tsim/stream/basic3.sim index 48fb860a72272077051f16d0df7723a46706e0b3..e598919e34b60d22478de67cce5df9c0c976216d 100644 --- a/tests/script/tsim/stream/basic3.sim +++ b/tests/script/tsim/stream/basic3.sim @@ -1,7 +1,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c debugflag -v 131 -system sh/exec.sh -n dnode1 -s start -v +system sh/cfg.sh -n dnode1 -c keepColumnName -v 1 +system sh/exec.sh -n dnode1 -s start sleep 5000 @@ -9,7 +10,7 @@ sql connect print ========== interval\session\state window -sql CREATE DATABASE test1 BUFFER 96 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 STRICT 'off' WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0; +sql CREATE DATABASE test1 VGROUPS 2; sql use test1; sql CREATE STABLE st (time TIMESTAMP, ca DOUBLE, cb DOUBLE, cc int) TAGS (ta VARCHAR(10) ); @@ -29,6 +30,76 @@ sql create stream streamd4 into streamt4 as select tbname, _wstart,_wend, count( sql create stream streamd5 into streamt5 as select tbname, _wstart,_wend, count(*), max(ca), min(cb) from st where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by tbname state_window(cc); sql create stream streamd6 into streamt6 as select ca, _wstart,_wend, count(*), max(ca), min(cb) from t1 where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by ca state_window(cc); + +sql alter local 'keepColumnName' '1' + +sql CREATE STABLE `meters_test_data` (`ts` TIMESTAMP, `close` FLOAT, `parttime` TIMESTAMP, `parttime_str` VARCHAR(32)) TAGS (`id` VARCHAR(32)); + +sql_error create stream realtime_meters fill_history 1 into realtime_meters as select last(parttime),first(close),last(close) from meters_test_data partition by tbname state_window(parttime_str); +sql_error create stream streamd7 into streamt7 as select _wstart, _wend, count(*), first(ca), last(ca) from t1 interval(10s); +sql_error create stream streamd71 into streamt71 as select _wstart, _wend, count(*) as ca, first(ca), last(ca) as c2 from t1 interval(10s); + +sql create stream streamd8 into streamt8 as select _wstart, _wend, count(*), first(ca) as c1, last(ca) as c2 from t1 interval(10s); +sql desc streamt8; + +if $rows == 0 then + return -1 +endi + +sql create stream streamd9 into streamt9 as select _wstart, _wend, count(*), first(ca) as c1, last(ca) from t1 interval(10s); +sql desc streamt9; + +if $rows == 0 then + return -1 +endi + +sql_error create stream streamd11 into streamd11 as select _wstart, _wend, count(*), last(ca), last(ca) from t1 interval(10s); + +sql alter local 'keepColumnName' '0' + +sql create stream realtime_meters fill_history 1 into realtime_meters as select last(parttime),first(close),last(close) from meters_test_data partition by tbname state_window(parttime_str); + +sql desc realtime_meters; + +if $rows == 0 then + return -1 +endi + +sql create stream streamd7 into streamt7 as select _wstart, _wend, count(*), first(ca), last(ca) from t1 interval(10s); + +sql desc streamt7; + +if $rows == 0 then + return -1 +endi + +sql create stream streamd71 into streamt71 as select _wstart, _wend, count(*) as ca, first(ca), last(ca) as c2 from t1 interval(10s); + +sql desc streamt71; + +if $rows == 0 then + return -1 +endi + +sql create stream streamd10 into streamd10 as select _wstart, _wend, count(*), first(ca), last(cb) as c2 from t1 interval(10s); + +sql desc streamd10; + +if $rows == 0 then + return -1 +endi + +sql_error create stream streamd11 into streamd11 as select _wstart, _wend, count(*), last(ca), last(ca) from t1 interval(10s); + + +sql create stream streamd12 into streamd12 as select _wstart, _wend, count(*), last(ca), last(cb) as c2 from t1 interval(10s); + +sql desc streamd12; + +if $rows == 0 then + return -1 +endi + sleep 3000 sql drop stream if exists streamd1; diff --git a/tests/script/tsim/stream/checkStreamSTable.sim b/tests/script/tsim/stream/checkStreamSTable.sim new file mode 100644 index 0000000000000000000000000000000000000000..2ed6958196e7defc114378f06f889d5b00b032a1 --- /dev/null +++ b/tests/script/tsim/stream/checkStreamSTable.sim @@ -0,0 +1,310 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 + +print ===== step1 + +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print ===== step2 + +sql create database result vgroups 1; + +sql create database test vgroups 4; +sql use test; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stable result.streamt0(ts timestamp,a int,b int) tags(ta int,tb int,tc int); + +sql create stream streams0 trigger at_once into result.streamt0 as select _wstart, count(*) c1, max(a) c2 from st partition by tbname interval(10s); +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,2,2,3); + +$loop_count = 0 + +sql select _wstart, count(*) c1, max(a) c2 from st partition by tbname interval(10s); +print $data00, $data01, $data02 +print $data10, $data11, $data12 +print $data20, $data21, $data22 + +loop0: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result.streamt0 order by ta; + +if $rows != 2 then + print =====rows=$rows + print $data00, $data01, $data02 + print $data10, $data11, $data12 + print $data20, $data21, $data22 + goto loop0 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop0 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop0 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop0 +endi + +if $data12 != 2 then + print =====data12=$data12 + goto loop0 +endi + +print ===== step3 + +sql create database result1 vgroups 1; + +sql create database test1 vgroups 4; +sql use test1; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stable result1.streamt1(ts timestamp,a int,b int,c int) tags(ta bigint unsigned,tb int,tc int); + +sql create stream streams1 trigger at_once into result1.streamt1(ts,c,a,b) as select _wstart, count(*) c1, max(a),min(b) c2 from st partition by tbname interval(10s); +sql insert into t1 values(1648791213000,10,20,30); +sql insert into t2 values(1648791213000,40,50,60); + +$loop_count = 0 + +sql select _wstart, count(*) c1, max(a),min(b) c2 from st partition by tbname interval(10s); +print $data00, $data01, $data02, $data03 +print $data10, $data11, $data12, $data13 +print $data20, $data21, $data22, $data23 + +loop1: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result1.streamt1 order by ta; + +if $rows != 2 then + print =====rows=$rows + print $data00, $data01, $data02, $data03 + print $data10, $data11, $data12, $data13 + print $data20, $data21, $data22, $data23 + goto loop1 +endi + +if $data01 != 10 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != 20 then + print =====data02=$data02 + goto loop1 +endi + +if $data03 != 1 then + print =====data03=$data03 + goto loop1 +endi + +if $data11 != 40 then + print =====data11=$data11 + goto loop1 +endi + +if $data12 != 50 then + print =====data12=$data12 + goto loop1 +endi + +if $data13 != 1 then + print =====data13=$data13 + goto loop1 +endi + + +print ===== step4 + +sql create database result2 vgroups 1; + +sql create database test2 vgroups 4; +sql use test2; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stable result2.streamt2(ts timestamp, a int , b int) tags(ta varchar(20)); + +# tag dest 1, source 2 +##sql_error create stream streams2 trigger at_once into result2.streamt2 TAGS(aa varchar(100), ta int) as select _wstart, count(*) c1, max(a) from st partition by tbname as aa, ta interval(10s); + +# column dest 3, source 4 +sql_error create stream streams2 trigger at_once into result2.streamt2 as select _wstart, count(*) c1, max(a), max(b) from st partition by tbname interval(10s); + +# column dest 3, source 4 +sql_error create stream streams2 trigger at_once into result2.streamt2(ts, a, b) as select _wstart, count(*) c1, max(a), max(b) from st partition by tbname interval(10s); + +# column dest 3, source 2 +sql_error create stream streams2 trigger at_once into result2.streamt2 as select _wstart, count(*) c1 from st partition by tbname interval(10s); + +# column dest 3, source 2 +sql create stream streams2 trigger at_once into result2.streamt2(ts, a) as select _wstart, count(*) c1 from st partition by tbname interval(10s); + + +print ===== step5 + +sql create database result3 vgroups 1; + +sql create database test3 vgroups 4; +sql use test3; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,2,3); +sql create table t2 using st tags(4,5,6); + +sql create stable result3.streamt3(ts timestamp,a int,b int,c int, d int) tags(ta int,tb int,tc int); + +sql create stream streams3 trigger at_once into result3.streamt3(ts,c,a,b) as select _wstart, count(*) c1, max(a),min(b) c2 from st interval(10s); + +sql insert into t1 values(1648791213000,10,20,30); +sql insert into t2 values(1648791213000,40,50,60); + +$loop_count = 0 + +sql select _wstart, count(*) c1, max(a),min(b) c2 from st interval(10s); +print $data00, $data01, $data02, $data03, $data04 +print $data10, $data11, $data12, $data13, $data14 +print $data20, $data21, $data22, $data23, $data24 + +loop2: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result3.streamt3; + +if $rows != 1 then + print =====rows=$rows + print $data00, $data01, $data02, $data03 + print $data10, $data11, $data12, $data13 + print $data20, $data21, $data22, $data23 + goto loop2 +endi + +if $data01 != 40 then + print =====data01=$data01 + goto loop2 +endi + +if $data02 != 20 then + print =====data02=$data02 + goto loop2 +endi + +if $data03 != 2 then + print =====data03=$data03 + goto loop2 +endi + +if $data04 != NULL then + print =====data04=$data04 + goto loop2 +endi + +print ===== step6 + +sql create database result4 vgroups 1; + +sql create database test4 vgroups 4; +sql use test4; + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,2,3); +sql create table t2 using st tags(4,5,6); + +sql create stable result4.streamt4(ts timestamp,a int,b int,c int, d int) tags(ta int,tb int,tc int); + +sql create stream streams4 trigger at_once into result4.streamt4(ts,c,a,b) tags(tg2 int, tg3 varchar(100), tg1 bigint) subtable(concat("tbl-", tg1)) as select _wstart, count(*) c1, max(a),min(b) c2 from st partition by ta+1 as tg1, cast(tb as bigint) as tg2, tc as tg3 interval(10s); + +sql insert into t1 values(1648791213000,10,20,30); +sql insert into t2 values(1648791213000,40,50,60); + +$loop_count = 0 + +sql select _wstart, count(*) c1, max(a),min(b) c2 from st interval(10s); +print $data00, $data01, $data02, $data03 +print $data10, $data11, $data12, $data13 +print $data20, $data21, $data22, $data23 + +loop2: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result4.streamt4; + +if $rows != 2 then + print =====rows=$rows + print $data00, $data01, $data02, $data03 + print $data10, $data11, $data12, $data13 + print $data20, $data21, $data22, $data23 + goto loop2 +endi + +if $data01 != 40 then + print =====data01=$data01 + goto loop2 +endi + +if $data02 != 20 then + print =====data02=$data02 + goto loop2 +endi + +if $data03 != 2 then + print =====data03=$data03 + goto loop2 +endi + +if $data04 != NULL then + print =====data04=$data04 + goto loop2 +endi + +print ======over + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/sliding.sim b/tests/script/tsim/stream/sliding.sim index c9a1ddd922b4c6d1194b87902ef6bdf22826165e..8287274cd2a008f264365ea06d049428ae645d2f 100644 --- a/tests/script/tsim/stream/sliding.sim +++ b/tests/script/tsim/stream/sliding.sim @@ -672,6 +672,123 @@ if $data61 != 1 then goto loop5 endi +print step 8 + +sql drop stream IF EXISTS streams4; +sql drop database IF EXISTS test4; + +sql create database test4 vgroups 6; +sql use test4; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams4 trigger at_once into streamt4 as select _wstart as ts, count(*),min(a) c1 from st interval(10s) sliding(5s); + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791243000,2,1,1,1.0); + +sql insert into t2 values(1648791273000,3,1,1,1.0); +sql insert into t2 values(1648791313000,4,1,1,1.0); + +$loop_count = 0 + +loop6: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt4 order by 1; + +# row 0 +if $rows != 8 then + print ====loop6=rows=$rows + goto loop6 +endi + +if $data01 != 1 then + print ====loop6=data01=$data01 + goto loop6 +endi + +if $data02 != 1 then + print ====loop6=data02=$data02 + return -1 +endi + +if $data11 != 1 then + print ====loop6=data11=$data11 + goto loop6 +endi + +if $data12 != 1 then + print ====loop6=data12=$data12 + return -1 +endi + +if $data21 != 1 then + print ====loop6=data21=$data21 + goto loop6 +endi + +if $data22 != 2 then + print ====loop6=data22=$data22 + return -1 +endi + +if $data31 != 1 then + print ====loop6=data31=$data31 + goto loop6 +endi + +if $data32 != 2 then + print ====loop6=data32=$data32 + return -1 +endi + +if $data41 != 1 then + print ====loop6=data41=$data41 + goto loop6 +endi + +if $data42 != 3 then + print ====loop6=data42=$data42 + return -1 +endi + +if $data51 != 1 then + print ====loop6=data51=$data51 + goto loop6 +endi + +if $data52 != 3 then + print ====loop6=data52=$data52 + return -1 +endi + +if $data61 != 1 then + print ====loop6=data61=$data61 + return -1 +endi + +if $data62 != 4 then + print ====loop6=data62=$data62 + return -1 +endi + +if $data71 != 1 then + print ====loop6=data71=$data71 + return -1 +endi + +if $data72 != 4 then + print ====loop6=data72=$data72 + return -1 +endi + $loop_all = $loop_all + 1 print ============loop_all=$loop_all diff --git a/tests/script/tsim/stream/udTableAndTag0.sim b/tests/script/tsim/stream/udTableAndTag0.sim new file mode 100644 index 0000000000000000000000000000000000000000..86feca19188b6fa1034830755de809cef9944490 --- /dev/null +++ b/tests/script/tsim/stream/udTableAndTag0.sim @@ -0,0 +1,372 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 + +print ===== step1 + +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print ===== step2 +print ===== table name + +sql create database result vgroups 1; + +sql create database test vgroups 4; +sql use test; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql_error create stream streams1 trigger at_once into result.streamt SUBTABLE("aaa") as select _wstart, count(*) c1 from st interval(10s); +sql create stream streams1 trigger at_once into result.streamt SUBTABLE(concat("aaa-", tbname)) as select _wstart, count(*) c1 from st partition by tbname interval(10s); +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,1,2,3); + +$loop_count = 0 +loop0: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select table_name from information_schema.ins_tables where db_name="result" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop0 +endi + +if $data00 != aaa-t1 then + print =====data00=$data00 + goto loop0 +endi + +if $data10 != aaa-t2 then + print =====data10=$data10 + goto loop0 +endi + +$loop_count = 0 +loop1: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result.streamt; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop1 +endi + + +print ===== step3 +print ===== tag name + +sql create database result2 vgroups 1; + +sql create database test2 vgroups 4; +sql use test2; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams2 trigger at_once into result2.streamt2 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as cc interval(10s); +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,1,2,3); + + +$loop_count = 0 +loop2: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select tag_name from information_schema.ins_tags where db_name="result2" and stable_name = "streamt2" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop2 +endi + +if $data00 != cc then + print data00 != cc + print =====data00=$data00 + goto loop2 +endi + +if $data10 != cc then + print =====data10=$data10 + goto loop2 +endi + +sql select cc from result2.streamt2 order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop2 +endi + +if $data00 != tag-t1 then + print data00 != tag-t1 + print =====data00=$data00 + goto loop2 +endi + +if $data10 != tag-t2 then + print =====data10=$data10 + goto loop2 +endi + +$loop_count = 0 +loop3: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result2.streamt2; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop3 +endi + + +print ===== step4 +print ===== tag name + table name + +sql create database result3 vgroups 1; + +sql create database test3 vgroups 4; +sql use test3; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams3 trigger at_once into result3.streamt3 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", tbname)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as dd, tbname interval(10s); +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,1,2,3); + + +$loop_count = 0 +loop4: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select tag_name from information_schema.ins_tags where db_name="result3" and stable_name = "streamt3" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop4 +endi + +if $data00 != dd then + print =====data00=$data00 + goto loop4 +endi + +if $data10 != dd then + print =====data10=$data10 + goto loop4 +endi + +sql select dd from result3.streamt3 order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop4 +endi + +if $data00 != tag-t1 then + print =====data00=$data00 + goto loop4 +endi + +if $data10 != tag-t2 then + print =====data10=$data10 + goto loop4 +endi + +$loop_count = 0 +loop5: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result3.streamt3; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop5 +endi + +$loop_count = 0 +loop6: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select table_name from information_schema.ins_tables where db_name="result3" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop6 +endi + +if $data00 != tbn-t1 then + print =====data00=$data00 + goto loop6 +endi + +if $data10 != tbn-t2 then + print =====data10=$data10 + goto loop6 +endi + + +print ===== step5 +print ===== tag name + table name + +sql create database result4 vgroups 1; + +sql create database test4 vgroups 4; +sql use test4; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create table t3 using st tags(3,3,3); + +sql create stream streams4 trigger at_once into result4.streamt4 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", tbname)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as dd, tbname interval(10s); +sql insert into t1 values(1648791213000,1,1,1) t2 values(1648791213000,2,2,2) t3 values(1648791213000,3,3,3); + + +$loop_count = 0 +loop7: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select table_name from information_schema.ins_tables where db_name="result4" order by 1; + +if $rows != 3 then + print =====rows=$rows + print $data00 $data10 + goto loop7 +endi + +if $data00 != tbn-t1 then + print =====data00=$data00 + goto loop7 +endi + +if $data10 != tbn-t2 then + print =====data10=$data10 + goto loop7 +endi + +if $data20 != tbn-t3 then + print =====data20=$data20 + goto loop7 +endi + +$loop_count = 0 +loop8: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result4.streamt4 order by 3; + +if $rows != 3 then + print =====rows=$rows + print $data00 $data10 + goto loop8 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop8 +endi + +if $data02 != tag-t1 then + print =====data02=$data02 + goto loop8 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop8 +endi + +if $data12 != tag-t2 then + print =====data12=$data12 + goto loop8 +endi + +if $data21 != 1 then + print =====data21=$data21 + goto loop8 +endi + +if $data22 != tag-t3 then + print =====data22=$data22 + goto loop8 +endi + +print ======over + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/udTableAndTag1.sim b/tests/script/tsim/stream/udTableAndTag1.sim new file mode 100644 index 0000000000000000000000000000000000000000..a0393a03cdd6930ad767a797f4275890ed160fb4 --- /dev/null +++ b/tests/script/tsim/stream/udTableAndTag1.sim @@ -0,0 +1,373 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 + +print ===== step1 + +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print ===== step2 +print ===== table name + +sql create database result vgroups 1; + +sql create database test vgroups 4; +sql use test; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql_error create stream streams1 trigger at_once into result.streamt SUBTABLE("aaa") as select _wstart, count(*) c1 from st interval(10s); +sql create stream streams1 trigger at_once into result.streamt SUBTABLE( concat("aaa-", cast(a as varchar(10) ) ) ) as select _wstart, count(*) c1 from st partition by a interval(10s); +print ===== insert into 1 +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,2,2,3); + +$loop_count = 0 +loop0: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select table_name from information_schema.ins_tables where db_name="result" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + print $data20 $data30 + goto loop0 +endi + +if $data00 != aaa-1 then + print =====data00=$data00 + goto loop0 +endi + +if $data10 != aaa-2 then + print =====data10=$data10 + goto loop0 +endi + +$loop_count = 0 +loop1: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result.streamt; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop1 +endi + + +print ===== step3 +print ===== column name + +sql create database result2 vgroups 1; + +sql create database test2 vgroups 4; +sql use test2; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams2 trigger at_once into result2.streamt2 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st partition by concat("col-", cast(a as varchar(10) ) ) as cc interval(10s); +print ===== insert into 2 +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,2,2,3); + + +$loop_count = 0 +loop2: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select tag_name from information_schema.ins_tags where db_name="result2" and stable_name = "streamt2" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop2 +endi + +if $data00 != cc then + print =====data00=$data00 + goto loop2 +endi + +if $data10 != cc then + print =====data10=$data10 + goto loop2 +endi + +sql select cc from result2.streamt2 order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop2 +endi + +if $data00 != col-1 then + print =====data00=$data00 + goto loop2 +endi + +if $data10 != col-2 then + print =====data10=$data10 + goto loop2 +endi + +$loop_count = 0 +loop3: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result2.streamt2; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop3 +endi + + +print ===== step4 +print ===== column name + table name + +sql create database result3 vgroups 1; + +sql create database test3 vgroups 4; +sql use test3; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams3 trigger at_once into result3.streamt3 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", cast(a as varchar(10) ) ) ) as select _wstart, count(*) c1 from st partition by concat("col-", cast(a as varchar(10) ) ) as dd, a interval(10s); +print ===== insert into 3 +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,2,2,3); + + +$loop_count = 0 +loop4: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select tag_name from information_schema.ins_tags where db_name="result3" and stable_name = "streamt3" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop4 +endi + +if $data00 != dd then + print =====data00=$data00 + goto loop4 +endi + +if $data10 != dd then + print =====data10=$data10 + goto loop4 +endi + +sql select dd from result3.streamt3 order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop4 +endi + +if $data00 != col-1 then + print =====data00=$data00 + goto loop4 +endi + +if $data10 != col-2 then + print =====data10=$data10 + goto loop4 +endi + +$loop_count = 0 +loop5: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result3.streamt3; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop5 +endi + +$loop_count = 0 +loop6: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select table_name from information_schema.ins_tables where db_name="result3" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop6 +endi + +if $data00 != tbn-1 then + print =====data00=$data00 + goto loop6 +endi + +if $data10 != tbn-2 then + print =====data10=$data10 + goto loop6 +endi + +print ===== step5 +print ===== tag name + table name + +sql create database result4 vgroups 1; + +sql create database test4 vgroups 4; +sql use test4; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create table t3 using st tags(3,3,3); + +sql create stream streams4 trigger at_once into result4.streamt4 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", dd)) as select _wstart, count(*) c1 from st partition by concat("t", cast(a as varchar(10) ) ) as dd interval(10s); +sql insert into t1 values(1648791213000,1,1,1) t2 values(1648791213000,2,2,2) t3 values(1648791213000,3,3,3); + + +$loop_count = 0 +loop7: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select table_name from information_schema.ins_tables where db_name="result4" order by 1; + +if $rows != 3 then + print =====rows=$rows + print $data00 $data10 + goto loop7 +endi + +if $data00 != tbn-t1 then + print =====data00=$data00 + goto loop7 +endi + +if $data10 != tbn-t2 then + print =====data10=$data10 + goto loop7 +endi + +if $data20 != tbn-t3 then + print =====data20=$data20 + goto loop7 +endi + +$loop_count = 0 +loop8: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result4.streamt4 order by 3; + +if $rows != 3 then + print =====rows=$rows + print $data00 $data10 + goto loop8 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop8 +endi + +if $data02 != t1 then + print =====data02=$data02 + goto loop8 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop8 +endi + +if $data12 != t2 then + print =====data12=$data12 + goto loop8 +endi + +if $data21 != 1 then + print =====data21=$data21 + goto loop8 +endi + +if $data22 != t3 then + print =====data22=$data22 + goto loop8 +endi + +print ======over + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/windowClose.sim b/tests/script/tsim/stream/windowClose.sim index 9fcdcfb9599db5bbd2d4e93a069647a2518abaac..d3bc25d731f7a3800f0ae84d4ce2177cd573242a 100644 --- a/tests/script/tsim/stream/windowClose.sim +++ b/tests/script/tsim/stream/windowClose.sim @@ -68,7 +68,7 @@ sql select * from streamt2; if $rows != 1 then print ======streamt2=$rows - return -1 + goto loop1 endi sql select * from streamt3; @@ -80,7 +80,7 @@ endi sql select * from streamt4; if $rows != 1 then print ======streamt4=$rows - return -1 + goto loop1 endi sql select * from streamt5; @@ -92,7 +92,7 @@ endi sql select * from streamt6; if $rows != 1 then print ======streamt6=$rows - return -1 + goto loop1 endi sql select * from streamt7; @@ -104,7 +104,7 @@ endi sql select * from streamt8; if $rows != 1 then print ======streamt8=$rows - return -1 + goto loop1 endi sql select * from streamt9; @@ -116,7 +116,7 @@ endi sql select * from streamt10; if $rows != 1 then print ======streamt10=$rows - return -1 + goto loop1 endi sql select * from streamt11; @@ -125,4 +125,6 @@ if $rows != 2 then goto loop1 endi +print ======over + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/system-test/0-others/backquote_check.py b/tests/system-test/0-others/backquote_check.py new file mode 100644 index 0000000000000000000000000000000000000000..33577232538074953457830934fa853ba81aec6a --- /dev/null +++ b/tests/system-test/0-others/backquote_check.py @@ -0,0 +1,87 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.dbname = 'db' + self.setsql = TDSetSql() + self.stbname = 'stb' + self.streamname = 'stm' + self.streamtb = 'stm_stb' + def topic_name_check(self): + tdSql.execute(f'create database if not exists {self.dbname}') + tdSql.execute(f'use {self.dbname}') + tdSql.execute(f'create stable {self.stbname} (ts timestamp,c0 int) tags(t0 int)') + for name in [self.dbname,self.stbname]: + type = '' + if name == self.dbname: + type = 'database' + elif name == self.stbname: + type = 'stable' + tdSql.execute(f'create topic if not exists {name} as {type} {name}') + tdSql.query('show topics') + tdSql.checkEqual(tdSql.queryResult[0][0],name) + tdSql.execute(f'drop topic {name}') + tdSql.execute(f'create topic if not exists `{name}` as {type} {name}') + tdSql.query('show topics') + tdSql.checkEqual(tdSql.queryResult[0][0],name) + tdSql.execute(f'drop topic {name}') + tdSql.execute(f'create topic if not exists `{name}` as {type} `{name}`') + tdSql.query('show topics') + tdSql.checkEqual(tdSql.queryResult[0][0],name) + tdSql.execute(f'drop topic {name}') + tdSql.execute(f'create topic if not exists `{name}` as {type} `{name}`') + tdSql.query('show topics') + tdSql.checkEqual(tdSql.queryResult[0][0],name) + tdSql.execute(f'drop topic `{name}`') + + def db_name_check(self): + tdSql.execute(f'create database if not exists `{self.dbname}`') + tdSql.execute(f'use `{self.dbname}`') + tdSql.execute(f'drop database {self.dbname}') + + def stream_name_check(self): + tdSql.execute(f'create database if not exists {self.dbname}') + tdSql.execute(f'use {self.dbname}') + tdSql.execute(f'create stable {self.stbname} (ts timestamp,c0 int) tags(t0 int)') + tdSql.execute(f'create stream `{self.streamname}` into `{self.streamtb}` as select count(*) from {self.stbname} interval(10s);') + tdSql.query('show streams') + tdSql.checkEqual(tdSql.queryResult[0][0],self.streamname) + tdSql.execute(f'drop stream {self.streamname}') + tdSql.execute(f'drop stable {self.streamtb}') + tdSql.execute(f'create stream {self.streamname} into `{self.streamtb}` as select count(*) from {self.stbname} interval(10s);') + tdSql.query('show streams') + tdSql.checkEqual(tdSql.queryResult[0][0],self.streamname) + tdSql.execute(f'drop stream `{self.streamname}`') + tdSql.execute(f'drop database {self.dbname}') + + def run(self): + self.topic_name_check() + self.db_name_check() + self.stream_name_check() + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 722008836964c8b8fcc6ed72960928d2cf8dd10e..1834432bc9a9364e039df2c95ce47fb01e945eab 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -12,6 +12,7 @@ from util.dnodes import * from util.dnodes import TDDnodes from util.dnodes import TDDnode from util.cluster import * +import subprocess BASEVERSION = "3.0.1.8" class TDTestCase: @@ -26,8 +27,21 @@ class TDTestCase: self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) + + def checkProcessPid(self,processName): + i=0 + while i<60: + print(f"wait stop {processName}") + processPid = subprocess.getstatusoutput(f'ps aux|grep {processName} |grep -v "grep"|awk \'{{print $2}}\'')[1] + print(f"times:{i},{processName}-pid:{processPid}") + if(processPid == ""): + break + i += 1 + sleep(1) + else: + print(f'this processName is not stoped in 60s') - + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -115,15 +129,15 @@ class TDTestCase: # tdsqlF.query(f"select count(*) from {stb}") # tdsqlF.checkData(0,0,tableNumbers*recordNumbers1) os.system("pkill taosd") - sleep(2) + self.checkProcessPid("taosd") print(f"start taosd: nohup taosd -c {cPath} & ") os.system(f" nohup taosd -c {cPath} & " ) sleep(10) tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y") - os.system("pkill -9 taosd") - + os.system("pkill taosd") # make sure all the data are saved in disk. + self.checkProcessPid("taosd") tdLog.printNoPrefix("==========step2:update new version ") diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py new file mode 100644 index 0000000000000000000000000000000000000000..1b82fa6e642bec0d3c24467be8a586a5bf8a7f6c --- /dev/null +++ b/tests/system-test/0-others/information_schema.py @@ -0,0 +1,113 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + self.dbname = 'db' + self.stbname = 'stb' + self.binary_length = 20 # the length of binary for column_dict + self.nchar_length = 20 # the length of nchar for column_dict + self.ts = 1537146000000 + self.column_dict = { + 'ts' : 'timestamp', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.binary_length})', + 'col13': f'nchar({self.nchar_length})' + } + self.tbnum = 20 + self.rowNum = 10 + self.tag_dict = { + 't0':'int' + } + self.tag_values = [ + f'1' + ] + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + self.ins_list = ['ins_dnodes','ins_mnodes','ins_modules','ins_qnodes','ins_snodes','ins_cluster','ins_databases','ins_functions',\ + 'ins_indexes','ins_stables','ins_tables','ins_tags','ins_users','ins_grants','ins_vgroups','ins_configs','ins_dnode_variables',\ + 'ins_topics','ins_subscriptions','ins_streams','ins_stream_tasks','ins_vnodes','ins_user_privileges'] + self.perf_list = ['perf_connections','perf_queries','perf_consumers','perf_trans','perf_apps'] + def insert_data(self,column_dict,tbname,row_num): + insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str) + for i in range(row_num): + insert_list = [] + self.setsql.insert_values(column_dict,i,insert_sql,insert_list,self.ts) + def prepare_data(self): + tdSql.execute(f"create database if not exists {self.dbname} vgroups 2") + tdSql.execute(f'use {self.dbname}') + tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) + for i in range(self.tbnum): + tdSql.execute(f"create table {self.stbname}_{i} using {self.stbname} tags({self.tag_values[0]})") + self.insert_data(self.column_dict,f'{self.stbname}_{i}',self.rowNum) + def count_check(self): + tdSql.query('select count(*) from information_schema.ins_tables') + tdSql.checkEqual(tdSql.queryResult[0][0],self.tbnum+len(self.ins_list)+len(self.perf_list)) + tdSql.query(f'select count(*) from information_schema.ins_tables where db_name = "{self.dbname}"') + tdSql.checkEqual(tdSql.queryResult[0][0],self.tbnum) + tdSql.query(f'select count(*) from information_schema.ins_tables where db_name = "{self.dbname}" and stable_name = "{self.stbname}"') + tdSql.checkEqual(tdSql.queryResult[0][0],self.tbnum) + tdSql.execute('create database db1') + tdSql.execute('create table stb1 (ts timestamp,c0 int) tags(t0 int)') + tdSql.execute('create table tb1 using stb1 tags(1)') + tdSql.query(f'select db_name, stable_name, count(*) from information_schema.ins_tables group by db_name, stable_name') + for i in tdSql.queryResult: + if i[0].lower() == 'information_schema': + tdSql.checkEqual(i[2],len(self.ins_list)) + elif i[0].lower() == self.dbname and i[1] == self.stbname: + tdSql.checkEqual(i[2],self.tbnum) + elif i[0].lower() == self.dbname and i[1] == 'stb1': + tdSql.checkEqual(i[2],1) + elif i[0].lower() == 'performance_schema': + tdSql.checkEqual(i[2],len(self.perf_list)) + tdSql.execute('create table db1.ntb (ts timestamp,c0 int)') + tdSql.query(f'select db_name, count(*) from information_schema.ins_tables group by db_name') + print(tdSql.queryResult) + for i in tdSql.queryResult: + if i[0].lower() == 'information_schema': + tdSql.checkEqual(i[1],len(self.ins_list)) + elif i[0].lower() == 'performance_schema': + tdSql.checkEqual(i[1],len(self.perf_list)) + elif i[0].lower() == self.dbname: + tdSql.checkEqual(i[1],self.tbnum+1) + def run(self): + self.prepare_data() + self.count_check() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/1-insert/opentsdb_json_taosc_insert.py b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py index 44243fe029233dc3458a7b6822600bc198e11824..5e493eab0af2bd0cf5ebe463cbde889fa957041b 100644 --- a/tests/system-test/1-insert/opentsdb_json_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py @@ -1432,9 +1432,9 @@ class TDTestCase: self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) query_sql = 'select * from `rFa$sta`' query_res = tdSql.query(query_sql, True) - tdSql.checkEqual(query_res, [(datetime.datetime(2021, 7, 11, 20, 33, 54), True, 'rFas$ta_1', 'ncharTagValue', 2147483647, 9223372036854775807, 22.123456789, 'binaryTagValue', 32767, 11.12345027923584, False, 127)]) + tdSql.checkEqual(query_res, [(datetime.datetime(2021, 7, 11, 20, 33, 54), True, False, 127, 32767, 2147483647, 9223372036854775807, 11.12345027923584, 22.123456789, 'binaryTagValue', 'ncharTagValue', 'rFas$ta_1')]) col_tag_res = tdSql.getColNameList(query_sql) - tdSql.checkEqual(col_tag_res, ['_ts', '_value', 'id', 't!@#$%^&*()_+[];:<>?,9', 't$3', 't%4', 't&6', 't*7', 't@2', 't^5', 'Tt!0', 'tT@1']) + tdSql.checkEqual(col_tag_res, ['_ts', '_value', 'Tt!0', 'tT@1', 't@2', 't$3', 't%4', 't^5', 't&6', 't*7', 't!@#$%^&*()_+[];:<>?,9', 'id']) tdSql.execute('drop table `rFa$sta`') def pointTransCheckCase(self, value_type="obj"): diff --git a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py index f58882720610b32794fc48e4cd652ae70e63b5d7..351cf49e3a217a44e93bbaf9c8c69ce2fa76c190 100644 --- a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py @@ -243,7 +243,7 @@ class TDTestCase: if t_add_tag is not None: sql_seq = f'{stb_name} {ts} {value} t0={t0} t1={t1} t2={t2} t3={t3} t4={t4} t5={t5} t6={t6} t7={t7} t8={t8} t9={t8}' if id_change_tag is not None: - sql_seq = f'{stb_name} {ts} {value} t0={t0} {id}={tb_name} t1={t1} t2={t2} t3={t3} t4={t4} t5={t5} t6={t6} t7={t7} t8={t8}' + sql_seq = f'{stb_name} {ts} {value} {id}={tb_name} t0={t0} t1={t1} t2={t2} t3={t3} t4={t4} t5={t5} t6={t6} t7={t7} t8={t8}' if id_double_tag is not None: sql_seq = f'{stb_name} {ts} {value} {id}=\"{tb_name}_1\" t0={t0} t1={t1} {id}=\"{tb_name}_2\" t2={t2} t3={t3} t4={t4} t5={t5} t6={t6} t7={t7} t8={t8}' if t_add_tag is not None: @@ -1126,9 +1126,9 @@ class TDTestCase: self._conn.schemaless_insert([input_sql], TDSmlProtocolType.TELNET.value, None) query_sql = 'select * from `rFa$sta`' query_res = tdSql.query(query_sql, True) - tdSql.checkEqual(query_res, [(datetime.datetime(2021, 7, 11, 20, 33, 54), 9.223372036854776e+18, '2147483647i32', 'L"ncharTagValue"', '32767i16', '9223372036854775807i64', '22.123456789f64', '"ddzhiksj"', '11.12345f32', 'true', '127Ii8')]) + tdSql.checkEqual(query_res, [(datetime.datetime(2021, 7, 11, 20, 33, 54), 9.223372036854776e+18, 'true', '127Ii8', '32767i16', '2147483647i32', '9223372036854775807i64', '11.12345f32', '22.123456789f64', '"ddzhiksj"', 'L"ncharTagValue"')]) col_tag_res = tdSql.getColNameList(query_sql) - tdSql.checkEqual(col_tag_res, ['_ts', '_value', '"t$3"', 't!@#$%^&*()_+[];:<>?,9', 't#2', 't%4', 't&6', 't*7', 't^5', 'Tt!0', 'tT@1']) + tdSql.checkEqual(col_tag_res, ['_ts', '_value', 'Tt!0', 'tT@1', 't#2', '"t$3"', 't%4', 't^5', 't&6', 't*7', 't!@#$%^&*()_+[];:<>?,9']) tdSql.execute('drop table `rFa$sta`') def tcpKeywordsCheckCase(self, protocol="telnet-tcp"): @@ -1207,7 +1207,6 @@ class TDTestCase: tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') tdCom.cleanTb(dbname="test") input_sql = self.genSqlList()[0] - print(input_sql) self.multiThreadRun(self.genMultiThreadSeq(input_sql)) tdSql.query(f"show tables;") tdSql.checkRows(5) @@ -1416,7 +1415,7 @@ class TDTestCase: self.symbolsCheckCase() self.tsCheckCase() self.openTstbTelnetTsCheckCase() - self.idSeqCheckCase() + # self.idSeqCheckCase() self.idLetterCheckCase() self.noIdCheckCase() self.maxColTagCheckCase() diff --git a/tests/system-test/1-insert/time_range_wise.py b/tests/system-test/1-insert/time_range_wise.py index 3d5c9197d1dbad6740e91dcc378dff2939e75d77..df1cc516c5b59f1431cea1f859b75bcd76988fc5 100644 --- a/tests/system-test/1-insert/time_range_wise.py +++ b/tests/system-test/1-insert/time_range_wise.py @@ -600,6 +600,11 @@ class TDTestCase: tdLog.printNoPrefix("==========step4:after wal, all check again ") self.all_test() + # add for TS-2440 + for i in range(self.rows): + tdSql.execute("drop database if exists db3 ") + tdSql.execute("create database db3 retentions 1s:4m,2s:8m,3s:12m") + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/blockSMA.py b/tests/system-test/2-query/blockSMA.py new file mode 100644 index 0000000000000000000000000000000000000000..85c0189e277eee3cf20a05edc447a6803f3c2202 --- /dev/null +++ b/tests/system-test/2-query/blockSMA.py @@ -0,0 +1,146 @@ +from wsgiref.headers import tspecials +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10000 + self.ts = 1537146000000 + + def run(self): + dbname = "db" + tdSql.prepare() + + tdSql.execute(f'''create table {dbname}.ntb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') + for i in range(self.rowNum): + tdSql.execute(f"insert into {dbname}.ntb values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i % 127 + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i % 255 + 1, i + 1, i + 1, i + 1)) + + + tdSql.execute('flush database db') + + # test functions using sma result + tdSql.query(f"select count(col1),min(col1),max(col1),avg(col1),sum(col1),spread(col1),percentile(col1, 0),first(col1),last(col1) from {dbname}.ntb") + tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 127) + tdSql.checkData(0, 3, 63.8449) + tdSql.checkData(0, 4, 638449) + tdSql.checkData(0, 5, 126.0) + tdSql.checkData(0, 6, 1.0) + tdSql.checkData(0, 7, 1) + tdSql.checkData(0, 8, 94) + + tdSql.query(f"select count(col2),min(col2),max(col2),avg(col2),sum(col2),spread(col2),percentile(col2, 0),first(col2),last(col2) from {dbname}.ntb") + tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 10000) + tdSql.checkData(0, 3, 5000.5) + tdSql.checkData(0, 4, 50005000) + tdSql.checkData(0, 5, 9999.0) + tdSql.checkData(0, 6, 1.0) + tdSql.checkData(0, 7, 1) + tdSql.checkData(0, 8, 10000) + + tdSql.query(f"select count(col3),min(col3),max(col3),avg(col3),sum(col3),spread(col3),percentile(col3, 0),first(col3),last(col3) from {dbname}.ntb") + tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 10000) + tdSql.checkData(0, 3, 5000.5) + tdSql.checkData(0, 4, 50005000) + tdSql.checkData(0, 5, 9999.0) + tdSql.checkData(0, 6, 1.0) + tdSql.checkData(0, 7, 1) + tdSql.checkData(0, 8, 10000) + + tdSql.query(f"select count(col4),min(col4),max(col4),avg(col4),sum(col4),spread(col4),percentile(col4, 0),first(col4),last(col4) from {dbname}.ntb") + tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 10000) + tdSql.checkData(0, 3, 5000.5) + tdSql.checkData(0, 4, 50005000) + tdSql.checkData(0, 5, 9999.0) + tdSql.checkData(0, 6, 1.0) + tdSql.checkData(0, 7, 1) + tdSql.checkData(0, 8, 10000) + + tdSql.query(f"select count(col5),min(col5),max(col5),avg(col5),sum(col5),spread(col5),percentile(col5, 0),first(col5),last(col5) from {dbname}.ntb") + tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 1, 0.1) + tdSql.checkData(0, 2, 9999.09961) + tdSql.checkData(0, 3, 4999.599985846) + tdSql.checkData(0, 4, 49995999.858455874) + tdSql.checkData(0, 5, 9998.999609374) + tdSql.checkData(0, 6, 0.100000001) + tdSql.checkData(0, 7, 0.1) + tdSql.checkData(0, 8, 9999.09961) + + tdSql.query(f"select count(col6),min(col6),max(col6),avg(col6),sum(col6),spread(col6),percentile(col6, 0),first(col6),last(col6) from {dbname}.ntb") + tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 1, 0.1) + tdSql.checkData(0, 2, 9999.100000000) + tdSql.checkData(0, 3, 4999.600000001) + tdSql.checkData(0, 4, 49996000.000005305) + tdSql.checkData(0, 5, 9999.000000000) + tdSql.checkData(0, 6, 0.1) + tdSql.checkData(0, 7, 0.1) + tdSql.checkData(0, 8, 9999.1) + + tdSql.query(f"select count(col11),min(col11),max(col11),avg(col11),sum(col11),spread(col11),percentile(col11, 0),first(col11),last(col11) from {dbname}.ntb") + tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 255) + tdSql.checkData(0, 3, 127.45) + tdSql.checkData(0, 4, 1274500) + tdSql.checkData(0, 5, 254.000000000) + tdSql.checkData(0, 6, 1.0) + tdSql.checkData(0, 7, 1) + tdSql.checkData(0, 8, 55) + + tdSql.query(f"select count(col12),min(col12),max(col12),avg(col12),sum(col12),spread(col12),percentile(col12, 0),first(col12),last(col12) from {dbname}.ntb") + tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 10000) + tdSql.checkData(0, 3, 5000.5) + tdSql.checkData(0, 4, 50005000) + tdSql.checkData(0, 5, 9999.0) + tdSql.checkData(0, 6, 1.0) + tdSql.checkData(0, 7, 1) + tdSql.checkData(0, 8, 10000) + + tdSql.query(f"select count(col13),min(col13),max(col13),avg(col13),sum(col13),spread(col13),percentile(col13, 0),first(col13),last(col13) from {dbname}.ntb") + tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 10000) + tdSql.checkData(0, 3, 5000.5) + tdSql.checkData(0, 4, 50005000) + tdSql.checkData(0, 5, 9999.0) + tdSql.checkData(0, 6, 1.0) + tdSql.checkData(0, 7, 1) + tdSql.checkData(0, 8, 10000) + + tdSql.query(f"select count(col14),min(col14),max(col14),avg(col14),sum(col14),spread(col14),percentile(col14, 0),first(col14),last(col14) from {dbname}.ntb") + tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 10000) + tdSql.checkData(0, 3, 5000.5) + tdSql.checkData(0, 4, 50005000) + tdSql.checkData(0, 5, 9999.0) + tdSql.checkData(0, 6, 1.0) + tdSql.checkData(0, 7, 1) + tdSql.checkData(0, 8, 10000) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/insert_null_none.py b/tests/system-test/2-query/insert_null_none.py index cf5636fb1fa5599cde9f096915667f0f4af7a3e8..4304dee89ef71b0d44c26567f393404fea8bedc2 100755 --- a/tests/system-test/2-query/insert_null_none.py +++ b/tests/system-test/2-query/insert_null_none.py @@ -24,7 +24,7 @@ from util.dnodes import tdDnodes from util.dnodes import * class TDTestCase: - updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"querySmaOptimize":1} + updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 131 ,"querySmaOptimize":1} def init(self, conn, logSql, replicaVar): tdLog.debug("start to execute %s" % __file__) diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index 1c2a6055bcf21522031f1eae519d8c082b311f78..df460df5c387acf863e1a7aab3b2de67ef040abe 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -408,8 +408,8 @@ class TDTestCase: # test group by & order by json tag tdSql.query(f"select ts,jtag->'tag1' from {dbname}.jsons1 partition by jtag->'tag1' order by jtag->'tag1' desc") tdSql.checkRows(11) - tdSql.checkData(0, 1, '"femail"') - tdSql.checkData(2, 1, '"收到货"') + tdSql.checkData(0, 1, '"收到货"') + tdSql.checkData(2, 1, '"femail"') tdSql.checkData(7, 1, "false") @@ -421,9 +421,10 @@ class TDTestCase: tdSql.query(f"select count(*),jtag->'tag1' from {dbname}.jsons1 group by jtag->'tag1' order by jtag->'tag1' desc") tdSql.checkRows(8) tdSql.checkData(0, 0, 2) - tdSql.checkData(0, 1, '"femail"') + tdSql.checkData(0, 1, '"收到货"') + tdSql.checkData(1, 1, '"femail"') tdSql.checkData(1, 0, 2) - tdSql.checkData(1, 1, '"收到货"') + tdSql.checkData(2, 0, 1) tdSql.checkData(2, 1, "11.000000000") tdSql.checkData(5, 0, 1) @@ -437,7 +438,7 @@ class TDTestCase: tdSql.checkData(5, 0, 1) tdSql.checkData(5, 1, "11.000000000") tdSql.checkData(7, 0, 2) - tdSql.checkData(7, 1, '"femail"') + tdSql.checkData(7, 1, '"收到货"') # test stddev with group by json tag tdSql.query(f"select stddev(dataint),jtag->'tag1' from {dbname}.jsons1 group by jtag->'tag1' order by jtag->'tag1'") @@ -445,8 +446,8 @@ class TDTestCase: tdSql.checkData(0, 1, None) tdSql.checkData(4, 0, 0) tdSql.checkData(4, 1, "5.000000000") - tdSql.checkData(7, 0, 11) - tdSql.checkData(7, 1, '"femail"') + tdSql.checkData(6, 0, 11) + tdSql.checkData(7, 1, '"收到货"') res = tdSql.getColNameList(f"select stddev(dataint),jsons1.jtag->'tag1' from {dbname}.jsons1 group by jsons1.jtag->'tag1' order by jtag->'tag1'") cname_list = [] diff --git a/tests/system-test/2-query/nestedQuery.py b/tests/system-test/2-query/nestedQuery.py index 034ab8dcdcc761b6a6abcf550e129a864b328fce..6557aad05f7dc0d3021f1e0bbba8dfb08964ad0f 100755 --- a/tests/system-test/2-query/nestedQuery.py +++ b/tests/system-test/2-query/nestedQuery.py @@ -24,9 +24,9 @@ from util.dnodes import tdDnodes from util.dnodes import * class TDTestCase: - updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , - "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, - "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 131 ,"cDebugFlag":131,"uDebugFlag":131 ,"rpcDebugFlag":131 , "tmrDebugFlag":131 , + "jniDebugFlag":131 ,"simDebugFlag":131,"dDebugFlag":131, "dDebugFlag":131,"vDebugFlag":131,"mDebugFlag":131,"qDebugFlag":131, + "wDebugFlag":131,"sDebugFlag":131,"tsdbDebugFlag":131,"tqDebugFlag":131 ,"fsDebugFlag":131 ,"fnDebugFlag":131} def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -851,6 +851,7 @@ class TDTestCase: tdLog.info("========mark==%s==="% mark); try: tdSql.query(sql,queryTimes=1) + self.explain_sql(sql) except: tdLog.info("sql is not support :=====%s; " %sql) tdSql.error(sql) @@ -4995,9 +4996,7 @@ class TDTestCase: sql += "%s ;" % random.choice(self.limit_u_where) tdLog.info(sql) tdLog.info(len(sql)) - tdSql.query(sql) - self.cur1.execute(sql) - self.explain_sql(sql) + self.data_check(sql,mark='15-2') tdSql.query("select 15-2.2 from stable_1;") for i in range(self.fornum): @@ -5013,9 +5012,7 @@ class TDTestCase: sql += "%s ;" % random.choice(self.limit_u_where) tdLog.info(sql) tdLog.info(len(sql)) - tdSql.query(sql) - self.cur1.execute(sql) - self.explain_sql(sql) + self.data_check(sql,mark='15-2.2') self.restartDnodes() tdSql.query("select 15-3 from stable_1;") @@ -5033,9 +5030,7 @@ class TDTestCase: sql += "%s " % random.choice(self.limit_where) tdLog.info(sql) tdLog.info(len(sql)) - tdSql.query(sql) - self.cur1.execute(sql) - self.explain_sql(sql) + self.data_check(sql,mark='15-3') tdSql.query("select 15-4 from stable_1;") for i in range(self.fornum): @@ -5052,9 +5047,7 @@ class TDTestCase: sql += "%s " % random.choice(self.limit_u_where) tdLog.info(sql) tdLog.info(len(sql)) - tdSql.query(sql) - self.cur1.execute(sql) - self.explain_sql(sql) + self.data_check(sql,mark='15-4') tdSql.query("select 15-4.2 from stable_1;") for i in range(self.fornum): @@ -5087,8 +5080,7 @@ class TDTestCase: tdLog.info(sql) tdLog.info(len(sql)) tdSql.query(sql) - self.cur1.execute(sql) - self.explain_sql(sql) + self.data_check(sql,mark='15-5') #16 select * from (select calc_aggregate_regulars as agg from regular_table where <\>\in\and\or order by limit offset ) #self.dropandcreateDB_random("%s" %db, 1) diff --git a/tests/system-test/2-query/odbc.py b/tests/system-test/2-query/odbc.py new file mode 100644 index 0000000000000000000000000000000000000000..09000fb3d2a191566e16fe39e5c9da7b081080e5 --- /dev/null +++ b/tests/system-test/2-query/odbc.py @@ -0,0 +1,76 @@ +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * +from util.common import tdCom + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def check_ins_cols(self): + tdSql.execute("create database if not exists db") + tdSql.execute("create table db.ntb (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 tinyint unsigned, c7 smallint unsigned, c8 int unsigned, c9 bigint unsigned, c10 float, c11 double, c12 varchar(100), c13 nchar(100))") + tdSql.execute("create table db.stb (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 tinyint unsigned, c7 smallint unsigned, c8 int unsigned, c9 bigint unsigned, c10 float, c11 double, c12 varchar(100), c13 nchar(100)) tags(t int)") + tdSql.execute("insert into db.ctb using db.stb tags(1) (ts, c1) values (now, 1)") + + tdSql.query("select count(*) from information_schema.ins_columns") + tdSql.checkData(0, 0, 265) + + tdSql.query("select * from information_schema.ins_columns where table_name = 'ntb'") + tdSql.checkRows(14) + tdSql.checkData(0, 2, "NORMAL_TABLE") + + + tdSql.query("select * from information_schema.ins_columns where table_name = 'stb'") + tdSql.checkRows(14) + tdSql.checkData(0, 2, "SUPER_TABLE") + + + tdSql.query("select db_name,table_type,col_name,col_type,col_length from information_schema.ins_columns where table_name = 'ctb'") + tdSql.checkRows(14) + tdSql.checkData(0, 0, "db") + tdSql.checkData(1, 1, "CHILD_TABLE") + tdSql.checkData(3, 2, "c3") + tdSql.checkData(4, 3, "INT") + tdSql.checkData(5, 4, 8) + + tdSql.query("desc information_schema.ins_columns") + tdSql.checkRows(9) + tdSql.checkData(0, 0, "table_name") + tdSql.checkData(5, 0, "col_length") + tdSql.checkData(1, 2, 64) + + def check_get_db_name(self): + buildPath = tdCom.getBuildPath() + cmdStr = '%s/build/bin/get_db_name_test'%(buildPath) + tdLog.info(cmdStr) + ret = os.system(cmdStr) + if ret != 0: + tdLog.exit("sml_test get_db_name_test != 0") + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare(replica = self.replicaVar) + + tdLog.printNoPrefix("==========start check_ins_cols run ...............") + self.check_ins_cols() + tdLog.printNoPrefix("==========end check_ins_cols run ...............") + + tdLog.printNoPrefix("==========start check_get_db_name run ...............") + self.check_get_db_name() + tdLog.printNoPrefix("==========end check_get_db_name run ...............") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/out_of_order.py b/tests/system-test/2-query/out_of_order.py new file mode 100644 index 0000000000000000000000000000000000000000..5b52661bae2244d482e150f7afcfde58e8cca123 --- /dev/null +++ b/tests/system-test/2-query/out_of_order.py @@ -0,0 +1,191 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import random + +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql, replicaVar): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run_benchmark(self,dbname,tables,per_table_num,order,replica): + #O :Out of order + #A :Repliaca + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + os.system("%staosBenchmark -d %s -t %d -n %d -O %d -a %d -b float,double,nchar\(200\),binary\(50\) -T 50 -y " % (binPath,dbname,tables,per_table_num,order,replica)) + + def sql_base(self,dbname): + self.check_sub(dbname) + sql1 = "select count(*) from %s.meters" %dbname + self.sql_base_check(sql1,sql1) + + self.check_sub(dbname) + sql2 = "select count(ts) from %s.meters" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(_c0) from %s.meters" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(c0) from %s.meters" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(c1) from %s.meters" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(c2) from %s.meters" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(c3) from %s.meters" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(t0) from %s.meters" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(t1) from %s.meters" %dbname + self.sql_base_check(sql1,sql2) + + + self.check_sub(dbname) + sql2 = "select count(ts) from (select * from %s.meters)" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(_c0) from (select * from %s.meters)" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(c0) from (select * from %s.meters)" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(c1) from (select * from %s.meters)" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(c2) from (select * from %s.meters)" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(c3) from (select * from %s.meters)" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(t0) from (select * from %s.meters)" %dbname + self.sql_base_check(sql1,sql2) + + self.check_sub(dbname) + sql2 = "select count(t1) from (select * from %s.meters)" %dbname + self.sql_base_check(sql1,sql2) + + + def sql_base_check(self,sql1,sql2): + tdSql.query(sql1) + sql1_result = tdSql.getData(0,0) + tdLog.info("sql:%s , result: %s" %(sql1,sql1_result)) + + tdSql.query(sql2) + sql2_result = tdSql.getData(0,0) + tdLog.info("sql:%s , result: %s" %(sql2,sql2_result)) + + if sql1_result==sql2_result: + tdLog.info(f"checkEqual success, sql1_result={sql1_result},sql2_result={sql2_result}") + else : + tdLog.exit(f"checkEqual error, sql1_result=={sql1_result},sql2_result={sql2_result}") + + def run_sql(self,dbname): + self.sql_base(dbname) + + tdSql.execute(" flush database %s;" %dbname) + + self.sql_base(dbname) + + def check_sub(self,dbname): + + sql = "select count(*) from (select distinct(tbname) from %s.meters)" %dbname + tdSql.query(sql) + num = tdSql.getData(0,0) + + for i in range(0,num): + sql1 = "select count(*) from %s.d%d" %(dbname,i) + tdSql.query(sql1) + sql1_result = tdSql.getData(0,0) + tdLog.info("sql:%s , result: %s" %(sql1,sql1_result)) + + def check_out_of_order(self,dbname,tables,per_table_num,order,replica): + self.run_benchmark(dbname,tables,per_table_num,order,replica) + print("sleep 10 seconds") + #time.sleep(10) + print("sleep 10 seconds finish") + + self.run_sql(dbname) + + def run(self): + startTime = time.time() + + #self.check_out_of_order('db1',10,random.randint(10000,50000),random.randint(1,10),1) + self.check_out_of_order('db1',random.randint(50,200),random.randint(10000,20000),random.randint(1,5),1) + + # self.check_out_of_order('db2',random.randint(50,200),random.randint(10000,50000),random.randint(5,50),1) + + # self.check_out_of_order('db3',random.randint(50,200),random.randint(10000,50000),random.randint(50,100),1) + + # self.check_out_of_order('db4',random.randint(50,200),random.randint(10000,50000),100,1) + + endTime = time.time() + print("total time %ds" % (endTime - startTime)) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index b764edebd732ad7d35fea98f0e75c08307991ff9..ec6309c71ad295eb504d8f97b493a062a044fed1 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -15,15 +15,20 @@ sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: + updatecfgDict = {'clientCfg': {'smlChildTableName': 'dataModelName', 'fqdn': 'localhost'}, 'fqdn': 'localhost'} + print("===================: ", updatecfgDict) + def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), True) #tdSql.init(conn.cursor(), logSql) # output sql.txt file def checkFileContent(self, dbname="sml_db"): + simClientCfg="%s/taos.cfg"%tdDnodes.getSimCfgPath() buildPath = tdCom.getBuildPath() - cmdStr = '%s/build/bin/sml_test'%(buildPath) + cmdStr = '%s/build/bin/sml_test %s'%(buildPath, simClientCfg) + print("cmdStr:", cmdStr) tdLog.info(cmdStr) ret = os.system(cmdStr) if ret != 0: @@ -71,22 +76,26 @@ class TDTestCase: tdSql.checkData(0, 2, "web01") tdSql.query(f"select distinct tbname from {dbname}.`sys.cpu.nice`") - tdSql.checkRows(2) + tdSql.checkRows(3) tdSql.query(f"select * from {dbname}.`sys.cpu.nice` order by _ts") - tdSql.checkRows(2) - tdSql.checkData(0, 1, 9.000000000) - tdSql.checkData(0, 2, "lga") - tdSql.checkData(0, 3, "web02") - tdSql.checkData(0, 4, None) - tdSql.checkData(1, 1, 18.000000000) - tdSql.checkData(1, 2, "lga") - tdSql.checkData(1, 3, "web01") - tdSql.checkData(1, 4, "t1") + tdSql.checkRows(4) + tdSql.checkData(0, 1, 13.000000000) + tdSql.checkData(0, 2, "web01") + tdSql.checkData(0, 3, None) + tdSql.checkData(0, 4, "lga") + + tdSql.checkData(1, 1, 9.000000000) + tdSql.checkData(1, 2, "web02") + tdSql.checkData(3, 3, "t1") + tdSql.checkData(0, 4, "lga") tdSql.query(f"select * from {dbname}.macylr") tdSql.checkRows(2) + tdSql.query(f"select * from {dbname}.qelhxo") + tdSql.checkRows(5) + tdSql.query(f"desc {dbname}.macylr") tdSql.checkRows(25) return diff --git a/tests/system-test/2-query/stablity.py b/tests/system-test/2-query/stablity.py index ff026bf1202af2f3e69b2f5316f193c4cdc54296..5e4d5dcbaf0368cbc5f5955d4bd172131eddfacf 100755 --- a/tests/system-test/2-query/stablity.py +++ b/tests/system-test/2-query/stablity.py @@ -24,9 +24,9 @@ from util.dnodes import tdDnodes from util.dnodes import * class TDTestCase: - updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , - "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, - "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 131 ,"cDebugFlag":131,"uDebugFlag":131 ,"rpcDebugFlag":131 , "tmrDebugFlag":131 , + "jniDebugFlag":131 ,"simDebugFlag":131,"dDebugFlag":131, "dDebugFlag":131,"vDebugFlag":131,"mDebugFlag":131,"qDebugFlag":131, + "wDebugFlag":131,"sDebugFlag":131,"tsdbDebugFlag":131,"tqDebugFlag":131 ,"fsDebugFlag":131 ,"fnDebugFlag":131} def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) diff --git a/tests/system-test/6-cluster/5dnode1mnode.py b/tests/system-test/6-cluster/5dnode1mnode.py index d0157ec7c772182a2e24064dfec3cc29ed2a7ed6..ce87bced1b75d1994024586cf5084c5194befc22 100644 --- a/tests/system-test/6-cluster/5dnode1mnode.py +++ b/tests/system-test/6-cluster/5dnode1mnode.py @@ -137,11 +137,34 @@ class TDTestCase: config_dir = dnode.cfgDir return taos.connect(host=host, port=int(port), config=config_dir) + def check_alive(self): + # check cluster alive + tdLog.printNoPrefix("======== test cluster alive: ") + tdSql.checkDataLoop(0, 0, 1, "show cluster alive;", 20, 0.5) + + tdSql.query("show db.alive;") + tdSql.checkData(0, 0, 1) + + # stop 5 dnode + self.TDDnodes.stoptaosd(5) + tdSql.checkDataLoop(0, 0, 2, "show cluster alive;", 20, 0.5) + + tdSql.query("show db.alive;") + tdSql.checkData(0, 0, 2) + + # stop 2 dnode + self.TDDnodes.stoptaosd(2) + tdSql.checkDataLoop(0, 0, 0, "show cluster alive;", 20, 0.5) + + tdSql.query("show db.alive;") + tdSql.checkData(0, 0, 0) + def run(self): # print(self.master_dnode.cfgDict) self.five_dnode_one_mnode() - + # check cluster and db alive + self.check_alive() def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeDrop.py b/tests/system-test/6-cluster/5dnode3mnodeDrop.py index de9207ddd8e15f8d8d727d0427f8531ac8aff46c..9dd3c568054948d7062c2f87f145149509b1b575 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeDrop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeDrop.py @@ -112,7 +112,8 @@ class TDTestCase: dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1] cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;" tdLog.debug(cmd) - os.system(cmd) + if os.system(cmd) != 0: + raise Exception("failed to execute system command. cmd: %s" % cmd) time.sleep(2) tdLog.info(" create cluster with %d dnode done! " %dnodes_nums) @@ -120,7 +121,7 @@ class TDTestCase: def check3mnode(self): count=0 while count < 10: - time.sleep(1) + time.sleep(0.1) tdSql.query("select * from information_schema.ins_mnodes;") if tdSql.checkRows(3) : tdLog.debug("mnode is three nodes") @@ -157,7 +158,7 @@ class TDTestCase: def check3mnode1off(self): count=0 while count < 10: - time.sleep(1) + time.sleep(0.1) tdSql.query("select * from information_schema.ins_mnodes;") if tdSql.checkRows(3) : tdLog.debug("mnode is three nodes") @@ -189,7 +190,7 @@ class TDTestCase: def check3mnode2off(self): count=0 while count < 40: - time.sleep(1) + time.sleep(0.1) tdSql.query("select * from information_schema.ins_mnodes;") if tdSql.checkRows(3) : tdLog.debug("mnode is three nodes") @@ -219,7 +220,7 @@ class TDTestCase: def check3mnode3off(self): count=0 while count < 10: - time.sleep(1) + time.sleep(0.1) tdSql.query("select * from information_schema.ins_mnodes;") if tdSql.checkRows(3) : tdLog.debug("mnode is three nodes") @@ -279,32 +280,47 @@ class TDTestCase: # drop follower of mnode dropcount =0 - while dropcount <= 10: + while dropcount <= 5: for i in range(1,3): tdLog.debug("drop mnode on dnode %d"%(i+1)) tdSql.execute("drop mnode on dnode %d"%(i+1)) tdSql.query("select * from information_schema.ins_mnodes;") count=0 while count<10: - time.sleep(1) + time.sleep(0.1) tdSql.query("select * from information_schema.ins_mnodes;") - if tdSql.checkRows(2): + if tdSql.queryRows == 2: tdLog.debug("drop mnode %d successfully"%(i+1)) break count+=1 + self.wait_for_transactions(100) + tdLog.debug("create mnode on dnode %d"%(i+1)) tdSql.execute("create mnode on dnode %d"%(i+1)) count=0 while count<10: - time.sleep(1) + time.sleep(0.1) tdSql.query("select * from information_schema.ins_mnodes;") - if tdSql.checkRows(3): - tdLog.debug("drop mnode %d successfully"%(i+1)) + if tdSql.queryRows == 3: + tdLog.debug("create mnode %d successfully"%(i+1)) break count+=1 + self.wait_for_transactions(100) dropcount+=1 self.check3mnode() + def wait_for_transactions(self, timeout): + count=0 + while count= timeout: + tdLog.debug("transactions not finished before timeout (%d secs)"%timeout) def getConnection(self, dnode): host = dnode.cfgDict["fqdn"] diff --git a/tests/system-test/7-tmq/subscribeDb.py b/tests/system-test/7-tmq/subscribeDb.py index fd06eedefdcd899fb71261bd4e97324ee0170a1b..0fa9bcfbd461c0498d301e7b702a2ee3f8dd8aa4 100644 --- a/tests/system-test/7-tmq/subscribeDb.py +++ b/tests/system-test/7-tmq/subscribeDb.py @@ -61,7 +61,7 @@ class TDTestCase: def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'): sql = "insert into %s.consumeinfo values "%cdbName - sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) + sql += "(now + %ds, %d, '%s', '%s', %d, %d, %d)"%(consumerId, consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) tdLog.info("consume info sql: %s"%sql) tdSql.query(sql) @@ -174,12 +174,13 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -271,12 +272,13 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -337,6 +339,7 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath @@ -406,12 +409,13 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() diff --git a/tests/system-test/7-tmq/subscribeDb0.py b/tests/system-test/7-tmq/subscribeDb0.py index d4c5e2f87fd6b06eb874f8398ce64ee6eb5947cc..50ef52cb15a5336395154813619976c7933f2dc7 100644 --- a/tests/system-test/7-tmq/subscribeDb0.py +++ b/tests/system-test/7-tmq/subscribeDb0.py @@ -174,12 +174,13 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -191,6 +192,7 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath @@ -254,12 +256,13 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() diff --git a/tests/system-test/7-tmq/subscribeDb1.py b/tests/system-test/7-tmq/subscribeDb1.py index ea78c90abddebac86391af732750d8f670b80492..c5ae44214a0cea58f2c4f3cfee03dd0c3e5918a2 100644 --- a/tests/system-test/7-tmq/subscribeDb1.py +++ b/tests/system-test/7-tmq/subscribeDb1.py @@ -61,7 +61,7 @@ class TDTestCase: def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'): sql = "insert into %s.consumeinfo values "%cdbName - sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) + sql += "(now + %ds, %d, '%s', '%s', %d, %d, %d)"%(consumerId, consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) tdLog.info("consume info sql: %s"%sql) tdSql.query(sql) @@ -174,12 +174,13 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath self.initConsumerTable() - - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdLog.info("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -191,10 +192,11 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict2['dbName'], parameterDict2['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict2['dbName'], parameterDict2['vgroups'], parameterDict2['replica'])) prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) prepareEnvThread2.start() @@ -259,12 +261,13 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -276,10 +279,11 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict2['dbName'], parameterDict2['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict2['dbName'], parameterDict2['vgroups'], parameterDict2['replica'])) prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) prepareEnvThread2.start() diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 141d013270e5d9a545332a1cad173ee066841e73..4cda0624010305743e4249dd87594f1cf255b037 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -60,7 +60,7 @@ class TMQCom: def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'): sql = "insert into %s.consumeinfo values "%cdbName - sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) + sql += "(now + %ds, %d, '%s', '%s', %d, %d, %d)"%(consumerId, consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) tdLog.info("consume info sql: %s"%sql) tdSql.query(sql) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py index 9bfc01529fda135789e55cbe0d01b39e2a6c2639..87832ac0ef2169ad2895928ceb20121ae17ecba9 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py @@ -117,7 +117,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") @@ -186,7 +186,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor 0") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb.py b/tests/system-test/7-tmq/tmqConsFromTsdb.py index 975f89cbd7977df5640d2000a885eaf975a401e6..8ed4a6df973b57f7302d5a2c193debffbf7286a1 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb.py @@ -53,7 +53,7 @@ class TDTestCase: paraDict['rowsPerTbl'] = self.rowsPerTbl tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -116,7 +116,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") @@ -130,7 +130,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) - tmqCom.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) time.sleep(10) for i in range(len(topicNameList)): @@ -185,7 +185,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor 0") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py index 009862137f8ccbb45f0f9bbfa134ae1a5b0479f1..4dcc0b963f8e883326132f5db3bc04a6edd349d1 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py @@ -116,7 +116,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) consumerId = 4 @@ -188,7 +188,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor 0") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py index 528b3a80887c3cc62b1092e5183249e962f5e8db..da8ac6c57deaf99d9871134489b290381b570306 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py @@ -116,7 +116,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) consumerId = 4 @@ -188,7 +188,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor 0") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1.py b/tests/system-test/7-tmq/tmqConsFromTsdb1.py index c0c459d315efa541115cf666b85203f69794b306..b910caf420de5efd6fef315e220920599f946435 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1.py @@ -53,7 +53,7 @@ class TDTestCase: paraDict['rowsPerTbl'] = self.rowsPerTbl tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -116,7 +116,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) consumerId = 4 @@ -188,7 +188,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor 0") diff --git a/tests/system-test/7-tmq/tmqDelete-multiCtb.py b/tests/system-test/7-tmq/tmqDelete-multiCtb.py index 3b72b4aea5d1df34e0230a98953d671536442cab..e59e3d6ecda6e13127eba3f087029ad281cc55b3 100644 --- a/tests/system-test/7-tmq/tmqDelete-multiCtb.py +++ b/tests/system-test/7-tmq/tmqDelete-multiCtb.py @@ -53,7 +53,7 @@ class TDTestCase: paraDict['rowsPerTbl'] = self.rowsPerTbl tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1,wal_retention_size=-1, wal_retention_period=-1) + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar,wal_retention_size=-1, wal_retention_period=-1) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -146,7 +146,7 @@ class TDTestCase: ifManualCommit = 1 keyList = 'group.id:cgrp1,\ enable.auto.commit:true,\ - auto.commit.interval.ms:1000,\ + auto.commit.interval.ms:200,\ auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) @@ -247,7 +247,7 @@ class TDTestCase: ifManualCommit = 1 keyList = 'group.id:cgrp1,\ enable.auto.commit:true,\ - auto.commit.interval.ms:1000,\ + auto.commit.interval.ms:200,\ auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) @@ -335,7 +335,7 @@ class TDTestCase: ifManualCommit = 1 keyList = 'group.id:cgrp1,\ enable.auto.commit:true,\ - auto.commit.interval.ms:1000,\ + auto.commit.interval.ms:200,\ auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) diff --git a/tests/system-test/7-tmq/tmqUpdateWithConsume.py b/tests/system-test/7-tmq/tmqUpdateWithConsume.py index 49a475ff16db96534c13107e94f5d42c8f10454c..4f595788daee975f26cd2e30c1a0fe973e35aa68 100644 --- a/tests/system-test/7-tmq/tmqUpdateWithConsume.py +++ b/tests/system-test/7-tmq/tmqUpdateWithConsume.py @@ -53,7 +53,7 @@ class TDTestCase: paraDict['rowsPerTbl'] = self.rowsPerTbl tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1, wal_retention_size=-1, wal_retention_period=-1) + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar, wal_retention_size=-1, wal_retention_period=-1) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -124,7 +124,7 @@ class TDTestCase: ifManualCommit = 1 keyList = 'group.id:cgrp1,\ enable.auto.commit:true,\ - auto.commit.interval.ms:1000,\ + auto.commit.interval.ms:200,\ auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index f2fbd8486570e385332be5280f58f25af7ae4d75..7ca18f293cddf415a2b4ac3354f5848171e7b736 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -123,6 +123,11 @@ class TDTestCase: def checkData(self): tdSql.execute('use db_taosx') + tdSql.query("select * from tb1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 0) + tdSql.checkData(0, 2, 1) + tdSql.query("select * from ct3 order by c1 desc") tdSql.checkRows(2) tdSql.checkData(0, 1, 51) diff --git a/tests/system-test/99-TDcase/TD-21561.py b/tests/system-test/99-TDcase/TD-21561.py new file mode 100644 index 0000000000000000000000000000000000000000..5b852c77662c7e944762336827476e7fac75ea19 --- /dev/null +++ b/tests/system-test/99-TDcase/TD-21561.py @@ -0,0 +1,148 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import os +import time +import taos +import subprocess +from faker import Faker +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes +from util.dnodes import * + +class TDTestCase: + updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"querySmaOptimize":1} + + def init(self, conn, logSql, replicaVar): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.testcasePath = os.path.split(__file__)[0] + self.testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) + + self.db = "hyp" + + def dropandcreateDB_random(self,database,n): + ts = 1630000000000 + num_random = 10 + fake = Faker('zh_CN') + tdSql.execute('''drop database if exists %s ;''' %database) + tdSql.execute('''create database %s keep 36500 ;'''%(database)) + tdSql.execute('''use %s;'''%database) + + tdSql.execute('''create stable %s.stable_1 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , \ + q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) \ + tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);'''%database) + tdSql.execute('''create stable %s.stable_2 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , \ + q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) \ + tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);'''%database) + + for i in range(num_random): + tdSql.execute('''create table %s.table_%d \ + (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp ) ;'''%(database,i)) + tdSql.execute('''create table %s.stable_1_%d using %s.stable_1 tags('stable_1_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;''' + %(database,i,database,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1))) + + # insert data + for i in range(num_random): + for j in range(n): + tdSql.execute('''insert into %s.stable_1_%d (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts)\ + values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d) ;''' + % (database,i,ts + i*1000 + j, fake.random_int(min=-2147483647, max=2147483647, step=1), + fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i)) + + tdSql.execute('''insert into %s.table_%d (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) \ + values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d) ;''' + % (database,i,ts + i*1000 + j, fake.random_int(min=-2147483647, max=2147483647, step=1), + fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1), + fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) , + fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i )) + + tdSql.query("select count(*) from %s.stable_1;" %database) + tdSql.checkData(0,0,num_random*n) + tdSql.query("select count(*) from %s.table_0;"%database) + tdSql.checkData(0,0,n) + + + + def sqls(self,database,function): + sql0 = f"select {function}(q_tinyint) from {database}.stable_1 where tbname in ('stable_1_1') group by tbname ;" + + sql1 = f"select {function}(q_tinyint) from {database}.stable_1 where tbname in ('stable_1_1') group by tbname;" + + sql2 = f"select {function}(q_tinyint) from {database}.stable_1 where tbname in ('stable_1_1') and _C0 is not null and _C0 between 1600000000000 and now +1h and (q_nchar like 'nchar%' or q_binary = '0' or q_nchar = 'nchar_' ) and q_bool in (0 , 1) group by tbname ;" + + sql3 = f"select * from (select {function}(q_tinyint) from {database}.stable_1 where tbname in ('stable_1_1') and _C0 is not null and _C0 between 1600000000000 and now +1h and (q_nchar like 'nchar%' or q_binary = '0' or q_nchar = 'nchar_' ) and q_bool in (0 , 1) group by tbname) ;" + + self.constant_check(sql1,sql0) + self.constant_check(sql1,sql2) + self.constant_check(sql2,sql3) + + def check_flushdb(self,database): + functions = ['COUNT', 'HYPERLOGLOG'] + for f in functions: + for i in range(20): + self.sqls(database, f); + + tdSql.execute(" flush database %s;" %database) + tdLog.info("flush database success") + + for i in range(20): + self.sqls(database, f); + + + def constant_check(self,sql1,sql2): + tdLog.info("\n=============sql1:(%s)___sql2:(%s) ====================\n" %(sql1,sql2)) + tdSql.query(sql1) + sql1_value = tdSql.getData(0,0) + tdSql.query("reset query cache") + tdSql.query(sql2) + sql2_value = tdSql.getData(0,0) + self.value_check(sql1_value,sql2_value) + + def value_check(self,base_value,check_value): + if base_value==check_value: + tdLog.info(f"checkEqual success, base_value={base_value},check_value={check_value}") + else : + tdLog.exit(f"checkEqual error, base_value=={base_value},check_value={check_value}") + + def run(self): + + startTime = time.time() + + os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) + + self.dropandcreateDB_random("%s" %self.db, 100) + + self.check_flushdb("%s" %self.db) + + endTime = time.time() + print("total time %ds" % (endTime - startTime)) + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/compatibilityAllTest.sh b/tests/system-test/compatibilityAllTest.sh new file mode 100755 index 0000000000000000000000000000000000000000..8b599afd8659e9ce4ed2f92408523f9de8dda917 --- /dev/null +++ b/tests/system-test/compatibilityAllTest.sh @@ -0,0 +1,46 @@ +#!/bin/bash +ulimit -c unlimited +#======================p1-insert=============== + +python3 ./test.py -f 0-others/taosShell.py +python3 ./test.py -f 0-others/taosShellError.py +python3 ./test.py -f 0-others/taosShellNetChk.py +python3 ./test.py -f 1-insert/alter_database.py +python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py +python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py +python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py +python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py +python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py +python3 ./test.py -f 1-insert/alter_stable.py +python3 ./test.py -f 1-insert/alter_table.py +python3 ./test.py -f 1-insert/boundary.py +python3 ./test.py -f 2-query/top.py +python3 ./test.py -f 2-query/top.py -R +python3 ./test.py -f 2-query/tsbsQuery.py +python3 ./test.py -f 2-query/tsbsQuery.py -R +python3 ./test.py -f 2-query/ttl_comment.py +python3 ./test.py -f 2-query/ttl_comment.py -R +python3 ./test.py -f 2-query/twa.py +python3 ./test.py -f 2-query/twa.py -R +python3 ./test.py -f 2-query/union.py +python3 ./test.py -f 2-query/union.py -R +python3 ./test.py -f 6-cluster/5dnode1mnode.py +python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 -i False +python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 -i False +python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 -n 3 +python3 ./test.py -f 7-tmq/subscribeStb4.py +python3 ./test.py -f 7-tmq/db.py +python3 ./test.py -f 7-tmq/tmqError.py +python3 ./test.py -f 7-tmq/schema.py +python3 ./test.py -f 7-tmq/stbFilter.py +python3 ./test.py -f 7-tmq/tmqCheckData.py +python3 ./test.py -f 7-tmq/tmqCheckData1.py +python3 ./test.py -f 7-tmq/tmqConsumerGroup.py +python3 ./test.py -f 7-tmq/tmqShow.py +python3 ./test.py -f 7-tmq/tmqAlterSchema.py +python3 ./test.py -f 99-TDcase/TD-20582.py diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index 1e7d0ed1406d9fc251ffdb6319fede131febef78..342d6410d29f1b94d80b508c05507a0f5b81cdec 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -26,6 +26,10 @@ ELSE () SET(LINK_WEBSOCKET "") ENDIF () +IF (CUS_NAME OR CUS_PROMPT OR CUS_EMAIL) + ADD_DEFINITIONS(-I${CMAKE_CURRENT_SOURCE_DIR}/../../../enterprise/packaging) +ENDIF (CUS_NAME OR CUS_PROMPT OR CUS_EMAIL) + if(TD_WINDOWS) target_link_libraries(shell PUBLIC taos_static ${LINK_WEBSOCKET}) else() diff --git a/tools/shell/inc/shellAuto.h b/tools/shell/inc/shellAuto.h index b7bf5fa1019502acbeaefc8884d4553704f58702..f3ea87e4a540aa34084e0c3c5d91ad58bbae1adf 100644 --- a/tools/shell/inc/shellAuto.h +++ b/tools/shell/inc/shellAuto.h @@ -24,13 +24,13 @@ void pressTabKey(SShellCmd* cmd); // press othr key void pressOtherKey(char c); -// init shell auto funciton , shell start call once +// init shell auto function , shell start call once bool shellAutoInit(); // set conn void shellSetConn(TAOS* conn); -// exit shell auto funciton, shell exit call once +// exit shell auto function, shell exit call once void shellAutoExit(); // callback autotab module diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h index af724c1533177c2203d58b6f5cd1f02fc236f5c4..6b3bc56dc767712b3607075aa07fafb85530d39b 100644 --- a/tools/shell/inc/shellInt.h +++ b/tools/shell/inc/shellInt.h @@ -80,8 +80,9 @@ typedef struct { } SShellArgs; typedef struct { - const char* clientVersion; - const char* promptHeader; + const char *clientVersion; + char cusName[32]; + char promptHeader[32]; const char* promptContinue; const char* osname; int32_t promptSize; diff --git a/tools/shell/inc/shellTire.h b/tools/shell/inc/shellTire.h index bdcf7bcfb3310832b60cc66af5d07c16b7eb6134..e87c3ee4f33726171d107544a137bac7b649174a 100644 --- a/tools/shell/inc/shellTire.h +++ b/tools/shell/inc/shellTire.h @@ -19,7 +19,7 @@ // // The prefix search tree is a efficient storage words and search words tree, it support 95 visible ascii code character // -#define FIRST_ASCII 40 // first visiable char is '0' +#define FIRST_ASCII 40 // first visible char is '0' #define LAST_ASCII 122 // last visilbe char is 'z' // capacity save char is 95 diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 4d40de66bde2c8504369110c436275b27268ac08..d899249b97f0160d366db385653c1c65a9864d9a 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -19,10 +19,25 @@ #include "shellInt.h" -#define TAOS_CONSOLE_PROMPT_HEADER "taos> " +#ifndef CUS_NAME + char cusName[] = "TDengine"; +#endif + +#ifndef CUS_PROMPT + char cusPrompt[] = "taos"; +#endif + +#ifndef CUS_EMAIL + char cusEmail[] = ""; +#endif + +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#endif + #define TAOS_CONSOLE_PROMPT_CONTINUE " -> " -#define SHELL_HOST "TDengine server FQDN to connect. The default host is localhost." +#define SHELL_HOST "The server FQDN to connect. The default host is localhost." #define SHELL_PORT "The TCP/IP port number to use for the connection." #define SHELL_USER "The user name to use when connecting to the server." #define SHELL_PASSWORD "The password to use when connecting to the server." @@ -41,7 +56,6 @@ #define SHELL_PKT_LEN "Packet length used for net test, default is 1024 bytes." #define SHELL_PKT_NUM "Packet numbers used for net test, default is 100." #define SHELL_VERSION "Print program version." -#define SHELL_EMAIL "" #ifdef WEBSOCKET #define SHELL_DSN "The dsn to use when connecting to cloud server." @@ -78,7 +92,7 @@ void shellPrintHelp() { #endif printf("%s%s%s%s\r\n", indent, "-w,", indent, SHELL_WIDTH); printf("%s%s%s%s\r\n", indent, "-V,", indent, SHELL_VERSION); - printf("\r\n\r\nReport bugs to %s.\r\n", SHELL_EMAIL); + printf("\r\n\r\nReport bugs to %s.\r\n", cusEmail); } #ifdef LINUX @@ -86,7 +100,7 @@ void shellPrintHelp() { #include const char *argp_program_version = version; -const char *argp_program_bug_address = SHELL_EMAIL; +const char *argp_program_bug_address = cusEmail; static struct argp_option shellOptions[] = { {"host", 'h', "HOST", 0, SHELL_HOST}, @@ -388,12 +402,13 @@ static int32_t shellCheckArgs() { int32_t shellParseArgs(int32_t argc, char *argv[]) { shellInitArgs(argc, argv); - shell.info.clientVersion = - "Welcome to the TDengine Command Line Interface, Client Version:%s\r\n" - "Copyright (c) 2022 by TDengine, all rights reserved.\r\n\r\n"; - shell.info.promptHeader = TAOS_CONSOLE_PROMPT_HEADER; + shell.info.clientVersion = + "Welcome to the %s Command Line Interface, Client Version:%s\r\n" + "Copyright (c) 2022 by %s, all rights reserved.\r\n\r\n"; + strcpy(shell.info.cusName, cusName); + sprintf(shell.info.promptHeader, "%s> ", cusPrompt); shell.info.promptContinue = TAOS_CONSOLE_PROMPT_CONTINUE; - shell.info.promptSize = 6; + shell.info.promptSize = strlen(shell.info.promptHeader); snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), "version: %s", version); #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index b391d59725938295fcffba13dfac81821a40a0cb..81af5d7fe8e860a242256160aa133124d2cf2503 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -138,6 +138,7 @@ SWords shellCommands[] = { {"show create table \\G;", 0, 0, NULL}, {"show connections;", 0, 0, NULL}, {"show cluster;", 0, 0, NULL}, + {"show cluster alive;", 0, 0, NULL}, {"show databases;", 0, 0, NULL}, {"show dnodes;", 0, 0, NULL}, {"show dnode variables;", 0, 0, NULL}, @@ -263,7 +264,7 @@ char* key_tags[] = {"tags("}; char* key_select[] = {"select "}; // -// ------- gobal variant define --------- +// ------- global variant define --------- // int32_t firstMatchIndex = -1; // first match shellCommands index int32_t lastMatchIndex = -1; // last match shellCommands index @@ -328,7 +329,15 @@ int cntDel = 0; // delete byte count after next press tab // show auto tab introduction void printfIntroduction() { printf(" ****************************** Tab Completion **********************************\n"); - printf(" * The TDengine CLI supports tab completion for a variety of items, *\n"); + char secondLine[160] = "\0"; + sprintf(secondLine, " * The %s CLI supports tab completion for a variety of items, ", + shell.info.cusName); + printf("%s", secondLine); + int secondLineLen = strlen(secondLine); + while (84-(secondLineLen++) > 0) { + printf(" "); + } + printf("*\n"); printf(" * including database names, table names, function names and keywords. *\n"); printf(" * The full list of shortcut keys is as follows: *\n"); printf(" * [ TAB ] ...... complete the current word *\n"); @@ -343,7 +352,7 @@ void printfIntroduction() { } void showHelp() { - printf("\nThe TDengine CLI supports the following commands:"); + printf("\nThe %s CLI supports the following commands:", shell.info.cusName); printf( "\n\ ----- A ----- \n\ @@ -425,6 +434,7 @@ void showHelp() { show create table ;\n\ show connections;\n\ show cluster;\n\ + show cluster alive;\n\ show databases;\n\ show dnodes;\n\ show dnode variables;\n\ @@ -593,7 +603,7 @@ void GenerateVarType(int type, char** p, int count) { // -------------------- shell auto ---------------- // -// init shell auto funciton , shell start call once +// init shell auto function , shell start call once bool shellAutoInit() { // command int32_t count = SHELL_COMMAND_COUNT(); @@ -626,7 +636,7 @@ bool shellAutoInit() { // set conn void shellSetConn(TAOS* conn) { varCon = conn; } -// exit shell auto funciton, shell exit call once +// exit shell auto function, shell exit call once void shellAutoExit() { // free command int32_t count = SHELL_COMMAND_COUNT(); @@ -643,7 +653,7 @@ void shellAutoExit() { } } taosThreadMutexUnlock(&tiresMutex); - // destory + // destroy taosThreadMutexDestroy(&tiresMutex); // free threads @@ -664,7 +674,7 @@ void shellAutoExit() { // // ------------------- auto ptr for tires -------------------------- // -bool setNewAuotPtr(int type, STire* pNew) { +bool setNewAutoPtr(int type, STire* pNew) { if (pNew == NULL) return false; taosThreadMutexLock(&tiresMutex); @@ -707,13 +717,13 @@ void putBackAutoPtr(int type, STire* tire) { if (tires[type] != tire) { // update by out, can't put back , so free if (--tire->ref == 1) { - // support multi thread getAuotPtr + // support multi thread getAutoPtr freeTire(tire); } } else { tires[type]->ref--; - assert(tires[type]->ref > 0); + ASSERT(tires[type]->ref > 0); } taosThreadMutexUnlock(&tiresMutex); @@ -762,7 +772,7 @@ int writeVarNames(int type, TAOS_RES* tres) { } while (row != NULL); // replace old tire - setNewAuotPtr(type, tire); + setNewAutoPtr(type, tire); return numOfRows; } @@ -1030,7 +1040,7 @@ SWords* matchCommand(SWords* input, bool continueSearch) { for (int32_t i = 0; i < count; i++) { SWords* shellCommand = shellCommands + i; if (continueSearch && lastMatchIndex != -1 && i <= lastMatchIndex) { - // new match must greate than lastMatchIndex + // new match must greater than lastMatchIndex if (varMode && i == lastMatchIndex) { // do nothing, var match on lastMatchIndex } else { @@ -1159,7 +1169,7 @@ void createInputFromFirst(SWords* input, SWords* firstMatch) { for (int i = 0; i < firstMatch->matchIndex && word; i++) { // combine source from each word strncpy(input->source + input->source_len, word->word, word->len); - strcat(input->source, " "); // append blank splite + strcat(input->source, " "); // append blank space input->source_len += word->len + 1; // 1 is blank length // move next word = word->next; @@ -1388,7 +1398,7 @@ bool appendAfterSelect(TAOS* con, SShellCmd* cmd, char* sql, int32_t len) { return true; } - // fill funciton + // fill function if (fieldEnd) { // fields is end , need match keyword ret = fillWithType(con, cmd, last, WT_VAR_KEYWORD); @@ -1571,7 +1581,7 @@ bool matchCreateTable(TAOS* con, SShellCmd* cmd) { // tb options if (!ret) { - // find like create talbe st (...) tags(..) + // find like create table st (...) tags(..) char* p1 = strchr(ps, ')'); // first ')' end if (p1) { if (strchr(p1 + 1, ')')) { // second ')' end diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index 5235030cf91cac5d41b782544da9d8a7be4e1349..0e305f57e9c781a8149db1fecccd06a703bc1465 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -40,7 +40,7 @@ static void shellPositionCursorEnd(SShellCmd *cmd); static void shellPrintChar(char c, int32_t times); static void shellPositionCursor(int32_t step, int32_t direction); static void shellUpdateBuffer(SShellCmd *cmd); -static int32_t shellIsReadyGo(SShellCmd *cmd); +static bool shellIsReadyGo(SShellCmd *cmd); static void shellGetMbSizeInfo(const char *str, int32_t *size, int32_t *width); static void shellResetCommand(SShellCmd *cmd, const char s[]); void shellClearScreen(int32_t ecmd_pos, int32_t cursor_pos); @@ -62,7 +62,8 @@ int32_t shellCountPrefixOnes(uint8_t c) { } void shellGetPrevCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width) { - assert(pos > 0); + ASSERT(pos > 0); + if (pos <= 0) return; TdWchar wc; *size = 0; @@ -75,13 +76,14 @@ void shellGetPrevCharSize(const char *str, int32_t pos, int32_t *size, int32_t * } taosMbToWchar(&wc, str + pos, MB_CUR_MAX); - // assert(rc == *size); // it will be core, if str is encode by utf8 and taos charset is gbk + // ASSERT(rc == *size); // it will be core, if str is encode by utf8 and taos charset is gbk *width = taosWcharWidth(wc); } void shellGetNextCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width) { - assert(pos >= 0); + ASSERT(pos >= 0); + if(pos < 0) return; TdWchar wc; *size = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); @@ -89,7 +91,8 @@ void shellGetNextCharSize(const char *str, int32_t pos, int32_t *size, int32_t * } void shellInsertChar(SShellCmd *cmd, char *c, int32_t size) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; TdWchar wc; if (taosMbToWchar(&wc, c, size) < 0) return; @@ -135,7 +138,8 @@ void shellInsertStr(SShellCmd *cmd, char *str, int32_t size) { } void shellBackspaceChar(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset > 0) { shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -155,7 +159,8 @@ void shellBackspaceChar(SShellCmd *cmd) { } void shellClearLineBefore(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); memmove(cmd->command, cmd->command + cmd->cursorOffset, cmd->commandSize - cmd->cursorOffset); @@ -169,7 +174,8 @@ void shellClearLineBefore(SShellCmd *cmd) { } void shellClearLineAfter(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); cmd->commandSize -= cmd->endOffset - cmd->cursorOffset; @@ -178,7 +184,8 @@ void shellClearLineAfter(SShellCmd *cmd) { } void shellDeleteChar(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset < cmd->commandSize) { shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -196,7 +203,8 @@ void shellDeleteChar(SShellCmd *cmd) { } void shellMoveCursorLeft(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset > 0) { shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -210,7 +218,8 @@ void shellMoveCursorLeft(SShellCmd *cmd) { } void shellMoveCursorRight(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset < cmd->commandSize) { shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -224,7 +233,8 @@ void shellMoveCursorRight(SShellCmd *cmd) { } void shellPositionCursorHome(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset > 0) { shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -244,7 +254,8 @@ void positionCursorMiddle(SShellCmd *cmd) { } void shellPositionCursorEnd(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset < cmd->commandSize) { shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -279,7 +290,8 @@ void shellPositionCursor(int32_t step, int32_t direction) { } void shellUpdateBuffer(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (shellRegexMatch(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " "); strcat(cmd->buffer, cmd->command); @@ -293,8 +305,9 @@ void shellUpdateBuffer(SShellCmd *cmd) { shellShowOnScreen(cmd); } -int32_t shellIsReadyGo(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); +bool shellIsReadyGo(SShellCmd *cmd) { + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return false; char *total = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE); memset(cmd->command + cmd->commandSize, 0, SHELL_MAX_COMMAND_SIZE - cmd->commandSize); @@ -305,11 +318,11 @@ int32_t shellIsReadyGo(SShellCmd *cmd) { "\\s*clear\\s*$)"; if (shellRegexMatch(total, reg_str, REG_EXTENDED | REG_ICASE)) { taosMemoryFree(total); - return 1; + return true; } taosMemoryFree(total); - return 0; + return false; } void shellGetMbSizeInfo(const char *str, int32_t *size, int32_t *width) { @@ -321,7 +334,8 @@ void shellGetMbSizeInfo(const char *str, int32_t *size, int32_t *width) { } void shellResetCommand(SShellCmd *cmd, const char s[]) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); memset(cmd->buffer, 0, SHELL_MAX_COMMAND_SIZE); @@ -399,7 +413,7 @@ void shellShowOnScreen(SShellCmd *cmd) { int32_t ret = taosMbToWchar(&wc, str, MB_CUR_MAX); if (ret < 0) break; size += ret; - /* assert(size >= 0); */ + /* ASSERT(size >= 0); */ int32_t width = taosWcharWidth(wc); if (remain_column > width) { printf("%lc", wc); diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 986806fdd83a7b2ee9d2b72dbfcf332c713323df..4a9cf8343120654899a9080e4f6e219685c88cdf 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -713,7 +713,7 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) { } default: - assert(false); + ASSERT(false); } return 0; @@ -1072,7 +1072,8 @@ void *shellThreadLoop(void *arg) { } int32_t shellExecute() { - printf(shell.info.clientVersion, taos_get_client_info()); + printf(shell.info.clientVersion, shell.info.cusName, + taos_get_client_info(), shell.info.cusName); fflush(stdout); SShellArgs *pArgs = &shell.args; @@ -1127,7 +1128,7 @@ int32_t shellExecute() { } if (tsem_init(&shell.cancelSem, 0, 0) != 0) { - printf("failed to create cancel semphore\r\n"); + printf("failed to create cancel semaphore\r\n"); return -1; } diff --git a/tools/shell/src/shellUtil.c b/tools/shell/src/shellUtil.c index 8c47d165557317dbcf710dda7d72b77037340692..e15b49efcc35da2682d003243c0a19eb278acbc7 100644 --- a/tools/shell/src/shellUtil.c +++ b/tools/shell/src/shellUtil.c @@ -50,19 +50,19 @@ bool shellRegexMatch(const char *s, const char *reg, int32_t cflags) { int32_t shellCheckIntSize() { if (sizeof(int8_t) != 1) { - printf("taos int8 size is %d(!= 1)", (int)sizeof(int8_t)); + printf("int8 size is %d(!= 1)", (int)sizeof(int8_t)); return -1; } if (sizeof(int16_t) != 2) { - printf("taos int16 size is %d(!= 2)", (int)sizeof(int16_t)); + printf("int16 size is %d(!= 2)", (int)sizeof(int16_t)); return -1; } if (sizeof(int32_t) != 4) { - printf("taos int32 size is %d(!= 4)", (int)sizeof(int32_t)); + printf("int32 size is %d(!= 4)", (int)sizeof(int32_t)); return -1; } if (sizeof(int64_t) != 8) { - printf("taos int64 size is %d(!= 8)", (int)sizeof(int64_t)); + printf("int64 size is %d(!= 8)", (int)sizeof(int64_t)); return -1; } return 0; @@ -80,7 +80,7 @@ void shellGenerateAuth() { void shellDumpConfig() { SConfig *pCfg = taosGetCfg(); if (pCfg == NULL) { - printf("TDengine read global config failed!\r\n"); + printf("read global config failed!\r\n"); } else { cfgDumpCfg(pCfg, 1, true); } diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c index bbb127b12846f113ee1583940ee8d005623bf22b..e3584b689009f9ca120af6e6a29e7ab8c643ec29 100644 --- a/tools/shell/src/shellWebsocket.c +++ b/tools/shell/src/shellWebsocket.c @@ -235,7 +235,7 @@ void shellRunSingleCommandWebsocketImp(char *command) { if (reconnectNum == 0) { continue; } else { - fprintf(stderr, "TDengine server is disconnected, will try to reconnect\n"); + fprintf(stderr, "The server is disconnected, will try to reconnect\n"); } return; } diff --git a/utils/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt index b048b79e9b71058faaccdadeb723c1e0f46dce58..6ca266c55533b644c47ad61eb3bebc273ae7be7f 100644 --- a/utils/test/c/CMakeLists.txt +++ b/utils/test/c/CMakeLists.txt @@ -4,6 +4,7 @@ add_executable(tmq_sim tmqSim.c) add_executable(create_table createTable.c) add_executable(tmq_taosx_ci tmq_taosx_ci.c) add_executable(sml_test sml_test.c) +add_executable(get_db_name_test get_db_name_test.c) target_link_libraries( create_table PUBLIC taos_static @@ -40,3 +41,11 @@ target_link_libraries( PUBLIC common PUBLIC os ) + +target_link_libraries( + get_db_name_test + PUBLIC taos_static + PUBLIC util + PUBLIC common + PUBLIC os +) diff --git a/utils/test/c/get_db_name_test.c b/utils/test/c/get_db_name_test.c new file mode 100644 index 0000000000000000000000000000000000000000..ebbfdc84a77a0eb3d76a5b561c9b280b3930c7a6 --- /dev/null +++ b/utils/test/c/get_db_name_test.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include "taos.h" +#include "types.h" +#include "tlog.h" + +int get_db_test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db vgroups 2"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + int code = taos_errno(pRes); + taos_free_result(pRes); + ASSERT(code == 0); + + code = taos_get_current_db(taos, NULL, 0, NULL); + ASSERT(code != 0); + + int required = 0; + code = taos_get_current_db(taos, NULL, 0, &required); + ASSERT(code != 0); + ASSERT(required == 7); + + char database[10] = {0}; + code = taos_get_current_db(taos, database, 3, &required); + ASSERT(code != 0); + ASSERT(required == 7); + ASSERT(strcpy(database, "sm")); + + char database1[10] = {0}; + code = taos_get_current_db(taos, database1, 10, &required); + ASSERT(code == 0); + ASSERT(strcpy(database1, "sml_db")); + + taos_close(taos); + + return code; +} + +int main(int argc, char *argv[]) { + int ret = 0; + ret = get_db_test(); + ASSERT(!ret); + return ret; +} diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index df416b3822a890f9b6e6e5d3790521686587f50b..33fbbba1c175c4ac37d273274d993efb23580e94 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -19,8 +19,8 @@ #include #include #include "taos.h" -#include "types.h" #include "tlog.h" +#include "types.h" int smlProcess_influx_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -78,12 +78,23 @@ int smlProcess_telnet_Test() { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - const char *sql[] = {"sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0", - "sys.if.bytes.out 1479496101 1.3E1 interface=eth0 host=web01 ", - "sys.if.bytes.out 1479496102 1.3E3 network=tcp", - " sys.procs.running 1479496100 42 host=web01 "}; - - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, + // char *sql[4] = {0}; + // sql[0] = taosMemoryCalloc(1, 128); + // sql[1] = taosMemoryCalloc(1, 128); + // sql[2] = taosMemoryCalloc(1, 128); + // sql[3] = taosMemoryCalloc(1, 128); + const char *sql1[] = {"sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0", + "sys.if.bytes.out 1479496101 1.3E1 interface=eth0 host=web01 ", + "sys.if.bytes.out 1479496102 1.3E3 network=tcp", + " sys.procs.running 1479496100 42 host=web01 "}; + + // for(int i = 0; i < 4; i++){ + // strncpy(sql[i], sql1[i], 128); + // } + + // pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, + // TSDB_SML_TIMESTAMP_NANO_SECONDS); + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); @@ -103,41 +114,62 @@ int smlProcess_json1_Test() { taos_free_result(pRes); const char *sql[] = { - "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":0,\"value\":18,\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":\"lga\"}},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":\"lga\"}}]" - }; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":0,\"value\":18,\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":" + "\"lga\"}},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":" + "\"lga\"}}]"}; + + char *sql1[1] = {0}; + for (int i = 0; i < 1; i++) { + sql1[i] = taosMemoryCalloc(1, 1024); + strncpy(sql1[i], sql[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } taos_free_result(pRes); - taos_close(taos); - return code; -} - -int smlProcess_json2_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql1[i]); + } - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); + const char *sql2[] = { + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":\"web01\",\"dc\":\"lga\"}" + "},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":\"lga\"}" + "}]", + }; - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); + char *sql3[1] = {0}; + for (int i = 0; i < 1; i++) { + sql3[i] = taosMemoryCalloc(1, 1024); + strncpy(sql3[i], sql2[i], 1023); + } - const char *sql[] = { - "{\"metric\":\"meter_current0\",\"timestamp\":{\"value\":1662344042,\"type\":\"s\"},\"value\":{\"value\":10.3,\"type\":\"i64\"},\"tags\":{\"groupid\":{\"value\":2,\"type\":\"bigint\"},\"location\":{\"value\":\"北京\",\"type\":\"binary\"},\"id\":\"d1001\"}}" - }; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); + code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } taos_free_result(pRes); + + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql3[i]); + } + taos_close(taos); return code; } -int smlProcess_json3_Test() { +int smlProcess_json2_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); @@ -147,18 +179,33 @@ int smlProcess_json3_Test() { taos_free_result(pRes); const char *sql[] = { - "{\"metric\":\"meter_current1\",\"timestamp\":{\"value\":1662344042,\"type\":\"s\"},\"value\":{\"value\":10.3,\"type\":\"i64\"},\"tags\":{\"t1\":{\"value\":2,\"type\":\"bigint\"},\"t2\":{\"value\":2,\"type\":\"int\"},\"t3\":{\"value\":2,\"type\":\"i16\"},\"t4\":{\"value\":2,\"type\":\"i8\"},\"t5\":{\"value\":2,\"type\":\"f32\"},\"t6\":{\"value\":2,\"type\":\"double\"},\"t7\":{\"value\":\"8323\",\"type\":\"binary\"},\"t8\":{\"value\":\"北京\",\"type\":\"nchar\"},\"t9\":{\"value\":true,\"type\":\"bool\"},\"id\":\"d1001\"}}"}; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + "{\"metric\":\"meter_current0\",\"timestamp\":{\"value\":1662344042,\"type\":\"s\"},\"value\":{\"value\":10.3," + "\"type\":\"i64\"},\"tags\":{\"groupid\":{\"value\":2,\"type\":\"bigint\"},\"location\":{\"value\":\"北京\"," + "\"type\":\"binary\"},\"id\":\"d1001\"}}"}; + char *sql1[1] = {0}; + for (int i = 0; i < 1; i++) { + sql1[i] = taosMemoryCalloc(1, 1024); + strncpy(sql1[i], sql[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } taos_free_result(pRes); taos_close(taos); + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql1[i]); + } return code; } -int smlProcess_json4_Test() { +int smlProcess_json3_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); @@ -168,15 +215,28 @@ int smlProcess_json4_Test() { taos_free_result(pRes); const char *sql[] = { - "{\"metric\":\"meter_current2\",\"timestamp\":{\"value\":1662344042000,\"type\":\"ms\"},\"value\":\"ni\",\"tags\":{\"t1\":{\"value\":20,\"type\":\"i64\"},\"t2\":{\"value\":25,\"type\":\"i32\"},\"t3\":{\"value\":2,\"type\":\"smallint\"},\"t4\":{\"value\":2,\"type\":\"tinyint\"},\"t5\":{\"value\":2,\"type\":\"float\"},\"t6\":{\"value\":0.2,\"type\":\"f64\"},\"t7\":\"nsj\",\"t8\":{\"value\":\"北京\",\"type\":\"nchar\"},\"t9\":false,\"id\":\"d1001\"}}" - }; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + "[{\"metric\":\"sys.cpu.nice3\",\"timestamp\":0,\"value\":\"18\",\"tags\":{\"host\":\"web01\",\"id\":\"t1\"," + "\"dc\":\"lga\"}}]"}; + char *sql1[1] = {0}; + for (int i = 0; i < 1; i++) { + sql1[i] = taosMemoryCalloc(1, 1024); + strncpy(sql1[i], sql[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } taos_free_result(pRes); taos_close(taos); + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql1[i]); + } return code; } @@ -242,10 +302,10 @@ int sml_16384_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); - if(code) return code; + if (code) return code; const char *sql1[] = { - "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c0=f,c1=127i8,c11=L\"ncharColValue\",c10=t 1626006833639000000", + "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c0=f,c1=127i8,c11=L\"ncharColValue\",c10=t 1626006833631000000", }; pRes = taos_schemaless_insert(taos, (char **)sql1, 1, TSDB_SML_LINE_PROTOCOL, 0); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -677,52 +737,6 @@ int sml_oom_Test() { return code; } -int sml_16368_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "[{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639000, \"type\": \"us\"}, \"value\": 1, " - "\"tags\": {\"t1\": 3, \"t2\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t3\", \"type\": " - "\"binary\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833739000, \"type\": \"us\"}, \"value\": 2, " - "\"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " - "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}," - "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639100, \"type\": \"us\"}, \"value\": 3, " - "\"tags\": {\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t3\": {\"value\": \"ste\", \"type\": \"nchar\"}}}," - "{\"metric\": \"stf567890\", \"timestamp\": {\"value\": 1626006833639200, \"type\": \"us\"}, \"value\": 4, " - "\"tags\": {\"t1\": {\"value\": 4, \"type\": \"bigint\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " - "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639300, \"type\": \"us\"}, \"value\": " - "{\"value\": 5, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t2\": 5.0, " - "\"t3\": {\"value\": \"t4\", \"type\": \"binary\"}}}," - "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639400, \"type\": \"us\"}, \"value\": " - "{\"value\": 6, \"type\": \"double\"}, \"tags\": {\"t2\": 5.0, \"t3\": {\"value\": \"ste2\", \"type\": " - "\"nchar\"}}}," - "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006834639400, \"type\": \"us\"}, \"value\": " - "{\"value\": 7, \"type\": \"double\"}, \"tags\": {\"t2\": {\"value\": 5.0, \"type\": \"double\"}, \"t3\": " - "{\"value\": \"ste2\", \"type\": \"nchar\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833839006, \"type\": \"us\"}, \"value\": " - "{\"value\": 8, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": " - "{\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, " - "\"type\": \"double\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833939007, \"type\": \"us\"}, \"value\": " - "{\"value\": 9, \"type\": \"double\"}, \"tags\": {\"t1\": 4, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " - "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}]"}; - pRes = taos_schemaless_insert(taos, (char **)sql, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MICRO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - taos_close(taos); - - return code; -} - int sml_dup_time_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -732,7 +746,7 @@ int sml_dup_time_Test() { const char *sql[] = {//"test_ms,t0=t c0=f 1626006833641", "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=false,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "c0=f,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." "123456789f64,c7=\"xcxvwjvf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " @@ -762,214 +776,6 @@ int sml_dup_time_Test() { return code; } -int sml_16960_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "[" - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955000, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": 830525384, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955001, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -588348364, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955002, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -370310823, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955003, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -811250191, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955004, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -330340558, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}" - "]"}; - - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, - TSDB_SML_TIMESTAMP_MILLI_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - taos_close(taos); - - return code; -} - int sml_add_tag_col_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -977,8 +783,10 @@ int sml_add_tag_col_Test() { taos_free_result(pRes); const char *sql[] = { - "macylr,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000" - }; + "macylr,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000"}; pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); @@ -989,8 +797,10 @@ int sml_add_tag_col_Test() { if (code) return code; const char *sql1[] = { - "macylr,id=macylr_17875_1804,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\",t11=127i8,t10=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c8=L\"ncharColValue\",c9=7u64,c11=L\"ncharColValue\",c10=f 1626006833639000000" - }; + "macylr,id=macylr_17875_1804,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=" + "22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\",t11=127i8,t10=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c8=" + "L\"ncharColValue\",c9=7u64,c11=L\"ncharColValue\",c10=f 1626006833639000000"}; pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, 0); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -1004,10 +814,10 @@ int sml_add_tag_col_Test() { int smlProcess_18784_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + TAOS_RES *pRes = taos_query(taos, "create database if not exists db_18784 schemaless 1"); taos_free_result(pRes); - pRes = taos_query(taos, "use sml_db"); + pRes = taos_query(taos, "use db_18784"); taos_free_result(pRes); const char *sql[] = { @@ -1018,7 +828,7 @@ int smlProcess_18784_Test() { printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes)); int code = taos_errno(pRes); ASSERT(!code); - ASSERT(taos_affected_rows(pRes) == 2); + ASSERT(taos_affected_rows(pRes) == 1); taos_free_result(pRes); pRes = taos_query(taos, "select * from disk"); @@ -1027,26 +837,26 @@ int smlProcess_18784_Test() { ASSERT(fieldNum == 5); printf("fieldNum:%d\n", fieldNum); TAOS_ROW row = NULL; - int32_t rowIndex = 0; - while((row = taos_fetch_row(pRes)) != NULL) { - int64_t ts = *(int64_t*)row[0]; - int64_t used = *(int64_t*)row[1]; - int64_t total = *(int64_t*)row[2]; - int64_t freed = *(int64_t*)row[3]; - if(rowIndex == 0){ + int32_t rowIndex = 0; + while ((row = taos_fetch_row(pRes)) != NULL) { + int64_t ts = *(int64_t *)row[0]; + int64_t used = *(int64_t *)row[1]; + int64_t total = *(int64_t *)row[2]; + int64_t freed = *(int64_t *)row[3]; + if (rowIndex == 0) { ASSERT(ts == 1661943960000); ASSERT(used == 176059); ASSERT(total == 1081101176832); ASSERT(freed == 66932805); -// ASSERT_EQ(latitude, 24.5208); -// ASSERT_EQ(longitude, 28.09377); -// ASSERT_EQ(elevation, 428); -// ASSERT_EQ(velocity, 0); -// ASSERT_EQ(heading, 304); -// ASSERT_EQ(grade, 0); -// ASSERT_EQ(fuel_consumption, 25); - }else{ -// ASSERT(0); + // ASSERT_EQ(latitude, 24.5208); + // ASSERT_EQ(longitude, 28.09377); + // ASSERT_EQ(elevation, 428); + // ASSERT_EQ(velocity, 0); + // ASSERT_EQ(heading, 304); + // ASSERT_EQ(grade, 0); + // ASSERT_EQ(fuel_consumption, 25); + } else { + // ASSERT(0); } rowIndex++; } @@ -1063,17 +873,21 @@ int sml_19221_Test() { taos_free_result(pRes); const char *sql[] = { - "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000\nqelhxo,id=pnnhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000\n#comment\nqelhxo,id=pnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000", + "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 " + "1626006833632000000\nqelhxo,id=pnnhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 " + "1626006833633000000\n#comment\nqelhxo,id=pnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 " + "1626006833634000000", }; pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - char* tmp = (char*)taosMemoryCalloc(1024, 1); + char *tmp = (char *)taosMemoryCalloc(1024, 1); memcpy(tmp, sql[0], strlen(sql[0])); - *(char*)(tmp+44) = 0; + *(char *)(tmp + 44) = 0; int32_t totalRows = 0; - pRes = taos_schemaless_insert_raw(taos, tmp, strlen(sql[0]), &totalRows, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + pRes = taos_schemaless_insert_raw(taos, tmp, strlen(sql[0]), &totalRows, TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT(totalRows == 3); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -1088,19 +902,22 @@ int sml_19221_Test() { int sml_ts2164_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - TAOS_RES *pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS line_test BUFFER 384 MINROWS 1000 PAGES 256 PRECISION 'ns'"); + TAOS_RES *pRes = + taos_query(taos, "CREATE DATABASE IF NOT EXISTS line_test BUFFER 384 MINROWS 1000 PAGES 256 PRECISION 'ns'"); taos_free_result(pRes); const char *sql[] = { + // "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", + "meters,location=la,groupid=ca current=11.8,voltage=221", "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", - "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", - "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27", + // "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27", }; pRes = taos_query(taos, "use line_test"); taos_free_result(pRes); - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); @@ -1117,22 +934,37 @@ int sml_ttl_Test() { taos_free_result(pRes); const char *sql[] = { - "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" 1626006833739000000", + "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" " + "1626006833739000000", + }; + const char *sql1[] = { + "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" " + "1626006833339000000", }; pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - pRes = taos_schemaless_insert_ttl(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, 20); + pRes = taos_schemaless_insert_ttl(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS, 20); printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes)); taos_free_result(pRes); - pRes = taos_query(taos, "select `ttl` from information_schema.ins_tables where table_name='t_be97833a0e1f523fcdaeb6291d6fdf27'"); + pRes = taos_schemaless_insert_ttl(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS, 20); + + printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes)); + taos_free_result(pRes); + + pRes = taos_query( + taos, "select `ttl` from information_schema.ins_tables where table_name='t_be97833a0e1f523fcdaeb6291d6fdf27'"); printf("%s result2:%s\n", __FUNCTION__, taos_errstr(pRes)); TAOS_ROW row = taos_fetch_row(pRes); - int32_t ttl = *(int32_t*)row[0]; - ASSERT(ttl == 20); + if (row != NULL && *row != NULL) { + int32_t ttl = *(int32_t *)row[0]; + ASSERT(ttl == 20); + } int code = taos_errno(pRes); taos_free_result(pRes); @@ -1141,8 +973,132 @@ int sml_ttl_Test() { return code; } +// char *str[] ={ +// "", +// "f64", +// "F64", +// "f32", +// "F32", +// "i", +// "I", +// "i64", +// "I64", +// "u", +// "U", +// "u64", +// "U64", +// "i32", +// "I32", +// "u32", +// "U32", +// "i16", +// "I16", +// "u16", +// "U16", +// "i8", +// "I8", +// "u8", +// "U8", +// }; +// uint8_t smlCalTypeSum(char* endptr, int32_t left){ +// uint8_t sum = 0; +// for(int i = 0; i < left; i++){ +// sum += endptr[i]; +// } +// return sum; +// } + +int sml_ts2385_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS ts2385"); + taos_free_result(pRes); + + const char *sql[] = { + "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 " + "s5=false,s18=false,k14=0,k2=0,k8=0,k10=0,s9=false,s19=false,k11=0,k13=0,s22=false,k15=0,m2=37.416671660000006," + "m8=600,m10=1532,m1=20.25,m13=0,s7=false,k7=0,m16=0,s17=false,k4=0,s11=false,s15=true,m7=600,m12=1490,s1=true," + "m14=0,s14=false,s16=true,k5=0,hex=" + "\"7b3b00000001030301030200000000323231313233304339344b30002b01012a10028003000000070d05da025802580258025802580258" + "045305fc05f505d200000000000000000afc7d\",k6=0,m3=600,s3=false,s24=false,k3=0,m6=600,m15=0,s12=false,k1=0,k16=0," + "s10=false,s21=false,k12=0,m5=600,s8=false,m4=600,m9=1107,s2=false,s13=false,s20=false,s23=false,k9=0,m11=1525," + "s4=false,s6=false 1672818929178749400", + "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 " + "k2=0,k11=0,m3=600,m12=1506,s17=false,m5=600,s11=false,s22=false,k6=0,m13=0,s16=true,k5=0,s21=false,m4=600,m7=" + "600,s9=false,s10=false,s18=false,k7=0,m8=600,k1=0,hex=" + "\"7b3a00000001030301030200000000323231313233304339344b30002b01012a10028003000000071105e8025802580258025802580258" + "044905eb05ef05e200000000000000000afc7d\",m11=1519,m16=0,s19=false,s23=false,s24=false,s14=false,s6=false,k10=0," + "k15=0,k14=0,s2=false,s4=false,s8=false,s13=false,s15=true,s20=false,m2=38.000005040000005,s3=false,s7=false,k3=" + "0,k8=0,k13=0,m6=600,m14=0,m15=0,k4=0,m1=20.450000000000003,m9=1097,s1=true,m10=1515,s5=false,s12=false,k9=0,k12=" + "0,k16=0 1672818919126971000", + "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 " + "k7=0,k14=0,m3=600,m7=600,s5=false,k2=0,k3=0,k8=0,s3=false,s20=false,k15=0,m10=1482,s17=false,k1=0,k16=0,m15=0," + "s12=false,k9=0,m16=0,s11=false,m4=600,s10=false,s15=true,s24=false,m8=600,m13=0,s2=false,s18=false,k12=0,s14=" + "false,s19=false,hex=" + "\"7b3900000001030301030200000000323231313233304339344b30002b01012a10028003000000071505ef025802580258025802580258" + "045005ca05b105d800000000000000000aa47d\",s1=true,s4=false,s7=false,s8=false,s13=false,m6=600,s6=false,s21=false," + "k11=0,m12=1496,m9=1104,s16=true,k5=0,s9=false,k10=0,k13=0,m2=38.291671730000004,s22=false,m5=600,m11=1457,m14=0," + "k4=0,m1=20.650000000000006,s23=false,k6=0 1672818909130866800", + "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 " + "m7=600,k4=0,k14=0,s22=false,k13=0,s2=false,m11=1510,m14=0,s4=false,s10=false,m1=21,m16=0,m13=0,s9=false,s13=" + "false,s14=false,k10=0,m3=600,m9=1107,s18=false,s19=false,k2=0,hex=" + "\"7b3600000001030301030200000000323231313233304339344b30002b01012a10028003000000071c0619025802580258025802580258" + "045305dc05e6058d00000000000000000ad27d\",m2=40.04167187,m8=600,k7=0,k8=0,m10=1500,s23=false,k5=0,s11=false,s21=" + "false,k9=0,m15=0,m12=1421,s1=true,s5=false,s8=false,m5=600,k16=0,k15=0,m6=600,s3=false,s6=false,s7=false,s15=" + "true,s20=false,s24=false,k11=0,k1=0,k6=0,k12=0,m4=600,s16=true,s17=false,k3=0,s12=false 1672818879189483200", + "DataRTU,deviceId=2106070C11M0_2,dataModelName=DataRTU_2106070C11M0_2 " + "m1=5691,k14=0,m6=0,s14=false,k8=0,s19=false,s20=false,k12=0,s17=false,k3=0,m8=0,s8=false,m7=0,s9=false,s4=false," + "s11=false,s13=false,s16=false,k5=0,k15=0,k16=0,s10=false,s23=false,s1=false,s2=false,s3=false,s12=false,s24=" + "false,k2=0,k10=0,hex=" + "\"7b1400000001030301030200000000323130363037304331314d30002b01022a080400000000000008af0c000000000000000000000000" + "000000000000000000000000000000000ad47d\",m2=0,s7=false,s18=false,s21=false,m3=0,m5=0,k4=0,k11=0,m4=0,k1=0,k6=0," + "k13=0,s6=false,s15=false,s5=false,s22=false,k7=0,k9=0 1672818779549848800"}; + pRes = taos_query(taos, "use ts2385"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + ASSERT(!code); + taos_free_result(pRes); + + pRes = taos_query(taos, "select distinct tbname from `DataRTU` order by tbname"); + printf("%s result2:%s\n", __FUNCTION__, taos_errstr(pRes)); + int num = 0; + TAOS_ROW row = NULL; + while ((row = taos_fetch_row(pRes))) { + if (row[0] != NULL && num == 0) { + ASSERT(strncmp((char *)row[0], "DataRTU_2106070C11M0_2", sizeof("DataRTU_2106070C11M0_2") - 1) == 0); + } + + if (row[0] != NULL && num == 1) { + ASSERT(strncmp((char *)row[0], "DataRTU_2211230C94K0_1", sizeof("DataRTU_2211230C94K0_1") - 1) == 0); + } + num++; + } + ASSERT(num == 2); + + code = taos_errno(pRes); + taos_free_result(pRes); + taos_close(taos); + + return code; +} + int main(int argc, char *argv[]) { + if (argc == 2) { + taos_options(TSDB_OPTION_CONFIGDIR, argv[1]); + } + int ret = 0; + ret = sml_ts2385_Test(); + ASSERT(!ret); + // for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){ + // printf("str:%s \t %d\n", str[i], smlCalTypeSum(str[i], strlen(str[i]))); + // } + // int ret = 0; ret = sml_ttl_Test(); ASSERT(!ret); ret = sml_ts2164_Test(); @@ -1157,8 +1113,6 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = smlProcess_json3_Test(); ASSERT(!ret); - ret = smlProcess_json4_Test(); - ASSERT(!ret); ret = sml_TD15662_Test(); ASSERT(!ret); ret = sml_TD15742_Test(); @@ -1167,12 +1121,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_oom_Test(); ASSERT(!ret); - ret = sml_16368_Test(); - ASSERT(!ret); ret = sml_dup_time_Test(); ASSERT(!ret); - ret = sml_16960_Test(); - ASSERT(!ret); ret = sml_add_tag_col_Test(); ASSERT(!ret); ret = smlProcess_18784_Test(); diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 024a253a2e2aad97b0639468b79ff04cb177b8a4..69bf52bb1ac1c80015e12a6ecf6701dc73f8d139 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -78,28 +78,27 @@ static void msg_process(TAOS_RES* msg) { int buildDatabase(TAOS* pConn, TAOS_RES* pRes){ /* test for TD-20612 start*/ -// pRes = taos_query(pConn,"create table tb1 (ts timestamp, c1 int, c2 int)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn,"insert into tb1 (ts, c1) values(1669092069069, 0)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn,"insert into tb1 (ts, c2) values(1669092069069, 1)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// return 0; + pRes = taos_query(pConn,"create table tb1 (ts timestamp, c1 int, c2 int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn,"insert into tb1 (ts, c1) values(1669092069069, 0)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn,"insert into tb1 (ts, c2) values(1669092069069, 1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + /* test for TD-20612 end*/ pRes = taos_query(pConn, @@ -614,6 +613,7 @@ void initLogFile() { } }else{ char *result[] = { + "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"tb1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":4}],\"tags\":[]}", "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":64},{\"name\":\"c4\",\"type\":5}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1},{\"name\":\"t2\",\"type\":8,\"length\":64}]}", "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}", @@ -652,6 +652,7 @@ void initLogFile() { } }else{ char *result[] = { + "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"tb1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":4}],\"tags\":[]}", "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}", "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}", diff --git a/utils/tsim/inc/simInt.h b/utils/tsim/inc/simInt.h index f512b119b4299c9146a3a9a915a2359ca52fc4c4..8ff3f3b18344eaa3f9533d704b401c0eab757739 100644 --- a/utils/tsim/inc/simInt.h +++ b/utils/tsim/inc/simInt.h @@ -181,7 +181,7 @@ typedef struct _script_t { extern SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; extern SCommand simCmdList[]; extern int32_t simScriptPos; -extern int32_t simScriptSucced; +extern int32_t simScriptSucceed; extern int32_t simDebugFlag; extern char simScriptDir[]; extern bool abortExecution; diff --git a/utils/tsim/src/simSystem.c b/utils/tsim/src/simSystem.c index f2fefb903d926d38d16544253a1c6cae0fc1b5c9..98f9217fd6e55bff306d06aa25151ad713e743ca 100644 --- a/utils/tsim/src/simSystem.c +++ b/utils/tsim/src/simSystem.c @@ -20,7 +20,7 @@ SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; SCommand simCmdList[SIM_CMD_END]; int32_t simScriptPos = -1; -int32_t simScriptSucced = 0; +int32_t simScriptSucceed = 0; int32_t simDebugFlag = 143; void simCloseTaosdConnect(SScript *script); char simScriptDir[PATH_MAX] = {0}; @@ -88,13 +88,13 @@ SScript *simProcessCallOver(SScript *script) { } simCloseTaosdConnect(script); - simScriptSucced++; + simScriptSucceed++; simScriptPos--; simFreeScript(script); if (simScriptPos == -1 && simExecSuccess) { simInfo("----------------------------------------------------------------------"); - simInfo("Simulation Test Done, " SUCCESS_PREFIX "%d" SUCCESS_POSTFIX " Passed:\n", simScriptSucced); + simInfo("Simulation Test Done, " SUCCESS_PREFIX "%d" SUCCESS_POSTFIX " Passed:\n", simScriptSucceed); return NULL; }