diff --git a/cmake/cmake.define b/cmake/cmake.define index d32200bb9145672e18af24fb158c68e802fcf45d..a739a77d21c40d6b9b5a9ccafe0e3254d9a0d215 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -1,6 +1,7 @@ cmake_minimum_required(VERSION 3.0) set(CMAKE_VERBOSE_MAKEFILE OFF) +set(TD_BUILD_TAOSA_INTERNAL FALSE) #set output directory SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib) diff --git a/cmake/cmake.install b/cmake/cmake.install index fd1e080ddab1478f73689e7cced405ae8404fbc2..67634625ce53c6ea48dc03746a39fdb03975c2af 100644 --- a/cmake/cmake.install +++ b/cmake/cmake.install @@ -21,7 +21,7 @@ IF (TD_LINUX) ELSEIF (TD_WINDOWS) SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.bat") INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") - INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} :needAdmin ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Windows ${TD_VER_NUMBER})") + INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} :needAdmin ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Windows ${TD_VER_NUMBER} ${TD_BUILD_TAOSA_INTERNAL})") ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.sh") INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index a47b3b0febcbe968273ffae4cc428abdb6347a1a..3e2e879e389fc7f6686949efab43bc5fada33f3a 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG f0c1753 + GIT_TAG a2e9920 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/en/12-taos-sql/02-database.md b/docs/en/12-taos-sql/02-database.md index 0aec22fbc072409916cfa7a7a8ff0d1702fb226b..c3c7e5928bb6705939dfae7d4e0096b202025520 100644 --- a/docs/en/12-taos-sql/02-database.md +++ b/docs/en/12-taos-sql/02-database.md @@ -30,6 +30,8 @@ database_option: { | WAL_LEVEL {1 | 2} | VGROUPS value | SINGLE_STABLE {0 | 1} + | TABLE_PREFIX value + | TABLE_SUFFIX value | WAL_RETENTION_PERIOD value | WAL_ROLL_PERIOD value | WAL_RETENTION_SIZE value @@ -67,6 +69,8 @@ database_option: { - SINGLE_STABLE: specifies whether the database can contain more than one supertable. - 0: The database can contain multiple supertables. - 1: The database can contain only one supertable. +- TABLE_PREFIX：The prefix length in the table name that is ignored when distributing table to vnode based on table name. +- TABLE_SUFFIX：The suffix length in the table name that is ignored when distributing table to vnode based on table name. - WAL_RETENTION_PERIOD: specifies the time after which WAL files are deleted. This parameter is used for data subscription. Enter a time in seconds. The default value of single copy is 0. A value of 0 indicates that each WAL file is deleted immediately after its contents are written to disk. -1: WAL files are never deleted. The default value of multiple copy is 4 days. - WAL_RETENTION_SIZE: specifies the size at which WAL files are deleted. This parameter is used for data subscription. Enter a size in KB. The default value of single copy is 0. A value of 0 indicates that each WAL file is deleted immediately after its contents are written to disk. -1: WAL files are never deleted. The default value of multiple copy is -1. - WAL_ROLL_PERIOD: specifies the time after which WAL files are rotated. After this period elapses, a new WAL file is created. The default value of single copy is 0. A value of 0 indicates that a new WAL file is created only after the previous WAL file was written to disk. The default values of multiple copy is 1 day. diff --git a/docs/en/12-taos-sql/20-keywords.md b/docs/en/12-taos-sql/20-keywords.md index 4b479b866b77e1e354d20376ccb869755af76d00..23f85947e30ae2876cf631124e6485c09f2ff8da 100644 --- a/docs/en/12-taos-sql/20-keywords.md +++ b/docs/en/12-taos-sql/20-keywords.md @@ -17,6 +17,7 @@ The following list shows all reserved keywords: - ADD - AFTER - AGGREGATE +- ALIVE - ALL - ALTER - ANALYZE diff --git a/docs/en/12-taos-sql/24-show.md b/docs/en/12-taos-sql/24-show.md index 08e8df6252e406a6910d979657c2c231ad43a71a..2db3e7cb31463e20f024f48e62d06422519ba0e7 100644 --- a/docs/en/12-taos-sql/24-show.md +++ b/docs/en/12-taos-sql/24-show.md @@ -178,76 +178,138 @@ SHOW TABLE DISTRIBUTED table_name; Shows how table data is distributed. -Examples： show table distributed d0\G; Display the block distribution of table `d0` in detailed format. +Examples： Below is an example of this command to display the block distribution of table `d0` in detailed format. +```sql +show table distributed d0\G; +``` + +
+ Show Example + 

 *************************** 1.row ***************************
 _block_dist: Total_Blocks=[5] Total_Size=[93.65 Kb] Average_size=[18.73 Kb] Compression_Ratio=[23.98 %]
 
 Total_Blocks :  Table `d0` contains total 5 blocks
+
 Total_Size:  The total size of all the data blocks in table `d0` is 93.65 KB 
+
 Average_size:  The average size of each block is 18.73 KB
+
 Compression_Ratio: The data compression rate is 23.98%
  
 *************************** 2.row ***************************
 _block_dist: Total_Rows=[20000] Inmem_Rows=[0] MinRows=[3616] MaxRows=[4096] Average_Rows=[4000]
 
 Total_Rows: Table `d0` contains 20,000 rows
+
 Inmem_Rows： The rows still in memory, i.e. not committed in disk, is 0, i.e. none such rows
+
 MinRows：  The minimum number of rows in a block is 3,616 
+
 MaxRows： The maximum number of rows in a block is 4,096B
+
 Average_Rows： The average number of rows in a block is 4,000
 
 *************************** 3.row ***************************
 _block_dist: Total_Tables=[1] Total_Files=[2]
 
 Total_Tables:  The number of child tables, 1 in this example
+
 Total_Files：   The number of files storing the table's data, 2 in this example
 
 *************************** 4.row ***************************
+
 _block_dist: --------------------------------------------------------------------------------
+
 *************************** 5.row ***************************
+
 _block_dist: 0100 |
+
 *************************** 6.row ***************************
+
 _block_dist: 0299 |
+
 *************************** 7.row ***************************
+
 _block_dist: 0498 |
+
 *************************** 8.row ***************************
+
 _block_dist: 0697 |
+
 *************************** 9.row ***************************
+
 _block_dist: 0896 |
+
 *************************** 10.row ***************************
+
 _block_dist: 1095 |
+
 *************************** 11.row ***************************
+
 _block_dist: 1294 |
+
 *************************** 12.row ***************************
+
 _block_dist: 1493 |
+
 *************************** 13.row ***************************
+
 _block_dist: 1692 |
+
 *************************** 14.row ***************************
+
 _block_dist: 1891 |
+
 *************************** 15.row ***************************
+
 _block_dist: 2090 |
+
 *************************** 16.row ***************************
+
 _block_dist: 2289 |
+
 *************************** 17.row ***************************
+
 _block_dist: 2488 |
+
 *************************** 18.row ***************************
+
 _block_dist: 2687 |
+
 *************************** 19.row ***************************
+
 _block_dist: 2886 |
+
 *************************** 20.row ***************************
+
 _block_dist: 3085 |
+
 *************************** 21.row ***************************
+
 _block_dist: 3284 |
+
 *************************** 22.row ***************************
+
 _block_dist: 3483 |||||||||||||||||  1 (20.00%)
+
 *************************** 23.row ***************************
+
 _block_dist: 3682 |
+
 *************************** 24.row ***************************
+
 _block_dist: 3881 |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||  4 (80.00%)
+
 Query OK, 24 row(s) in set (0.002444s)
 
-  The above show the block distribution percentage according to the number of rows in each block. In the above example, `_block_dist: 3483 |||||||||||||||||  1 (20.00%)` means there is one block whose rows is between 3,483 and 3,681. `_block_dist: 3881 |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||  4 (80.00%)` means there are 4 blocks whose rows is between 3,881 and 4,096. The number of blocks whose rows fall in other range is zero.
+
+
+ + The above show the block distribution percentage according to the number of rows in each block. In the above example, we can get below information: + - `_block_dist: 3483 ||||||||||||||||| 1 (20.00%)` means there is one block whose rows is between 3,483 and 3,681. + - `_block_dist: 3881 ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| 4 (80.00%)` means there are 4 blocks whose rows is between 3,881 and 4,096. - The number of blocks whose rows fall in other range is zero. ## SHOW TAGS diff --git a/docs/en/14-reference/04-taosadapter.md b/docs/en/14-reference/04-taosadapter.md index 870cebb1033f0f0cddaec19b616228c2ac46d87c..9eb6cb921391df3ce494fddaae8caa23cb0f4adc 100644 --- a/docs/en/14-reference/04-taosadapter.md +++ b/docs/en/14-reference/04-taosadapter.md @@ -21,6 +21,7 @@ taosAdapter provides the following features. - Seamless connection to collectd - Seamless connection to StatsD - Supports Prometheus remote_read and remote_write +- Get table's VGroup ID ## taosAdapter architecture diagram @@ -178,6 +179,7 @@ See [example/config/taosadapter.toml](https://github.com/taosdata/taosadapter/bl node_export is an exporter for machine metrics. Please visit [https://github.com/prometheus/node_exporter](https://github.com/prometheus/node_exporter) for more information. - Support for Prometheus remote_read and remote_write remote_read and remote_write are interfaces for Prometheus data read and write from/to other data storage solution. Please visit [https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis) for more information. +- Get table's VGroup ID. For more information about VGroup, please refer to [primary-logic-unit](/tdinternal/arch/#primary-logic-unit). ## Interfaces @@ -199,7 +201,7 @@ Support InfluxDB query parameters as follows. - `precision` The time precision used by TDengine - `u` TDengine user name - `p` TDengine password -- `ttl` The time to live of automatically created sub-table. This value cannot be updated. TDengine will use the ttl value of the frist data of sub-table to create sub-table. For more information, please refer [Create Table](/taos-sql/table/#create-table) +- `ttl` The time to live of automatically created sub-table. This value cannot be updated. TDengine will use the ttl value of the first data of sub-table to create sub-table. For more information, please refer [Create Table](/taos-sql/table/#create-table) Note: InfluxDB token authorization is not supported at present. Only Basic authorization and query parameter validation are supported. Example: curl --request POST http://127.0.0.1:6041/influxdb/v1/write?db=test --user "root:taosdata" --data-binary "measurement,host=host1 field1=2i,field2=2.0 1577836800000000000" @@ -241,6 +243,10 @@ node_export is an exporter of hardware and OS metrics exposed by the \*NIX kerne +### Get table's VGroup ID + +You can call `http://:6041/rest/vgid?db=&table=` to get table's VGroup ID. For more information about VGroup, please refer to [primary-logic-unit](/tdinternal/arch/#primary-logic-unit). + ## Memory usage optimization methods taosAdapter will monitor its memory usage during operation and adjust it with two thresholds. Valid values are integers between 1 to 100, and represent a percentage of the system's physical memory. diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index bcfcaf9ffb37206b92f265435f577b47aa452807..9b83c5fd65a5aa364892b608ffb8b25bd56f66b4 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://w import Release from "/components/ReleaseV3"; +## 3.0.2.2 + + + ## 3.0.2.1 diff --git a/docs/en/28-releases/02-tools.md b/docs/en/28-releases/02-tools.md index 6013aacc35692dc75c15de4fd2427bfaee36ea34..dd44e43ab63a7ea227665706f7f7e593a3b81182 100644 --- a/docs/en/28-releases/02-tools.md +++ b/docs/en/28-releases/02-tools.md @@ -10,6 +10,10 @@ For other historical version installers, please visit [here](https://www.taosdat import Release from "/components/ReleaseV3"; +## 2.4.0 + + + ## 2.3.3 diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index 232a6c7326ba10740f9e8606254ae97591dc9084..df52a0890b6ff091b8d5cfb051618c88e8195cf0 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -30,6 +30,8 @@ database_option: { | WAL_LEVEL {1 | 2} | VGROUPS value | SINGLE_STABLE {0 | 1} + | TABLE_PREFIX value + | TABLE_SUFFIX value | WAL_RETENTION_PERIOD value | WAL_ROLL_PERIOD value | WAL_RETENTION_SIZE value @@ -67,6 +69,8 @@ database_option: { - SINGLE_STABLE：表示此数据库中是否只可以创建一个超级表，用于超级表列非常多的情况。 - 0：表示可以创建多张超级表。 - 1：表示只可以创建一张超级表。 +- TABLE_PREFIX：内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的前缀的长度。 +- TABLE_SUFFIX：内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的后缀的长度。 - WAL_RETENTION_PERIOD：wal 文件的额外保留策略，用于数据订阅。wal 的保存时长，单位为 s。单副本默认为 0，即落盘后立即删除。-1 表示不删除。多副本默认为 4 天。 - WAL_RETENTION_SIZE：wal 文件的额外保留策略，用于数据订阅。wal 的保存的最大上限，单位为 KB。单副本默认为 0，即落盘后立即删除。多副本默认为-1，表示不删除。 - WAL_ROLL_PERIOD：wal 文件切换时长，单位为 s。当 wal 文件创建并写入后，经过该时间，会自动创建一个新的 wal 文件。单副本默认为 0，即仅在落盘时创建新文件。多副本默认为 1 天。 diff --git a/docs/zh/12-taos-sql/20-keywords.md b/docs/zh/12-taos-sql/20-keywords.md index 8013698fced2f31ca21dfa220066b027e71cb856..8fd704ef551c9ebda61678bd129917e67cbfbef8 100644 --- a/docs/zh/12-taos-sql/20-keywords.md +++ b/docs/zh/12-taos-sql/20-keywords.md @@ -18,6 +18,7 @@ description: TDengine 保留关键字的详细列表 - ADD - AFTER - AGGREGATE +- ALIVE - ALL - ALTER - ANALYZE diff --git a/docs/zh/12-taos-sql/24-show.md b/docs/zh/12-taos-sql/24-show.md index a65746e7f93f438404703886127b56a325486daf..2b875199b501136afef4ce0ba1cbcdd1e0abc933 100644 --- a/docs/zh/12-taos-sql/24-show.md +++ b/docs/zh/12-taos-sql/24-show.md @@ -183,6 +183,10 @@ SHOW TABLE DISTRIBUTED table_name; 语句： show table distributed d0\G; 竖行显示表 d0 的 BLOCK 分布情况 +
+ 显示示例 + 

+
 *************************** 1.row ***************************
 
 _block_dist: Total_Blocks=[5] Total_Size=[93.65 Kb] Average_size=[18.73 Kb] Compression_Ratio=[23.98 %]
@@ -244,6 +248,8 @@ _block_dist: 3881 ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
 
 Query OK, 24 row(s) in set (0.002444s)
 
+
+
上面是块中包含数据行数的块儿分布情况图，这里的 0100 0299 0498 … 表示的是每个块中包含的数据行数，上面的意思就是这个表的 5 个块，分布在 3483 ~3681 行的块有 1 个，占整个块的 20%，分布在 3881 ~ 4096（最大行数）的块数为 4 个，占整个块的 80%， 其它区域内分布块数为 0。 diff --git a/docs/zh/12-taos-sql/29-changes.md b/docs/zh/12-taos-sql/29-changes.md index 1ab7f4f0169932576edf9087a825367573713b03..1e421f95e15f2465b0d71a48947f540a994baa71 100644 --- a/docs/zh/12-taos-sql/29-changes.md +++ b/docs/zh/12-taos-sql/29-changes.md @@ -94,6 +94,7 @@ description: "TDengine 3.0 版本的语法变更说明" | 9 | SAMPLE | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 | 10 | STATECOUNT | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 | 11 | STATEDURATION | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 +| 12 | TIMETRUNCATE | 增强 | 增加ignore_timezone参数，可选是否使用，默认值为1. ## SCHEMALESS 变更 diff --git a/docs/zh/14-reference/04-taosadapter.md b/docs/zh/14-reference/04-taosadapter.md index 0909ddf639b621b39af26e7bd378f8bf7e3b99e7..5c155bdd6ebe9f1a4a38886f30b753cbda4ab6ab 100644 --- a/docs/zh/14-reference/04-taosadapter.md +++ b/docs/zh/14-reference/04-taosadapter.md @@ -21,6 +21,7 @@ taosAdapter 提供以下功能： - 无缝连接到 collectd - 无缝连接到 StatsD - 支持 Prometheus remote_read 和 remote_write +- 获取 table 所在的虚拟节点组（VGroup）的 VGroup ID ## taosAdapter 架构图 @@ -178,6 +179,7 @@ AllowWebSockets node_export 是一个机器指标的导出器。请访问 [https://github.com/prometheus/node_exporter](https://github.com/prometheus/node_exporter) 了解更多信息。 - 支持 Prometheus remote_read 和 remote_write remote_read 和 remote_write 是 Prometheus 数据读写分离的集群方案。请访问[https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis) 了解更多信息。 +- 获取 table 所在的虚拟节点组（VGroup）的 VGroup ID。关于虚拟节点组（VGroup）的更多信息，请访问[整体架构文档](/tdinternal/arch/#主要逻辑单元) 。 ## 接口 @@ -240,6 +242,10 @@ Prometheus 使用的由 \*NIX 内核暴露的硬件和操作系统指标的输 +### 获取 table 的 VGroup ID + +可以访问 http 接口 `http://:6041/rest/vgid?db=&table=
` 获取 table 的 VGroup ID。关于虚拟节点组（VGroup）的更多信息，请访问[整体架构文档](/tdinternal/arch/#主要逻辑单元) 。 + ## 内存使用优化方法 taosAdapter 将监测自身运行过程中内存使用率并通过两个阈值进行调节。有效值范围为 -1 到 100 的整数，单位为系统物理内存的百分比。 @@ -282,7 +288,7 @@ http 返回内容： ## taosAdapter 监控指标 -taosAdapter 采集 http 相关指标、cpu 百分比和内存百分比。 +taosAdapter 采集 http 相关指标、CPU 百分比和内存百分比。 ### http 接口 @@ -294,13 +300,13 @@ http://:6041/metrics ### 写入 TDengine -taosAdapter 支持将 http 监控、cpu 百分比和内存百分比写入 TDengine。 +taosAdapter 支持将 http 监控、CPU 百分比和内存百分比写入 TDengine。 有关配置参数 | **配置项** | **描述** | **默认值** | |-------------------------|--------------------------------------------|----------| -| monitor.collectDuration | cpu 和内存采集间隔 | 3s | +| monitor.collectDuration | CPU 和内存采集间隔 | 3s | | monitor.identity | 当前taosadapter 的标识符如果不设置将使用 'hostname:port' | | | monitor.incgroup | 是否是 cgroup 中运行(容器中运行设置为 true) | false | | monitor.writeToTD | 是否写入到 TDengine | false | diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index 0fe6555162dc2b043288bfba882dab621916f35b..e19b2133e4e1a1ff67d3d7e289055f58397c8ece 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.0.2.2 + + + ## 3.0.2.1 diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md index 331f6832c00888941f4c1a22565a3d5d62512ade..b2ead5b2640f3fcf221e19e66eb4971e7daaa911 100644 --- a/docs/zh/28-releases/02-tools.md +++ b/docs/zh/28-releases/02-tools.md @@ -10,6 +10,10 @@ taosTools 各版本安装包下载链接如下： import Release from "/components/ReleaseV3"; +## 2.4.0 + + + ## 2.3.3 diff --git a/include/common/tcommon.h b/include/common/tcommon.h index aad69862f0ec5591fccbf0e0f8e024c34e426be2..9ecbd2f839411ffe8e93946713941c39ac718d60 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -341,7 +341,7 @@ typedef struct SExprInfo { typedef struct { const char* key; - int32_t keyLen; + size_t keyLen; uint8_t type; union { const char* value; @@ -350,7 +350,7 @@ typedef struct { double d; float f; }; - int32_t length; + size_t length; } SSmlKv; #define QUERY_ASC_FORWARD_STEP 1 diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index e1d3b016113e6a117f81d2ceaf2034da38e7879c..d9b8ae266b358507a8b396180b67c1bb536bdf18 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -265,7 +265,7 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag); // for debug char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf); -int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataBlocks, STSchema* pTSchema, int32_t vgId, +int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pDataBlocks, const STSchema* pTSchema, int64_t uid, int32_t vgId, tb_uid_t suid); char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 6855287fb2d354f52998342d368c46b3a852bf89..e0aacbfec9db8804b4003417689f404b78549afb 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -44,18 +44,38 @@ typedef struct SColData SColData; #define HAS_VALUE ((uint8_t)0x4) // bitmap ================================ -const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, - {0b00000000, 0b00000100, 0b00001000, 2}, - {0b00000000, 0b00010000, 0b00100000, 4}, - {0b00000000, 0b01000000, 0b10000000, 6}}; - -#define N1(n) ((((uint8_t)1) << (n)) - 1) -#define BIT1_SIZE(n) ((((n)-1) >> 3) + 1) -#define BIT2_SIZE(n) ((((n)-1) >> 2) + 1) -#define SET_BIT1(p, i, v) ((p)[(i) >> 3] = (p)[(i) >> 3] & N1((i)&7) | (((uint8_t)(v)) << ((i)&7))) -#define GET_BIT1(p, i) (((p)[(i) >> 3] >> ((i)&7)) & ((uint8_t)1)) -#define SET_BIT2(p, i, v) ((p)[(i) >> 2] = (p)[(i) >> 2] & N1(BIT2_MAP[(i)&3][3]) | BIT2_MAP[(i)&3][(v)]) -#define GET_BIT2(p, i) (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3)) +const static uint8_t BIT1_MAP[8] = {0b11111110, 0b11111101, 0b11111011, 0b11110111, + 0b11101111, 0b11011111, 0b10111111, 0b01111111}; + +const static uint8_t BIT2_MAP[4] = {0b11111100, 0b11110011, 0b11001111, 0b00111111}; + +#define ONE ((uint8_t)1) +#define THREE ((uint8_t)3) +#define DIV_8(i) ((i) >> 3) +#define MOD_8(i) ((i)&7) +#define DIV_4(i) ((i) >> 2) +#define MOD_4(i) ((i)&3) +#define MOD_4_TIME_2(i) (MOD_4(i) << 1) +#define BIT1_SIZE(n) (DIV_8((n)-1) + 1) +#define BIT2_SIZE(n) (DIV_4((n)-1) + 1) +#define SET_BIT1(p, i, v) ((p)[DIV_8(i)] = (p)[DIV_8(i)] & BIT1_MAP[MOD_8(i)] | ((v) << MOD_8(i))) +#define SET_BIT1_EX(p, i, v) \ + do { \ + if (MOD_8(i) == 0) { \ + (p)[DIV_8(i)] = 0; \ + } \ + SET_BIT1(p, i, v); \ + } while (0) +#define GET_BIT1(p, i) (((p)[DIV_8(i)] >> MOD_8(i)) & ONE) +#define SET_BIT2(p, i, v) ((p)[DIV_4(i)] = (p)[DIV_4(i)] & BIT2_MAP[MOD_4(i)] | ((v) << MOD_4_TIME_2(i))) +#define SET_BIT2_EX(p, i, v) \ + do { \ + if (MOD_4(i) == 0) { \ + (p)[DIV_4(i)] = 0; \ + } \ + SET_BIT2(p, i, v); \ + } while (0) +#define GET_BIT2(p, i) (((p)[DIV_4(i)] >> MOD_4_TIME_2(i)) & THREE) // SBuffer ================================ struct SBuffer { @@ -70,9 +90,6 @@ int32_t tBufferInit(SBuffer *pBuffer, int64_t size); int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData); int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData); -// STSchema ================================ -void tDestroyTSchema(STSchema *pTSchema); - // SColVal ================================ #define CV_FLAG_VALUE ((int8_t)0x0) #define CV_FLAG_NONE ((int8_t)0x1) @@ -87,8 +104,12 @@ void tDestroyTSchema(STSchema *pTSchema); #define COL_VAL_IS_VALUE(CV) ((CV)->flag == CV_FLAG_VALUE) // SRow ================================ -int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer); +int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow); void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); +void tRowDestroy(SRow *pRow); +void tRowSort(SArray *aRowP); +int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag); +int32_t tRowAppendToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData); // SRowIter ================================ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter); @@ -110,15 +131,28 @@ void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remov int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf); // SColData ================================ +typedef void *(*xMallocFn)(void *, int32_t); void tColDataDestroy(void *ph); void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn); void tColDataClear(SColData *pColData); +void tColDataDeepClear(SColData *pColData); int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal); void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal); uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal); -int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest); +int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMalloc, void *arg); extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull); +// for stmt bind +int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind); +void tColDataSortMerge(SArray *colDataArr); + +//for raw block +int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t bytes, + int32_t nRows, char* lengthOrbitmap, char *data); +// for encode/decode +int32_t tPutColData(uint8_t *pBuf, SColData *pColData); +int32_t tGetColData(uint8_t *pBuf, SColData *pColData); + // STRUCT ================================ struct STColumn { col_id_t colId; @@ -225,23 +259,9 @@ struct STag { memcpy(varDataVal(x), (str), (_size)); \ } while (0); -// ----------------- SCHEMA BUILDER DEFINITION -typedef struct { - int32_t tCols; - int32_t nCols; - schema_ver_t version; - uint16_t flen; - int32_t tlen; - STColumn *columns; -} STSchemaBuilder; - -int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); -void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); -void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); -int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes); -STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); - +// STSchema ================================ STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version); +void tDestroyTSchema(STSchema *pTSchema); #endif diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 9e8a139b31abe31d2a382fe1c6c30282c5bf2a27..d445fc26e882d840a3a3df87464c43d0225cbb5e 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -55,7 +55,7 @@ extern int32_t tsNumOfMnodeQueryThreads; extern int32_t tsNumOfMnodeFetchThreads; extern int32_t tsNumOfMnodeReadThreads; extern int32_t tsNumOfVnodeQueryThreads; -extern int32_t tsNumOfVnodeStreamThreads; +extern float tsRatioOfVnodeStreamThreads; extern int32_t tsNumOfVnodeFetchThreads; extern int32_t tsNumOfVnodeRsmaThreads; extern int32_t tsNumOfQnodeQueryThreads; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ad6077db098b18d2b10d95058831d4f8c25d046a..800f9e2eb7b8e9806d1c64eb6829c1191851439f 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -482,8 +482,6 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapperEx(SDecoder* pDecoder, SSchemaW return 0; } -STSchema* tdGetSTSChemaFromSSChema(SSchema* pSchema, int32_t nCols, int32_t sver); - typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igExists; @@ -1734,6 +1732,8 @@ typedef struct { int32_t execId; } STaskDropReq; +int32_t tSerializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); +int32_t tDeserializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); int32_t tSerializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); int32_t tDeserializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); @@ -1751,6 +1751,8 @@ typedef struct { #define STREAM_FILL_HISTORY_ON 1 #define STREAM_FILL_HISTORY_OFF 0 #define STREAM_DEFAULT_FILL_HISTORY STREAM_FILL_HISTORY_OFF +#define STREAM_CREATE_STABLE_TRUE 1 +#define STREAM_CREATE_STABLE_FALSE 0 typedef struct { char name[TSDB_STREAM_FNAME_LEN]; @@ -1768,6 +1770,8 @@ typedef struct { SArray* pTags; // array of SField // 3.0.20 int64_t checkpointFreq; // ms + // 3.0.2.3 + int8_t createStb; } SCMCreateStreamReq; typedef struct { @@ -2081,10 +2085,15 @@ typedef struct SVCreateTbReq { }; } SVCreateTbReq; -int tEncodeSVCreateTbReq(SEncoder* pCoder, const SVCreateTbReq* pReq); -int tDecodeSVCreateTbReq(SDecoder* pCoder, SVCreateTbReq* pReq); +int tEncodeSVCreateTbReq(SEncoder* pCoder, const SVCreateTbReq* pReq); +int tDecodeSVCreateTbReq(SDecoder* pCoder, SVCreateTbReq* pReq); +void tDestroySVCreateTbReq(SVCreateTbReq* pReq, int32_t flags); static FORCE_INLINE void tdDestroySVCreateTbReq(SVCreateTbReq* req) { + if (NULL == req) { + return; + } + taosMemoryFreeClear(req->name); taosMemoryFreeClear(req->comment); if (req->type == TSDB_CHILD_TABLE) { @@ -3232,6 +3241,57 @@ int32_t tSerializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); int32_t tDeserializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); int32_t tSerializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); int32_t tDeserializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); +int32_t tSerializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); +int32_t tDeserializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); +int32_t tSerializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); +int32_t tDeserializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); + +#define SUBMIT_REQ_AUTO_CREATE_TABLE 0x1 +#define SUBMIT_REQ_COLUMN_DATA_FORMAT 0x2 + +typedef struct { + int32_t flags; + SVCreateTbReq* pCreateTbReq; + int64_t suid; + int64_t uid; + int32_t sver; + union { + SArray* aRowP; + SArray* aCol; + }; +} SSubmitTbData; + +typedef struct { + SArray* aSubmitTbData; // SArray +} SSubmitReq2; + +int32_t tEncodeSSubmitReq2(SEncoder* pCoder, const SSubmitReq2* pReq); +int32_t tDecodeSSubmitReq2(SDecoder* pCoder, SSubmitReq2* pReq); +void tDestroySSubmitTbData(SSubmitTbData* pTbData, int32_t flag); +void tDestroySSubmitReq2(SSubmitReq2* pReq, int32_t flag); + +typedef struct { + int32_t affectedRows; + SArray* aCreateTbRsp; // SArray +} SSubmitRsp2; + +int32_t tEncodeSSubmitRsp2(SEncoder* pCoder, const SSubmitRsp2* pRsp); +int32_t tDecodeSSubmitRsp2(SDecoder* pCoder, SSubmitRsp2* pRsp); +void tDestroySSubmitRsp2(SSubmitRsp2* pRsp, int32_t flag); + +#define TSDB_MSG_FLG_ENCODE 0x1 +#define TSDB_MSG_FLG_DECODE 0x2 + +typedef struct { + union { + struct { + void* msgStr; + int32_t msgLen; + int64_t ver; + }; + void* pDataBlock; + }; +} SPackedData; #pragma pack(pop) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 412b4b4cf6c950113a2f7e6d319692695cde0c07..871dd6074185569135cef7b36f8a957b693cb990 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -190,7 +190,9 @@ int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts); int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType); -int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq); +// int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq, int64_t ver); +// +int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit); int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset); @@ -213,6 +215,7 @@ int32_t qStreamSourceRecoverStep1(qTaskInfo_t tinfo, int64_t ver); int32_t qStreamSourceRecoverStep2(qTaskInfo_t tinfo, int64_t ver); int32_t qStreamRecoverFinish(qTaskInfo_t tinfo); int32_t qStreamRestoreParam(qTaskInfo_t tinfo); +bool qStreamRecoverScanFinished(qTaskInfo_t tinfo); #ifdef __cplusplus } diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 66988ff135a59918e200d5138f6f5142351227b6..524e75acb81422e6a8f50f74edcd91331ed73926 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -295,7 +295,7 @@ typedef struct SShowTableTagsStmt { SNodeList* pTags; } SShowTableTagsStmt; -typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT } EIndexType; +typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT, INDEX_TYPE_NORMAL } EIndexType; typedef struct SIndexOptions { ENodeType type; @@ -401,6 +401,7 @@ typedef struct SCreateStreamStmt { SNode* pQuery; SNodeList* pTags; SNode* pSubtable; + SNodeList* pCols; } SCreateStreamStmt; typedef struct SDropStreamStmt { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index f7220be753d110b94d555b06c920a8a94968a75c..2d143912474cb9ef769d2db915f24f110d37b1a2 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -361,33 +361,33 @@ typedef struct SVgDataBlocks { void* pData; // SSubmitReq + SSubmitBlk + ... } SVgDataBlocks; -typedef void (*FFreeDataBlockHash)(SHashObj*); -typedef void (*FFreeDataBlockArray)(SArray*); +typedef void (*FFreeTableBlockHash)(SHashObj*); +typedef void (*FFreeVgourpBlockArray)(SArray*); typedef struct SVnodeModifyOpStmt { - ENodeType nodeType; - ENodeType sqlNodeType; - SArray* pDataBlocks; // data block for each vgroup, SArray. - uint32_t insertType; // insert data from [file|sql statement| bound statement] - const char* pSql; // current sql statement position - int32_t totalRowsNum; - int32_t totalTbNum; - SName targetTableName; - SName usingTableName; - const char* pBoundCols; - struct STableMeta* pTableMeta; - SHashObj* pVgroupsHashObj; - SHashObj* pTableBlockHashObj; - SHashObj* pSubTableHashObj; - SHashObj* pTableNameHashObj; - SHashObj* pDbFNameHashObj; - SArray* pVgDataBlocks; - SVCreateTbReq createTblReq; - TdFilePtr fp; - FFreeDataBlockHash freeHashFunc; - FFreeDataBlockArray freeArrayFunc; - bool usingTableProcessing; - bool fileProcessing; + ENodeType nodeType; + ENodeType sqlNodeType; + SArray* pDataBlocks; // data block for each vgroup, SArray. + uint32_t insertType; // insert data from [file|sql statement| bound statement] + const char* pSql; // current sql statement position + int32_t totalRowsNum; + int32_t totalTbNum; + SName targetTableName; + SName usingTableName; + const char* pBoundCols; + struct STableMeta* pTableMeta; + SHashObj* pVgroupsHashObj; + SHashObj* pTableBlockHashObj; // SHashObj + SHashObj* pSubTableHashObj; + SHashObj* pTableNameHashObj; + SHashObj* pDbFNameHashObj; + SArray* pVgDataBlocks; // SArray + SVCreateTbReq* pCreateTblReq; + TdFilePtr fp; + FFreeTableBlockHash freeHashFunc; + FFreeVgourpBlockArray freeArrayFunc; + bool usingTableProcessing; + bool fileProcessing; } SVnodeModifyOpStmt; typedef struct SExplainOptions { diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 9be79a539f468d20ad28d2278651fbae169ff4a6..8f227459735f8c43f74b760b16b4b133c883c95b 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -58,7 +58,6 @@ typedef struct SParseContext { bool isSuperUser; bool enableSysInfo; bool async; - int8_t schemalessType; const char* svrVer; bool nodeOffline; SArray* pTableMetaPos; // sql table pos => catalog data pos @@ -85,12 +84,12 @@ int32_t qSetSTableIdForRsma(SNode* pStmt, int64_t uid); void qCleanupKeywordsTable(); int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash); -int32_t qResetStmtDataBlock(void* block, bool keepBuf); -int32_t qCloneStmtDataBlock(void** pDst, void* pSrc); -void qFreeStmtDataBlock(void* pDataBlock); -int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgId); -void qDestroyStmtDataBlock(void* pBlock); -STableMeta* qGetTableMetaInDataBlock(void* pDataBlock); +int32_t qResetStmtDataBlock(STableDataCxt* block, bool keepBuf); +int32_t qCloneStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, bool reset); +int32_t qRebuildStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_t uid, uint64_t suid, int32_t vgId, bool rebuildCreateTb); +void qDestroyStmtDataBlock(STableDataCxt* pBlock); +STableMeta* qGetTableMetaInDataBlock(STableDataCxt* pDataBlock); +int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData **pData); int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx); int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery); @@ -105,11 +104,18 @@ void destroyBoundColumnInfo(void* pBoundInfo); int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf, int32_t msgBufLen); -void* smlInitHandle(SQuery* pQuery); -void smlDestroyHandle(void* pHandle); -int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta, +void qDestroyBoundColInfo(void* pInfo); + +SQuery* smlInitHandle(); +int32_t smlBuildRow(STableDataCxt* pTableCxt); +int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void *kv, int32_t index); +STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta); + +int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int16_t msgBufLen); -int32_t smlBuildOutput(void* handle, SHashObj* pVgHash); +int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash); + +int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD *fields, int numFields); int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray); SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap); diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 3f11d2a21832ab5b5bab3c0edc8c0fd5ae085a86..bbf332c4d4c5972d5007a77838c81065764d9a4f 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -163,6 +163,23 @@ typedef struct STargetInfo { int32_t vgId; } STargetInfo; +typedef struct SBoundColInfo { + int16_t* pColIndex; // bound index => schema index + int32_t numOfCols; + int32_t numOfBound; +} SBoundColInfo; + +typedef struct STableDataCxt { + STableMeta* pMeta; + STSchema* pSchema; + SBoundColInfo boundColsInfo; + SArray* pValues; + SSubmitTbData* pData; + TSKEY lastTs; + bool ordered; + bool duplicateTs; +} STableDataCxt; + typedef int32_t (*__async_send_cb_fn_t)(void* param, SDataBuf* pMsg, int32_t code); typedef int32_t (*__async_exec_fn_t)(void* param); @@ -238,6 +255,7 @@ int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t char* parseTagDatatoJson(void* p); int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst); +int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst); void freeVgInfo(SDBVgInfo* vgInfo); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen, @@ -268,7 +286,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_RESTORING || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR) #define SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_MNODE_NOT_FOUND) -#define NO_RET_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) +#define NO_RET_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || (_code) == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED) #define NEED_REDIRECT_ERROR(_code) \ (NO_RET_REDIRECT_ERROR(_code) || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 16cf96072426aee9cf1608accccd840ae56da7ac..0f57da2b804c7c58411ee7ffa2cbb1fc4d68b3e8 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -103,6 +103,7 @@ typedef struct { int8_t type; } SStreamQueueItem; +#if 0 typedef struct { int8_t type; int64_t ver; @@ -116,6 +117,21 @@ typedef struct { SArray* dataRefs; // SArray SArray* reqs; // SArray } SStreamMergedSubmit; +#endif + +typedef struct { + int8_t type; + int64_t ver; + int32_t* dataRef; + SPackedData submit; +} SStreamDataSubmit2; + +typedef struct { + int8_t type; + int64_t ver; + SArray* dataRefs; // SArray + SArray* submits; // SArray +} SStreamMergedSubmit2; typedef struct { int8_t type; @@ -219,11 +235,11 @@ static FORCE_INLINE void* streamQueueNextItem(SStreamQueue* queue) { } } -SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq); +SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit); -void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit); +void streamDataSubmitRefDec(SStreamDataSubmit2* pDataSubmit); -SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit); +SStreamDataSubmit2* streamSubmitRefClone(SStreamDataSubmit2* pSubmit); typedef struct { char* qmsg; @@ -355,14 +371,15 @@ void tFreeSStreamTask(SStreamTask* pTask); static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem* pItem) { if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit*)pItem); + SStreamDataSubmit2* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit2*)pItem); if (pSubmitClone == NULL) { qDebug("task %d %p submit enqueue failed since out of memory", pTask->taskId, pTask); terrno = TSDB_CODE_OUT_OF_MEMORY; atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); return -1; } - qDebug("task %d %p submit enqueue %p %p %p", pTask->taskId, pTask, pItem, pSubmitClone, pSubmitClone->data); + qDebug("task %d %p submit enqueue %p %p %p %d %" PRId64, pTask->taskId, pTask, pItem, pSubmitClone, + pSubmitClone->submit.msgStr, pSubmitClone->submit.msgLen, pSubmitClone->submit.ver); taosWriteQitem(pTask->inputQueue->queue, pSubmitClone); // qStreamInput(pTask->exec.executor, pSubmitClone); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE || @@ -392,21 +409,6 @@ static FORCE_INLINE void streamTaskInputFail(SStreamTask* pTask) { atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); } -static FORCE_INLINE int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock) { - if (pTask->outputType == TASK_OUTPUT__TABLE) { - pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pBlock->blocks); - taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes); - taosFreeQitem(pBlock); - } else if (pTask->outputType == TASK_OUTPUT__SMA) { - pTask->smaSink.smaSink(pTask->smaSink.vnode, pTask->smaSink.smaId, pBlock->blocks); - taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes); - taosFreeQitem(pBlock); - } else { - taosWriteQitem(pTask->outputQueue->queue, pBlock); - } - return 0; -} - typedef struct { SMsgHead head; int64_t streamId; @@ -584,6 +586,7 @@ int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp); int32_t streamTryExec(SStreamTask* pTask); int32_t streamSchedExec(SStreamTask* pTask); +int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock); int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz); diff --git a/include/os/osMemory.h b/include/os/osMemory.h index 4681ff66741d4781c88dba5ccc950094c96f6769..e6a4488287816f21ca9eeee76a3208ff6e6ff12d 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -22,12 +22,16 @@ extern "C" { // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following sectio +#if !defined(WINDOWS) + #ifndef ALLOW_FORBID_FUNC #define malloc MALLOC_FUNC_TAOS_FORBID #define calloc CALLOC_FUNC_TAOS_FORBID #define realloc REALLOC_FUNC_TAOS_FORBID #define free FREE_FUNC_TAOS_FORBID -#endif +#endif // ifndef ALLOW_FORBID_FUNC + +#endif // if !defined(WINDOWS) void *taosMemoryMalloc(int64_t size); void *taosMemoryCalloc(int64_t num, int64_t size); diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 7765a60f8833206a5b86cd38100e1d7fd884d8dd..dbe4d6801e42fa04a9b9da0f2d3f864d0b75f94d 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -70,6 +70,7 @@ typedef struct { SysNameInfo taosGetSysNameInfo(); bool taosCheckCurrentInDll(); +int taosGetlocalhostname(char *hostname, size_t maxLen); #ifdef __cplusplus } diff --git a/include/os/osSystem.h b/include/os/osSystem.h index 8f1f5c58d5200d22583dcc7cc8c13663ce070941..58f34d26f07dafc55e7ab0c9526581545bdb8171 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -62,11 +62,38 @@ void taosResetTerminalMode(); taosMemoryFree(strings); \ } #else +#include +#include + +#define STACKSIZE 64 #define taosPrintTrace(flags, level, dflag) \ { \ - taosPrintLog(flags, level, dflag, \ - "backtrace not implemented on windows, so detailed stack information cannot be printed"); \ - } + unsigned int i; \ + void* stack[STACKSIZE]; \ + unsigned short frames; \ + SYMBOL_INFO* symbol; \ + HANDLE process; \ + \ + process = GetCurrentProcess(); \ + \ + SymInitialize(process, NULL, TRUE); \ + \ + frames = CaptureStackBackTrace(0, STACKSIZE, stack, NULL); \ + symbol = (SYMBOL_INFO*)calloc(sizeof(SYMBOL_INFO) + 256 * sizeof(char), 1); \ + if (symbol != NULL) { \ + symbol->MaxNameLen = 255; \ + symbol->SizeOfStruct = sizeof(SYMBOL_INFO); \ + \ + if (frames > 0) { \ + taosPrintLog(flags, level, dflag, "obtained %d stack frames", frames); \ + for (i = 0; i < frames; i++) { \ + SymFromAddr(process, (DWORD64)(stack[i]), 0, symbol); \ + taosPrintLog(flags, level, dflag, "frame:%i: %s - 0x%0X", frames - i - 1, symbol->Name, symbol->Address); \ + } \ + } \ + free(symbol); \ + } \ + } #endif #ifdef __cplusplus diff --git a/include/util/tRealloc.h b/include/util/tRealloc.h index f3593d5818cd626c7268ec1b721bd961fba3a845..3229c53039f6dc857dd7e0b2d4774e6875ef8c16 100644 --- a/include/util/tRealloc.h +++ b/include/util/tRealloc.h @@ -52,11 +52,13 @@ _exit: return code; } -static FORCE_INLINE void tFree(uint8_t *pBuf) { - if (pBuf) { - taosMemoryFree(pBuf - sizeof(int64_t)); - } -} +#define tFree(BUF) \ + do { \ + if (BUF) { \ + taosMemoryFree((uint8_t *)(BUF) - sizeof(int64_t)); \ + (BUF) = NULL; \ + } \ + } while (0) #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index e3f5d9e6bf302adbb390d9f74b8307484111705e..1bfcae56813e139f6076b192588fbd54c86897fa 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -65,6 +65,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RPC_PORT_EADDRINUSE TAOS_DEF_ERROR_CODE(0, 0x0017) // #define TSDB_CODE_RPC_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0018) // #define TSDB_CODE_RPC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x0019) // +#define TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED TAOS_DEF_ERROR_CODE(0, 0x0020) // "Vgroup could not be connected" +#define TSDB_CODE_RPC_SOMENODE_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0021) // //common & util #define TSDB_CODE_OPS_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x0100) // @@ -156,6 +158,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_QUERY_KILLED TAOS_DEF_ERROR_CODE(0, 0X022D) #define TSDB_CODE_TSC_NO_EXEC_NODE TAOS_DEF_ERROR_CODE(0, 0X022E) #define TSDB_CODE_TSC_NOT_STABLE_ERROR TAOS_DEF_ERROR_CODE(0, 0X022F) +#define TSDB_CODE_TSC_STMT_CACHE_ERROR TAOS_DEF_ERROR_CODE(0, 0X0230) // mnode-common // #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) // 2.x @@ -517,6 +520,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SYN_STANDBY_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0912) #define TSDB_CODE_SYN_BATCH_ERROR TAOS_DEF_ERROR_CODE(0, 0x0913) #define TSDB_CODE_SYN_RESTORING TAOS_DEF_ERROR_CODE(0, 0x0914) +#define TSDB_CODE_SYN_INVALID_SNAPSHOT_MSG TAOS_DEF_ERROR_CODE(0, 0x0915) // internal +#define TSDB_CODE_SYN_BUFFER_FULL TAOS_DEF_ERROR_CODE(0, 0x0916) // #define TSDB_CODE_SYN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x09FF) // tq diff --git a/include/util/tarray.h b/include/util/tarray.h index 0e78397ecb7c35d75211e06170ea5cdb99030e34..f8e872ec66aa2caf38bf984d8b55ef4928526016 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -22,19 +22,6 @@ extern "C" { #endif -#if 0 -#define TARRAY(TYPE) \ - struct { \ - int32_t tarray_size_; \ - int32_t tarray_neles_; \ - struct TYPE* td_array_data_; \ - } - -#define TARRAY_SIZE(ARRAY) (ARRAY)->tarray_size_ -#define TARRAY_NELES(ARRAY) (ARRAY)->tarray_neles_ -#define TARRAY_ELE_AT(ARRAY, IDX) ((ARRAY)->td_array_data_ + idx) -#endif - #define TARRAY_MIN_SIZE 8 #define TARRAY_GET_ELEM(array, index) ((void*)((char*)((array)->pData) + (index) * (array)->elemSize)) #define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize) @@ -46,6 +33,9 @@ typedef struct SArray { void* pData; } SArray; +#define TARRAY_SIZE(array) ((array)->size) +#define TARRAY_DATA(array) ((array)->pData) + /** * * @param size @@ -194,6 +184,13 @@ void taosArrayPopTailBatch(SArray* pArray, size_t cnt); */ void taosArrayRemove(SArray* pArray, size_t index); +/** + * remove batch entry from the given index + * @param pArray + * @param index + */ +void taosArrayRemoveBatch(SArray* pArray, size_t index, size_t num, FDelete fp); + /** * copy the whole array from source to destination * @param pDst diff --git a/include/util/tdef.h b/include/util/tdef.h index e1d421a39930368a5f66744c0f9fbf7a07475831..4cf4f73b9bfe02320594bd112568406629c8e895 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -189,12 +189,13 @@ typedef enum ELogicConditionType { #define TSDB_MAX_COLUMNS 4096 #define TSDB_MIN_COLUMNS 2 // PRIMARY COLUMN(timestamp) + other columns -#define TSDB_NODE_NAME_LEN 64 -#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string -#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string -#define TSDB_CGROUP_LEN 193 // it is a null-terminated string -#define TSDB_DB_NAME_LEN 65 -#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) +#define TSDB_NODE_NAME_LEN 64 +#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string +#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string +#define TSDB_CGROUP_LEN 193 // it is a null-terminated string +#define TSDB_USER_CGROUP_LEN (TSDB_USER_LEN + TSDB_CGROUP_LEN) // it is a null-terminated string +#define TSDB_DB_NAME_LEN 65 +#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_FUNC_NAME_LEN 65 #define TSDB_FUNC_COMMENT_LEN 1024 * 1024 @@ -254,7 +255,7 @@ typedef enum ELogicConditionType { #define TSDB_EP_LEN (TSDB_FQDN_LEN + 6) #define TSDB_IPv4ADDR_LEN 16 #define TSDB_FILENAME_LEN 128 -#define TSDB_SHOW_SQL_LEN 1024 +#define TSDB_SHOW_SQL_LEN 2048 #define TSDB_SLOW_QUERY_SQL_LEN 512 #define TSDB_SHOW_SUBQUERY_LEN 1000 diff --git a/include/util/tencode.h b/include/util/tencode.h index a6dd58297e8c1dba644d86eb5145b273406fbf9e..ff97a205073822be2c98e39469066a0470794a51 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -116,6 +116,7 @@ static int32_t tEncodeI64v(SEncoder* pCoder, int64_t val); static int32_t tEncodeFloat(SEncoder* pCoder, float val); static int32_t tEncodeDouble(SEncoder* pCoder, double val); static int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, uint32_t len); +static int32_t tEncodeBinaryEx(SEncoder* pCoder, const uint8_t* val, uint32_t len); static int32_t tEncodeCStrWithLen(SEncoder* pCoder, const char* val, uint32_t len); static int32_t tEncodeCStr(SEncoder* pCoder, const char* val); diff --git a/include/util/tworker.h b/include/util/tworker.h index 8766f87a0870aef3b3a4d08ed84ba55dc96c50f7..0636f16dbbd50b709de18ce556d8a9dcbdcfba8d 100644 --- a/include/util/tworker.h +++ b/include/util/tworker.h @@ -17,6 +17,7 @@ #define _TD_UTIL_WORKER_H_ #include "tqueue.h" +#include "tarray.h" #ifdef __cplusplus extern "C" { @@ -26,10 +27,10 @@ typedef struct SQWorkerPool SQWorkerPool; typedef struct SWWorkerPool SWWorkerPool; typedef struct SQWorker { - int32_t id; // worker id - int64_t pid; // thread pid - TdThread thread; // thread id - SQWorkerPool *pool; + int32_t id; // worker id + int64_t pid; // thread pid + TdThread thread; // thread id + void *pool; } SQWorker; typedef struct SQWorkerPool { @@ -42,6 +43,14 @@ typedef struct SQWorkerPool { TdThreadMutex mutex; } SQWorkerPool; +typedef struct SAutoQWorkerPool { + float ratio; + STaosQset *qset; + const char *name; + SArray *workers; + TdThreadMutex mutex; +} SAutoQWorkerPool; + typedef struct SWWorker { int32_t id; // worker id int64_t pid; // thread pid @@ -65,6 +74,11 @@ void tQWorkerCleanup(SQWorkerPool *pool); STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp); void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue); +int32_t tAutoQWorkerInit(SAutoQWorkerPool *pool); +void tAutoQWorkerCleanup(SAutoQWorkerPool *pool); +STaosQueue *tAutoQWorkerAllocQueue(SAutoQWorkerPool *pool, void *ahandle, FItem fp); +void tAutoQWorkerFreeQueue(SAutoQWorkerPool *pool, STaosQueue *queue); + int32_t tWWorkerInit(SWWorkerPool *pool); void tWWorkerCleanup(SWWorkerPool *pool); STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp); diff --git a/packaging/tools/make_install.bat b/packaging/tools/make_install.bat index c19519f9a1fc4c014c01496ca32925ba1eb19516..bf7418ad7935cb33f4a9d765e6c319d86383db69 100644 --- a/packaging/tools/make_install.bat +++ b/packaging/tools/make_install.bat @@ -14,6 +14,7 @@ set binary_dir=%3 set binary_dir=%binary_dir:/=\\% set osType=%4 set verNumber=%5 +set Enterprise=%6 set target_dir=C:\\TDengine if not exist %target_dir% ( @@ -57,7 +58,33 @@ if exist %binary_dir%\\build\\lib\\taosws.dll ( if exist %binary_dir%\\build\\bin\\taosdump.exe ( copy %binary_dir%\\build\\bin\\taosdump.exe %target_dir% > nul ) - +if %Enterprise% == TRUE ( + if exist %binary_dir%\\build\\bin\\taosx.exe ( + copy %binary_dir%\\build\\bin\\taosx.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\tmq_sim.exe ( + copy %binary_dir%\\build\\bin\\tmq_sim.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\tsim.exe ( + copy %binary_dir%\\build\\bin\\tsim.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\tmq_taosx_ci.exe ( + copy %binary_dir%\\build\\bin\\tmq_taosx_ci.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\tmq_demo.exe ( + copy %binary_dir%\\build\\bin\\tmq_demo.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\dumper.exe ( + copy %binary_dir%\\build\\bin\\dumper.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\runUdf.exe ( + copy %binary_dir%\\build\\bin\\runUdf.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\create_table.exe ( + copy %binary_dir%\\build\\bin\\create_table.exe %target_dir% > nul + ) +) + copy %binary_dir%\\build\\bin\\taosd.exe %target_dir% > nul copy %binary_dir%\\build\\bin\\udfd.exe %target_dir% > nul diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index f30a8a637e27086bb31fc37a5307e28aa3c39263..0ee548242f2a69b025dddb46381d8d8c1d30d799 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -348,7 +348,8 @@ cd ${release_dir} # install_dir has been distinguishes cluster from edege, so comments this code pkg_name=${install_dir}-${osType}-${cpuType} -taostools_pkg_name=${taostools_install_dir}-${osType}-${cpuType} +versionCompFirst=$(echo ${versionComp} | awk -F '.' '{print $1}') +taostools_pkg_name=${taostools_install_dir}-${osType}-${cpuType}-comp${versionCompFirst} # if [ "$verMode" == "cluster" ]; then # pkg_name=${install_dir}-${osType}-${cpuType} diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index ea76f726ea1b4ee7e89cd28053c11ac1d6dca2e1..c3b79d78298af72ce3e9b6894c7f66ef048a2382 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -149,7 +149,6 @@ typedef struct STscObj { int32_t numOfReqs; // number of sqlObj bound to this connection SAppInstInfo* pAppInfo; SHashObj* pRequests; - int8_t schemalessType; // todo remove it, this attribute should be move to request } STscObj; typedef struct SResultColumn { diff --git a/source/client/inc/clientLog.h b/source/client/inc/clientLog.h index 0cb36ff61db570f2c6fb488d2964fc38c2cfd7f4..c29f495201d7b9ec0360abf6a5414798739b3da8 100644 --- a/source/client/inc/clientLog.h +++ b/source/client/inc/clientLog.h @@ -30,7 +30,7 @@ extern "C" { #define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0) #define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", DEBUG_TRACE, cDebugFlag, __VA_ARGS__); }} while(0) #define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0) -//#define tscPerf(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", DEBUG_INFO, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscPerf(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", 0, cDebugFlag, __VA_ARGS__); }} while(0) // clang-format on #ifdef __cplusplus diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h new file mode 100644 index 0000000000000000000000000000000000000000..b74d0b34943af7e5a61e3c0fd8d4bc481736c9d6 --- /dev/null +++ b/source/client/inc/clientSml.h @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TDENGINE_CLIENTSML_H +#define TDENGINE_CLIENTSML_H +#ifdef __cplusplus +extern "C" { +#endif + +#include "catalog.h" +#include "clientInt.h" +#include "osThread.h" +#include "query.h" +#include "taos.h" +#include "taoserror.h" +#include "tcommon.h" +#include "tdef.h" +#include "tglobal.h" +#include "tlog.h" +#include "tmsg.h" +#include "tname.h" +#include "ttime.h" +#include "ttypes.h" +#include "cJSON.h" + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#ifndef likely +#define likely(expr) expect((expr) != 0, 1) +#endif +#ifndef unlikely +#define unlikely(expr) expect((expr) != 0, 0) +#endif + +#define SPACE ' ' +#define COMMA ',' +#define EQUAL '=' +#define QUOTE '"' +#define SLASH '\\' + +#define JUMP_SPACE(sql, sqlEnd) \ + while (sql < sqlEnd) { \ + if (unlikely(*sql == SPACE)) \ + sql++; \ + else \ + break; \ + } + +#define IS_INVALID_COL_LEN(len) ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN) +#define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN) + +#define TS "_ts" +#define TS_LEN 3 +#define VALUE "_value" +#define VALUE_LEN 6 + +#define MAX_RETRY_TIMES 5 +typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; + +typedef enum { + SCHEMA_ACTION_NULL, + SCHEMA_ACTION_CREATE_STABLE, + SCHEMA_ACTION_ADD_COLUMN, + SCHEMA_ACTION_ADD_TAG, + SCHEMA_ACTION_CHANGE_COLUMN_SIZE, + SCHEMA_ACTION_CHANGE_TAG_SIZE, +} ESchemaAction; + +typedef struct { + const void *key; + int32_t keyLen; + void *value; + bool used; +}Node; + +typedef struct NodeList{ + Node data; + struct NodeList* next; +}NodeList; + +typedef struct { + char *measure; + char *tags; + char *cols; + char *timestamp; + + int32_t measureLen; + int32_t measureTagsLen; + int32_t tagsLen; + int32_t colsLen; + int32_t timestampLen; + + SArray *colArray; +} SSmlLineInfo; + +typedef struct { + const char *sTableName; // super table name + int32_t sTableNameLen; + char childTableName[TSDB_TABLE_NAME_LEN]; + uint64_t uid; + void *key; // for openTsdb + + SArray *tags; + + // elements are SHashObj for find by key quickly + SArray *cols; + STableDataCxt *tableDataCtx; +} SSmlTableInfo; + +typedef struct { + SArray *tags; // save the origin order to create table + SHashObj *tagHash; // elements are + + SArray *cols; + SHashObj *colHash; + + STableMeta *tableMeta; +} SSmlSTableMeta; + +typedef struct { + int32_t len; + char *buf; +} SSmlMsgBuf; + +typedef struct { + int32_t code; + int32_t lineNum; + + int32_t numOfSTables; + int32_t numOfCTables; + int32_t numOfCreateSTables; + int32_t numOfAlterColSTables; + int32_t numOfAlterTagSTables; + + int64_t parseTime; + int64_t schemaTime; + int64_t insertBindTime; + int64_t insertRpcTime; + int64_t endTime; +} SSmlCostInfo; + +typedef struct { + int64_t id; + + SMLProtocolType protocol; + int8_t precision; + bool reRun; + bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) + bool isRawLine; + int32_t ttl; + + NodeList *childTables; + NodeList *superTables; + SHashObj *pVgHash; + + STscObj *taos; + SCatalog *pCatalog; + SRequestObj *pRequest; + SQuery *pQuery; + + SSmlCostInfo cost; + int32_t lineNum; + SSmlMsgBuf msgBuf; + +// cJSON *root; // for parse json + int8_t offset[4]; + SSmlLineInfo *lines; // element is SSmlLineInfo + + // + SArray *preLineTagKV; + SArray *preLineColKV; + + SSmlLineInfo preLine; + STableMeta *currSTableMeta; + STableDataCxt *currTableDataCtx; + bool needModifySchema; +} SSmlHandle; + +#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \ +&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0) + +#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \ +&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0) + +#define IS_SAME_KEY (preKV->keyLen == kv.keyLen && memcmp(preKV->key, kv.key, kv.keyLen) == 0) + +extern int64_t smlFactorNS[3]; +extern int64_t smlFactorS[3]; + +typedef int32_t (*_equal_fn_sml)(const void *, const void *); + +SSmlHandle *smlBuildSmlInfo(TAOS *taos); +void smlDestroyInfo(SSmlHandle *info); +void smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset); +void smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset); +SArray *smlJsonParseTags(char *start, char *end); +bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); +void* nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn); +int nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn); +int nodeListSize(NodeList* list); +bool smlDoubleToInt64OverFlow(double num); +int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); +bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); +int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision); +int8_t smlGetTsTypeByLen(int32_t len); +SSmlTableInfo* smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen); +SSmlSTableMeta* smlBuildSTableMeta(bool isDataFormat); +int32_t smlSetCTableName(SSmlTableInfo *oneTable); +STableMeta* smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen); +int32_t is_same_child_table_telnet(const void *a, const void *b); +int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len); +int32_t smlClearForRerun(SSmlHandle *info); +int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg); + +int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); +int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); +int32_t smlParseJSON(SSmlHandle *info, char *payload); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_CLIENTSML_H diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index e5507ccf816b9fd591e6e27ee2974badcb5734ed..2b42de93e3f45ccb95c085c31825d4e0a117538c 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -21,8 +21,6 @@ extern "C" { #endif #include "catalog.h" -typedef void STableDataBlocks; - typedef enum { STMT_TYPE_INSERT = 1, STMT_TYPE_MULTI_INSERT, @@ -43,8 +41,8 @@ typedef enum { } STMT_STATUS; typedef struct SStmtTableCache { - STableDataBlocks *pDataBlock; - void *boundTags; + STableDataCxt *pDataCtx; + void *boundTags; } SStmtTableCache; typedef struct SStmtQueryResInfo { @@ -71,10 +69,11 @@ typedef struct SStmtBindInfo { } SStmtBindInfo; typedef struct SStmtExecInfo { - int32_t affectedRows; - SRequestObj *pRequest; - SHashObj *pBlockHash; - bool autoCreateTbl; + int32_t affectedRows; + SRequestObj *pRequest; + SHashObj *pBlockHash; + STableDataCxt *pCurrBlock; + SSubmitTbData *pCurrTbData; } SStmtExecInfo; typedef struct SStmtSQLInfo { diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 75a72647bd4b62ec8b328828f1a06691bfc8ac15..d429e521117669e0b61cb950d56151f4443ec20f 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -76,13 +76,19 @@ static void deregisterRequest(SRequestObj *pRequest) { "current:%d, app current:%d", pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000.0, num, currentInst); + tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 + "us, exec:%" PRId64 "us, stmtType:%d", + duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, + pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - pRequest->metric.ctgEnd, + pRequest->metric.execEnd - pRequest->metric.semanticEnd, pRequest->stmtType); + if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType) { - // tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 - // "us, exec:%" PRId64 "us", - // duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, - // pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - - // pRequest->metric.ctgEnd, pRequest->metric.execEnd - pRequest->metric.semanticEnd); - atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); + // tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 + // "us, exec:%" PRId64 "us", + // duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, + // pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - + // pRequest->metric.ctgEnd, pRequest->metric.execEnd - pRequest->metric.semanticEnd); + // atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { // tscPerf("select duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 // "us, planner:%" PRId64 "us, exec:%" PRId64 "us, reqId:0x%" PRIx64, @@ -264,7 +270,6 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c taosThreadMutexInit(&pObj->mutex, NULL); pObj->id = taosAddRef(clientConnRefPool, pObj); - pObj->schemalessType = 1; atomic_add_fetch_64(&pObj->pAppInfo->numOfConns, 1); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index aaf27a77161a9f415cf63782ce266000977e5297..6f61d59846a7acb5afc4ff61bfaf60af5ae6917c 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -239,7 +239,6 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC .pTransporter = pTscObj->pAppInfo->pTransporter, .pStmtCb = pStmtCb, .pUser = pTscObj->user, - .schemalessType = pTscObj->schemalessType, .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)), .enableSysInfo = pTscObj->sysInfo, .svrVer = pTscObj->sVer, @@ -741,47 +740,21 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList } int32_t handleSubmitExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet* epset) { - int32_t code = 0; - SArray* pArray = NULL; - SSubmitRsp* pRsp = (SSubmitRsp*)res; - if (pRsp->nBlocks <= 0) { - taosMemoryFreeClear(pRsp->pBlocks); + SArray* pArray = NULL; + SSubmitRsp2* pRsp = (SSubmitRsp2*)res; + if (NULL == pRsp->aCreateTbRsp) { return TSDB_CODE_SUCCESS; } - pArray = taosArrayInit(pRsp->nBlocks, sizeof(STbSVersion)); - if (NULL == pArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < pRsp->nBlocks; ++i) { - SSubmitBlkRsp* blk = pRsp->pBlocks + i; - if (blk->pMeta) { - handleCreateTbExecRes(blk->pMeta, pCatalog); - tFreeSTableMetaRsp(blk->pMeta); - taosMemoryFreeClear(blk->pMeta); - } - - if (NULL == blk->tblFName || 0 == blk->tblFName[0]) { - continue; + int32_t tbNum = taosArrayGetSize(pRsp->aCreateTbRsp); + for (int32_t i = 0; i < tbNum; ++i) { + SVCreateTbRsp* pTbRsp = (SVCreateTbRsp*)taosArrayGet(pRsp->aCreateTbRsp, i); + if (pTbRsp->pMeta) { + handleCreateTbExecRes(pTbRsp->pMeta, pCatalog); } - - STbSVersion tbSver = {.tbFName = blk->tblFName, .sver = blk->sver}; - taosArrayPush(pArray, &tbSver); } - SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = *epset}; - - code = catalogChkTbMetaVersion(pCatalog, &conn, pArray); - -_return: - - taosArrayDestroy(pArray); - return code; + return TSDB_CODE_SUCCESS; } int32_t handleQueryExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet* epset) { @@ -1430,6 +1403,21 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { memcpy((void*)tEpSet, (void*)pEpSet, sizeof(SEpSet)); } + // pMsg is response msg + if (pMsg->msgType == TDMT_MND_CONNECT + 1) { + // restore origin code + if (pMsg->code == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED) { + pMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + } else if (pMsg->code == TSDB_CODE_RPC_SOMENODE_BROKEN_LINK) { + pMsg->code = TSDB_CODE_RPC_BROKEN_LINK; + } + } else { + // uniform to one error code: TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED + if (pMsg->code == TSDB_CODE_RPC_SOMENODE_BROKEN_LINK) { + pMsg->code = TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED; + } + } + AsyncArg* arg = taosMemoryCalloc(1, sizeof(AsyncArg)); arg->msg = *pMsg; arg->pEpset = tEpSet; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 87f5e5fa40ff06efb0983689767f3b4f582be55b..896d1d6ca6f34b06e359334eabaf4eb53434b1d5 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -866,7 +866,6 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) { .pTransporter = pTscObj->pAppInfo->pTransporter, .pStmtCb = NULL, .pUser = pTscObj->user, - .schemalessType = pTscObj->schemalessType, .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)), .enableSysInfo = pTscObj->sysInfo, .async = true, diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 1c0bef9e928ba98bec1f54bf3f6ed27dacb7a580..0918b99cdab1b01b809296eb19ee4af3ece2ac37 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1240,22 +1240,12 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { return code; } -typedef struct { - SVgroupInfo vg; - void* data; -} VgData; - -static void destroyVgHash(void* data) { - VgData* vgData = (VgData*)data; - taosMemoryFreeClear(vgData->data); -} - int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD* fields, int numFields) { int32_t code = TSDB_CODE_SUCCESS; STableMeta* pTableMeta = NULL; SQuery* pQuery = NULL; - SSubmitReq* subReq = NULL; + SHashObj* pVgHash = NULL; SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); if (!pRequest) { @@ -1300,149 +1290,25 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbname); goto end; } - uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); - uint64_t uid = pTableMeta->uid; - int32_t numOfCols = pTableMeta->tableInfo.numOfColumns; - - uint16_t fLen = 0; - int32_t rowSize = 0; - int16_t nVar = 0; - for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - SSchema* schema = pTableMeta->schema + i; - fLen += TYPE_BYTES[schema->type]; - rowSize += schema->bytes; - if (IS_VAR_DATA_TYPE(schema->type)) { - nVar++; - } - } - - fLen -= sizeof(TSKEY); - - int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(numOfCols - 1); - int32_t schemaLen = 0; - int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; - - int32_t totalLen = sizeof(SSubmitReq) + submitLen; - subReq = taosMemoryCalloc(1, totalLen); - SSubmitBlk* blk = POINTER_SHIFT(subReq, sizeof(SSubmitReq)); - void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); - STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTableMeta->sversion); - tdSRowSetTpInfo(&rb, numOfCols, fLen); - int32_t dataLen = 0; - - // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column - // length | - char* pStart = pData + getVersion1BlockMetaSize(pData, numFields); - int32_t* colLength = (int32_t*)pStart; - pStart += sizeof(int32_t) * numFields; - - SResultColumn* pCol = taosMemoryCalloc(numFields, sizeof(SResultColumn)); - - for (int32_t i = 0; i < numFields; ++i) { - if (IS_VAR_DATA_TYPE(fields[i].type)) { - pCol[i].offset = (int32_t*)pStart; - pStart += rows * sizeof(int32_t); - } else { - pCol[i].nullbitmap = pStart; - pStart += BitmapLen(rows); - } - - pCol[i].pData = pStart; - pStart += colLength[i]; - } - - SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - for (int i = 0; i < numFields; i++) { - TAOS_FIELD* schema = &fields[i]; - taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); - } - - for (int32_t j = 0; j < rows; j++) { - tdSRowResetBuf(&rb, rowData); - int32_t offset = 0; - for (int32_t k = 0; k < numOfCols; k++) { - const SSchema* pColumn = &pTableMeta->schema[k]; - int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); - if (!index) { // add none - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k); - } else { - if (IS_VAR_DATA_TYPE(pColumn->type)) { - if (pCol[*index].offset[j] != -1) { - char* data = pCol[*index].pData + pCol[*index].offset[j]; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); - } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } - } else { - if (!colDataIsNull_f(pCol[*index].nullbitmap, j)) { - char* data = pCol[*index].pData + pColumn->bytes * j; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); - } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } - } - } - - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } - } - tdSRowEnd(&rb); - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - dataLen += rowLen; - } - - taosHashCleanup(schemaHash); - taosMemoryFree(pCol); - - blk->uid = htobe64(uid); - blk->suid = htobe64(suid); - blk->sversion = htonl(pTableMeta->sversion); - blk->schemaLen = htonl(schemaLen); - blk->numOfRows = htonl(rows); - blk->dataLen = htonl(dataLen); - subReq->length = sizeof(SSubmitReq) + sizeof(SSubmitBlk) + schemaLen + dataLen; - subReq->numOfBlocks = 1; - - pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == pQuery) { - uError("create SQuery error"); + pQuery = smlInitHandle(); + if (pQuery == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto end; } - pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->haveResultSet = false; - pQuery->msgType = TDMT_VND_SUBMIT; - pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT); - if (NULL == pQuery->pRoot) { - uError("create pQuery->pRoot error"); - code = TSDB_CODE_OUT_OF_MEMORY; + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)); + + code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:rawBlockBindData failed"); goto end; } - SVnodeModifyOpStmt* nodeStmt = (SVnodeModifyOpStmt*)(pQuery->pRoot); - nodeStmt->pDataBlocks = taosArrayInit(1, POINTER_BYTES); - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; + code = smlBuildOutput(pQuery, pVgHash); + if (code != TSDB_CODE_SUCCESS) { + uError("smlBuildOutput failed"); + return code; } - dst->vg = vgData; - dst->numOfTables = subReq->numOfBlocks; - dst->size = subReq->length; - dst->pData = (char*)subReq; - subReq->header.vgId = htonl(dst->vg.vgId); - subReq->version = htonl(1); - subReq->header.contLen = htonl(subReq->length); - subReq->length = htonl(subReq->length); - subReq->numOfBlocks = htonl(subReq->numOfBlocks); - subReq = NULL; // no need free - taosArrayPush(nodeStmt->pDataBlocks, &dst); launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; @@ -1450,7 +1316,8 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch end: taosMemoryFreeClear(pTableMeta); qDestroyQuery(pQuery); - taosMemoryFree(subReq); + destroyRequest(pRequest); + taosHashCleanup(pVgHash); return code; } @@ -1458,7 +1325,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) int32_t code = TSDB_CODE_SUCCESS; STableMeta* pTableMeta = NULL; SQuery* pQuery = NULL; - SSubmitReq* subReq = NULL; + SHashObj* pVgHash = NULL; SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); if (!pRequest) { @@ -1503,138 +1370,25 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbname); goto end; } - uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); - uint64_t uid = pTableMeta->uid; - int32_t numOfCols = pTableMeta->tableInfo.numOfColumns; - - uint16_t fLen = 0; - int32_t rowSize = 0; - int16_t nVar = 0; - for (int i = 0; i < numOfCols; i++) { - SSchema* schema = pTableMeta->schema + i; - fLen += TYPE_BYTES[schema->type]; - rowSize += schema->bytes; - if (IS_VAR_DATA_TYPE(schema->type)) { - nVar++; - } - } - fLen -= sizeof(TSKEY); - - int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(numOfCols - 1); - int32_t schemaLen = 0; - int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; - - int32_t totalLen = sizeof(SSubmitReq) + submitLen; - subReq = taosMemoryCalloc(1, totalLen); - SSubmitBlk* blk = POINTER_SHIFT(subReq, sizeof(SSubmitReq)); - void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); - STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTableMeta->sversion); - tdSRowSetTpInfo(&rb, numOfCols, fLen); - int32_t dataLen = 0; - - // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column - // length | - char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols); - int32_t* colLength = (int32_t*)pStart; - pStart += sizeof(int32_t) * numOfCols; - - SResultColumn* pCol = taosMemoryCalloc(numOfCols, sizeof(SResultColumn)); - - for (int32_t i = 0; i < numOfCols; ++i) { - if (IS_VAR_DATA_TYPE(pTableMeta->schema[i].type)) { - pCol[i].offset = (int32_t*)pStart; - pStart += rows * sizeof(int32_t); - } else { - pCol[i].nullbitmap = pStart; - pStart += BitmapLen(rows); - } - - pCol[i].pData = pStart; - pStart += colLength[i]; - } - - for (int32_t j = 0; j < rows; j++) { - tdSRowResetBuf(&rb, rowData); - int32_t offset = 0; - for (int32_t k = 0; k < numOfCols; k++) { - const SSchema* pColumn = &pTableMeta->schema[k]; - - if (IS_VAR_DATA_TYPE(pColumn->type)) { - if (pCol[k].offset[j] != -1) { - char* data = pCol[k].pData + pCol[k].offset[j]; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); - } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } - } else { - if (!colDataIsNull_f(pCol[k].nullbitmap, j)) { - char* data = pCol[k].pData + pColumn->bytes * j; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); - } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } - } - - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } - } - tdSRowEnd(&rb); - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - dataLen += rowLen; - } - - taosMemoryFree(pCol); - - blk->uid = htobe64(uid); - blk->suid = htobe64(suid); - blk->sversion = htonl(pTableMeta->sversion); - blk->schemaLen = htonl(schemaLen); - blk->numOfRows = htonl(rows); - blk->dataLen = htonl(dataLen); - subReq->length = sizeof(SSubmitReq) + sizeof(SSubmitBlk) + schemaLen + dataLen; - subReq->numOfBlocks = 1; - - pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == pQuery) { - uError("create SQuery error"); + pQuery = smlInitHandle(); + if (pQuery == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(subReq); goto end; } - pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->haveResultSet = false; - pQuery->msgType = TDMT_VND_SUBMIT; - pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT); - if (NULL == pQuery->pRoot) { - uError("create pQuery->pRoot error"); - code = TSDB_CODE_OUT_OF_MEMORY; + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)); + + code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:rawBlockBindData failed"); goto end; } - SVnodeModifyOpStmt* nodeStmt = (SVnodeModifyOpStmt*)(pQuery->pRoot); - nodeStmt->pDataBlocks = taosArrayInit(1, POINTER_BYTES); - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; + code = smlBuildOutput(pQuery, pVgHash); + if (code != TSDB_CODE_SUCCESS) { + uError("smlBuildOutput failed"); + return code; } - dst->vg = vgData; - dst->numOfTables = subReq->numOfBlocks; - dst->size = subReq->length; - dst->pData = (char*)subReq; - subReq->header.vgId = htonl(dst->vg.vgId); - subReq->version = htonl(1); - subReq->header.contLen = htonl(subReq->length); - subReq->length = htonl(subReq->length); - subReq->numOfBlocks = htonl(subReq->numOfBlocks); - subReq = NULL; // no need free - taosArrayPush(nodeStmt->pDataBlocks, &dst); launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; @@ -1642,7 +1396,8 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) end: taosMemoryFreeClear(pTableMeta); qDestroyQuery(pQuery); - taosMemoryFree(subReq); + destroyRequest(pRequest); + taosHashCleanup(pVgHash); return code; } @@ -1679,8 +1434,6 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { goto end; } - pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - taosHashSetFreeFp(pVgHash, destroyVgHash); struct SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { @@ -1694,6 +1447,12 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { conn.requestObjRefId = pRequest->self; conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + pQuery = smlInitHandle(); + if (pQuery == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); uDebug("raw data block num:%d\n", rspObj.rsp.blockNum); while (++rspObj.resIter < rspObj.rsp.blockNum) { SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); @@ -1701,14 +1460,6 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { uError("WriteRaw:no schema, iter:%d", rspObj.resIter); goto end; } - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter); - setResSchemaInfo(&rspObj.resInfo, pSW->pSchema, pSW->nCols); - - code = setQueryResultFromRsp(&rspObj.resInfo, pRetrieve, false, false); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw: setQueryResultFromRsp error"); - goto end; - } const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.blockTbName, rspObj.resIter); if (!tbName) { @@ -1722,13 +1473,6 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { strcpy(pName.dbname, pRequest->pDb); strcpy(pName.tname, tbName); - VgData vgData = {0}; - code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &(vgData.vg)); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName); - goto end; - } - code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { uError("WriteRaw:catalogGetTableMeta table not exist. table name: %s", tbName); @@ -1740,164 +1484,29 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { goto end; } - uint16_t fLen = 0; - int32_t rowSize = 0; - int16_t nVar = 0; - for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - SSchema* schema = &pTableMeta->schema[i]; - fLen += TYPE_BYTES[schema->type]; - rowSize += schema->bytes; - if (IS_VAR_DATA_TYPE(schema->type)) { - nVar++; - } - } - fLen -= sizeof(TSKEY); - - int32_t rows = rspObj.resInfo.numOfRows; - int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(pTableMeta->tableInfo.numOfColumns - 1); - int32_t schemaLen = 0; - int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; - - SSubmitReq* subReq = NULL; - SSubmitBlk* blk = NULL; - void* hData = taosHashGet(pVgHash, &vgData.vg.vgId, sizeof(vgData.vg.vgId)); - if (hData) { - vgData = *(VgData*)hData; - - int32_t totalLen = ((SSubmitReq*)(vgData.data))->length + submitLen; - void* tmp = taosMemoryRealloc(vgData.data, totalLen); - if (tmp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - vgData.data = tmp; - ((VgData*)hData)->data = tmp; - subReq = (SSubmitReq*)(vgData.data); - blk = POINTER_SHIFT(vgData.data, subReq->length); - } else { - int32_t totalLen = sizeof(SSubmitReq) + submitLen; - void* tmp = taosMemoryCalloc(1, totalLen); - if (tmp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - vgData.data = tmp; - taosHashPut(pVgHash, (const char*)&vgData.vg.vgId, sizeof(vgData.vg.vgId), (char*)&vgData, sizeof(vgData)); - subReq = (SSubmitReq*)(vgData.data); - subReq->length = sizeof(SSubmitReq); - subReq->numOfBlocks = 0; - - blk = POINTER_SHIFT(vgData.data, sizeof(SSubmitReq)); + SVgroupInfo vg; + code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName); + goto end; } - // pSW->pSchema should be same as pTableMeta->schema - // ASSERT(pSW->nCols == pTableMeta->tableInfo.numOfColumns); - uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); - uint64_t uid = pTableMeta->uid; - int16_t sver = pTableMeta->sversion; - - void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); - STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - - SRowBuilder rb = {0}; - tdSRowInit(&rb, sver); - tdSRowSetTpInfo(&rb, pTableMeta->tableInfo.numOfColumns, fLen); - int32_t totalLen = 0; - - SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - for (int i = 0; i < pSW->nCols; i++) { - SSchema* schema = &pSW->pSchema[i]; - taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); + void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId)); + if (hData == NULL) { + taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); } - for (int32_t j = 0; j < rows; j++) { - tdSRowResetBuf(&rb, rowData); - - doSetOneRowPtr(&rspObj.resInfo); - rspObj.resInfo.current += 1; - - int32_t offset = 0; - for (int32_t k = 0; k < pTableMeta->tableInfo.numOfColumns; k++) { - const SSchema* pColumn = &pTableMeta->schema[k]; - int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); - if (!index) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k); - } else { - char* colData = rspObj.resInfo.row[*index]; - if (!colData) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } else { - if (IS_VAR_DATA_TYPE(pColumn->type)) { - colData -= VARSTR_HEADER_SIZE; - } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); - } - } - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } - } - tdSRowEnd(&rb); - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - totalLen += rowLen; + code = rawBlockBindData(pQuery, pTableMeta, pRetrieve->data, NULL, NULL, 0); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:rawBlockBindData failed"); + goto end; } - - taosHashCleanup(schemaHash); - blk->uid = htobe64(uid); - blk->suid = htobe64(suid); - blk->sversion = htonl(sver); - blk->schemaLen = htonl(schemaLen); - blk->numOfRows = htonl(rows); - blk->dataLen = htonl(totalLen); - subReq->length += sizeof(SSubmitBlk) + schemaLen + totalLen; - subReq->numOfBlocks++; - taosMemoryFreeClear(pTableMeta); - rspObj.resInfo.pRspMsg = NULL; - doFreeReqResultInfo(&rspObj.resInfo); - } - - pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == pQuery) { - uError("create SQuery error"); - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->haveResultSet = false; - pQuery->msgType = TDMT_VND_SUBMIT; - pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT); - if (NULL == pQuery->pRoot) { - uError("create pQuery->pRoot error"); - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; } - SVnodeModifyOpStmt* nodeStmt = (SVnodeModifyOpStmt*)(pQuery->pRoot); - - int32_t numOfVg = taosHashGetSize(pVgHash); - nodeStmt->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); - VgData* vData = (VgData*)taosHashIterate(pVgHash, NULL); - while (vData) { - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - dst->vg = vData->vg; - SSubmitReq* subReq = (SSubmitReq*)(vData->data); - dst->numOfTables = subReq->numOfBlocks; - dst->size = subReq->length; - dst->pData = (char*)subReq; - vData->data = NULL; // no need free - subReq->header.vgId = htonl(dst->vg.vgId); - subReq->version = htonl(1); - subReq->header.contLen = htonl(subReq->length); - subReq->length = htonl(subReq->length); - subReq->numOfBlocks = htonl(subReq->numOfBlocks); - taosArrayPush(nodeStmt->pDataBlocks, &dst); - vData = (VgData*)taosHashIterate(pVgHash, vData); + code = smlBuildOutput(pQuery, pVgHash); + if (code != TSDB_CODE_SUCCESS) { + uError("smlBuildOutput failed"); + return code; } launchQueryImpl(pRequest, pQuery, true, NULL); @@ -1905,8 +1514,6 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { end: tDeleteSMqDataRsp(&rspObj.rsp); - rspObj.resInfo.pRspMsg = NULL; - doFreeReqResultInfo(&rspObj.resInfo); tDecoderClear(&decoder); qDestroyQuery(pQuery); destroyRequest(pRequest); @@ -1948,8 +1555,6 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) goto end; } - pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - taosHashSetFreeFp(pVgHash, destroyVgHash); struct SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { @@ -1963,6 +1568,13 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) conn.requestObjRefId = pRequest->self; conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + pQuery = smlInitHandle(); + if (pQuery == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + uDebug("raw data block num:%d\n", rspObj.rsp.blockNum); while (++rspObj.resIter < rspObj.rsp.blockNum) { SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); @@ -1970,14 +1582,6 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) uError("WriteRaw:no schema, iter:%d", rspObj.resIter); goto end; } - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter); - setResSchemaInfo(&rspObj.resInfo, pSW->pSchema, pSW->nCols); - - code = setQueryResultFromRsp(&rspObj.resInfo, pRetrieve, false, false); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw: setQueryResultFromRsp error"); - goto end; - } const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.blockTbName, rspObj.resIter); if (!tbName) { @@ -1991,44 +1595,28 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) strcpy(pName.dbname, pRequest->pDb); strcpy(pName.tname, tbName); - VgData vgData = {0}; - code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &(vgData.vg)); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName); - goto end; - } - // find schema data info - int32_t schemaLen = 0; - void* schemaData = NULL; + SVCreateTbReq pCreateReq = {0}; + for (int j = 0; j < rspObj.rsp.createTableNum; j++) { void** dataTmp = taosArrayGet(rspObj.rsp.createTableReq, j); int32_t* lenTmp = taosArrayGet(rspObj.rsp.createTableLen, j); - SDecoder decoderTmp = {0}; - SVCreateTbReq pCreateReq = {0}; - + SDecoder decoderTmp = {0}; tDecoderInit(&decoderTmp, *dataTmp, *lenTmp); + memset(&pCreateReq, 0, sizeof(SVCreateTbReq)); if (tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq) < 0) { tDecoderClear(&decoderTmp); - taosMemoryFreeClear(pCreateReq.comment); - taosArrayDestroy(pCreateReq.ctb.tagName); goto end; } ASSERT(pCreateReq.type == TSDB_CHILD_TABLE); if (strcmp(tbName, pCreateReq.name) == 0) { - schemaLen = *lenTmp; - schemaData = *dataTmp; strcpy(pName.tname, pCreateReq.ctb.stbName); tDecoderClear(&decoderTmp); - taosMemoryFreeClear(pCreateReq.comment); - taosArrayDestroy(pCreateReq.ctb.tagName); break; } tDecoderClear(&decoderTmp); - taosMemoryFreeClear(pCreateReq.comment); - taosArrayDestroy(pCreateReq.ctb.tagName); } code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); @@ -2042,167 +1630,23 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) goto end; } - uint16_t fLen = 0; - int32_t rowSize = 0; - int16_t nVar = 0; - for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - SSchema* schema = &pTableMeta->schema[i]; - fLen += TYPE_BYTES[schema->type]; - rowSize += schema->bytes; - if (IS_VAR_DATA_TYPE(schema->type)) { - nVar++; - } - } - fLen -= sizeof(TSKEY); - - int32_t rows = rspObj.resInfo.numOfRows; - int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(pTableMeta->tableInfo.numOfColumns - 1); - - int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; - - SSubmitReq* subReq = NULL; - SSubmitBlk* blk = NULL; - void* hData = taosHashGet(pVgHash, &vgData.vg.vgId, sizeof(vgData.vg.vgId)); - if (hData) { - vgData = *(VgData*)hData; - - int32_t totalLen = ((SSubmitReq*)(vgData.data))->length + submitLen; - void* tmp = taosMemoryRealloc(vgData.data, totalLen); - if (tmp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - vgData.data = tmp; - ((VgData*)hData)->data = tmp; - subReq = (SSubmitReq*)(vgData.data); - blk = POINTER_SHIFT(vgData.data, subReq->length); - } else { - int32_t totalLen = sizeof(SSubmitReq) + submitLen; - void* tmp = taosMemoryCalloc(1, totalLen); - if (tmp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - vgData.data = tmp; - taosHashPut(pVgHash, (const char*)&vgData.vg.vgId, sizeof(vgData.vg.vgId), (char*)&vgData, sizeof(vgData)); - subReq = (SSubmitReq*)(vgData.data); - subReq->length = sizeof(SSubmitReq); - subReq->numOfBlocks = 0; - - blk = POINTER_SHIFT(vgData.data, sizeof(SSubmitReq)); - } - - // pSW->pSchema should be same as pTableMeta->schema - // ASSERT(pSW->nCols == pTableMeta->tableInfo.numOfColumns); - uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); - uint64_t uid = pTableMeta->uid; - int16_t sver = pTableMeta->sversion; - - void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); - if (schemaData) { - memcpy(blkSchema, schemaData, schemaLen); - } - STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - - SRowBuilder rb = {0}; - tdSRowInit(&rb, sver); - tdSRowSetTpInfo(&rb, pTableMeta->tableInfo.numOfColumns, fLen); - int32_t totalLen = 0; - - SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - for (int i = 0; i < pSW->nCols; i++) { - SSchema* schema = &pSW->pSchema[i]; - taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); + SVgroupInfo vg; + code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName); + goto end; } - for (int32_t j = 0; j < rows; j++) { - tdSRowResetBuf(&rb, rowData); - - doSetOneRowPtr(&rspObj.resInfo); - rspObj.resInfo.current += 1; - - int32_t offset = 0; - for (int32_t k = 0; k < pTableMeta->tableInfo.numOfColumns; k++) { - const SSchema* pColumn = &pTableMeta->schema[k]; - int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); - if (!index) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k); - } else { - char* colData = rspObj.resInfo.row[*index]; - if (!colData) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } else { - if (IS_VAR_DATA_TYPE(pColumn->type)) { - colData -= VARSTR_HEADER_SIZE; - } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); - } - } - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } - } - tdSRowEnd(&rb); - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - totalLen += rowLen; + void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId)); + if (hData == NULL) { + taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); } - taosHashCleanup(schemaHash); - blk->uid = htobe64(uid); - blk->suid = htobe64(suid); - blk->sversion = htonl(sver); - blk->schemaLen = htonl(schemaLen); - blk->numOfRows = htonl(rows); - blk->dataLen = htonl(totalLen); - subReq->length += sizeof(SSubmitBlk) + schemaLen + totalLen; - subReq->numOfBlocks++; - taosMemoryFreeClear(pTableMeta); - rspObj.resInfo.pRspMsg = NULL; - doFreeReqResultInfo(&rspObj.resInfo); - } - - pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == pQuery) { - uError("create SQuery error"); - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->haveResultSet = false; - pQuery->msgType = TDMT_VND_SUBMIT; - pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT); - if (NULL == pQuery->pRoot) { - uError("create pQuery->pRoot error"); - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - SVnodeModifyOpStmt* nodeStmt = (SVnodeModifyOpStmt*)(pQuery->pRoot); - - int32_t numOfVg = taosHashGetSize(pVgHash); - nodeStmt->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); - - VgData* vData = (VgData*)taosHashIterate(pVgHash, NULL); - while (vData) { - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = rawBlockBindData(pQuery, pTableMeta, pRetrieve->data, &pCreateReq, NULL, 0); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:rawBlockBindData failed"); goto end; } - dst->vg = vData->vg; - SSubmitReq* subReq = (SSubmitReq*)(vData->data); - dst->numOfTables = subReq->numOfBlocks; - dst->size = subReq->length; - dst->pData = (char*)subReq; - vData->data = NULL; // no need free - subReq->header.vgId = htonl(dst->vg.vgId); - subReq->version = htonl(1); - subReq->header.contLen = htonl(subReq->length); - subReq->length = htonl(subReq->length); - subReq->numOfBlocks = htonl(subReq->numOfBlocks); - taosArrayPush(nodeStmt->pDataBlocks, &dst); - vData = (VgData*)taosHashIterate(pVgHash, vData); } launchQueryImpl(pRequest, pQuery, true, NULL); @@ -2210,8 +1654,6 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) end: tDeleteSTaosxRsp(&rspObj.rsp); - rspObj.resInfo.pRspMsg = NULL; - doFreeReqResultInfo(&rspObj.resInfo); tDecoderClear(&decoder); qDestroyQuery(pQuery); destroyRequest(pRequest); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 8c77ef664a1b54f3e702b20fbfeb7e05a120b9e8..ba154c95bde3e6c8c3b2b9201b2b2c65a96b077c 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1,219 +1,101 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #include #include #include #include -#include "cJSON.h" -#include "catalog.h" -#include "clientInt.h" -#include "osSemaphore.h" -#include "osThread.h" -#include "query.h" -#include "taos.h" -#include "taoserror.h" -#include "tcommon.h" -#include "tdef.h" -#include "tglobal.h" -#include "tlog.h" -#include "tmsg.h" -#include "tname.h" -#include "ttime.h" -#include "ttypes.h" - -//================================================================================================= - -#define SPACE ' ' -#define COMMA ',' -#define EQUAL '=' -#define QUOTE '"' -#define SLASH '\\' - -#define JUMP_SPACE(sql, sqlEnd) \ - while (sql < sqlEnd) { \ - if (*sql == SPACE) \ - sql++; \ - else \ - break; \ - } -// comma , -#define IS_SLASH_COMMA(sql) (*(sql) == COMMA && *((sql)-1) == SLASH) -#define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH) -// space -#define IS_SLASH_SPACE(sql) (*(sql) == SPACE && *((sql)-1) == SLASH) -#define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH) -// equal = -#define IS_SLASH_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) == SLASH) -#define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH) -// quote " -#define IS_SLASH_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) == SLASH) -#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH) -// SLASH -#define IS_SLASH_SLASH(sql) (*(sql) == SLASH && *((sql)-1) == SLASH) - -#define IS_SLASH_LETTER(sql) \ - (IS_SLASH_COMMA(sql) || IS_SLASH_SPACE(sql) || IS_SLASH_EQUAL(sql) || IS_SLASH_QUOTE(sql) || IS_SLASH_SLASH(sql)) - -#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) - -#define PROCESS_SLASH(key, keyLen) \ - for (int i = 1; i < keyLen; ++i) { \ - if (IS_SLASH_LETTER(key + i)) { \ - MOVE_FORWARD_ONE(key + i, keyLen - i); \ - i--; \ - keyLen--; \ - } \ - } - -#define IS_INVALID_COL_LEN(len) ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN) -#define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN) - -#define OTD_JSON_SUB_FIELDS_NUM 2 -#define OTD_JSON_FIELDS_NUM 4 - -#define TS "_ts" -#define TS_LEN 3 -#define VALUE "_value" -#define VALUE_LEN 6 - -#define BINARY_ADD_LEN 2 // "binary" 2 means " " -#define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" " - -#define MAX_RETRY_TIMES 5 -//================================================================================================= -typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; - -typedef enum { - SCHEMA_ACTION_NULL, - SCHEMA_ACTION_CREATE_STABLE, - SCHEMA_ACTION_ADD_COLUMN, - SCHEMA_ACTION_ADD_TAG, - SCHEMA_ACTION_CHANGE_COLUMN_SIZE, - SCHEMA_ACTION_CHANGE_TAG_SIZE, -} ESchemaAction; - -typedef struct { - const char *measure; - const char *tags; - const char *cols; - const char *timestamp; - - int32_t measureLen; - int32_t measureTagsLen; - int32_t tagsLen; - int32_t colsLen; - int32_t timestampLen; -} SSmlLineInfo; - -typedef struct { - const char *sTableName; // super table name - int32_t sTableNameLen; - char childTableName[TSDB_TABLE_NAME_LEN]; - uint64_t uid; - - SArray *tags; - - // if info->formatData is true, elements are SArray. - // if info->formatData is false, elements are SHashObj for find by key quickly - SArray *cols; -} SSmlTableInfo; - -typedef struct { - SArray *tags; // save the origin order to create table - SHashObj *tagHash; // elements are - - SArray *cols; - SHashObj *colHash; - - STableMeta *tableMeta; -} SSmlSTableMeta; - -typedef struct { - int32_t len; - char *buf; -} SSmlMsgBuf; - -typedef struct { - int32_t code; - int32_t lineNum; - - int32_t numOfSTables; - int32_t numOfCTables; - int32_t numOfCreateSTables; - int32_t numOfAlterColSTables; - int32_t numOfAlterTagSTables; - - int64_t parseTime; - int64_t schemaTime; - int64_t insertBindTime; - int64_t insertRpcTime; - int64_t endTime; -} SSmlCostInfo; - -typedef struct { - SRequestObj *request; - tsem_t sem; - int32_t cnt; - int32_t total; - TdThreadSpinlock lock; -} Params; - -typedef struct { - int64_t id; - Params *params; - - SMLProtocolType protocol; - int8_t precision; - bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) - bool isRawLine; - int32_t ttl; - - SHashObj *childTables; - SHashObj *superTables; - SHashObj *pVgHash; - void *exec; - - STscObj *taos; - SCatalog *pCatalog; - SRequestObj *pRequest; - SQuery *pQuery; - - SSmlCostInfo cost; - int32_t affectedRows; - SSmlMsgBuf msgBuf; - SHashObj *dumplicateKey; // for dumplicate key - SArray *colsContainer; // for cols parse, if dataFormat == false - - cJSON *root; // for parse json -} SSmlHandle; -//================================================================================================= - -//================================================================================================= -static volatile int64_t linesSmlHandleId = 0; -static int64_t smlGenId() { - int64_t id; +#include "clientSml.h" - do { - id = atomic_add_fetch_64(&linesSmlHandleId, 1); - } while (id == 0); +int64_t smlToMilli[3] = {3600000LL, 60000LL, 1000LL}; +int64_t smlFactorNS[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; +int64_t smlFactorS[3] = {1000LL, 1000000LL, 1000000000LL}; - return id; +void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) { + NodeList *tmp = list; + while (tmp) { + if (fn == NULL) { + if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { + return tmp->data.value; + } + } else { + if (tmp->data.used && fn(tmp->data.key, key) == 0) { + return tmp->data.value; + } + } + + tmp = tmp->next; + } + return NULL; } -static inline bool smlDoubleToInt64OverFlow(double num) { - if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; - return false; +int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) { + NodeList *tmp = *list; + while (tmp) { + if (!tmp->data.used) break; + if (fn == NULL) { + if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { + return -1; + } + } else { + if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) { + return -1; + } + } + + tmp = tmp->next; + } + if (tmp) { + tmp->data.key = key; + tmp->data.keyLen = len; + tmp->data.value = value; + tmp->data.used = true; + } else { + NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList)); + if (newNode == NULL) { + return -1; + } + newNode->data.key = key; + newNode->data.keyLen = len; + newNode->data.value = value; + newNode->data.used = true; + newNode->next = *list; + *list = newNode; + } + return 0; } -static inline bool smlCheckDuplicateKey(const char *key, int32_t keyLen, SHashObj *pHash) { - void *val = taosHashGet(pHash, key, keyLen); - if (val) { - return true; +int nodeListSize(NodeList *list) { + int cnt = 0; + while (list) { + if (list->data.used) + cnt++; + else + break; + list = list->next; } - taosHashPut(pHash, key, keyLen, key, 1); + return cnt; +} + +inline bool smlDoubleToInt64OverFlow(double num) { + if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; return false; } -static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { +int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { if (pBuf->buf) { memset(pBuf->buf, 0, pBuf->len); if (msg1) strncat(pBuf->buf, msg1, pBuf->len); @@ -226,391 +108,310 @@ static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const return TSDB_CODE_SML_INVALID_DATA; } -static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSmlKv *kv, bool isTag, - ESchemaAction *action, SSmlHandle *info) { - uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL; - if (index) { - if (colField[*index].type != kv->type) { - uError("SML:0x%" PRIx64 " point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, - kv->key, colField[*index].type, kv->type); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if ((colField[*index].type == TSDB_DATA_TYPE_VARCHAR && - (colField[*index].bytes - VARSTR_HEADER_SIZE) < kv->length) || - (colField[*index].type == TSDB_DATA_TYPE_NCHAR && - ((colField[*index].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE < kv->length))) { - if (isTag) { - *action = SCHEMA_ACTION_CHANGE_TAG_SIZE; - } else { - *action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE; - } - } - } else { - if (isTag) { - *action = SCHEMA_ACTION_ADD_TAG; - } else { - *action = SCHEMA_ACTION_ADD_COLUMN; - } +int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision) { + char *endPtr = NULL; + int64_t tsInt64 = taosStr2Int64(value, &endPtr, 10); + if (unlikely(value + len != endPtr)) { + return -1; } - return 0; -} -static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { - int32_t result = 1; - while (result <= length) { - result *= 2; - } - if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { - result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE; - } else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - result = (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + if (unlikely(fromPrecision >= TSDB_TIME_PRECISION_HOURS)) { + int64_t unit = smlToMilli[fromPrecision - TSDB_TIME_PRECISION_HOURS]; + if (unit > INT64_MAX / tsInt64) { + return -1; + } + tsInt64 *= unit; + fromPrecision = TSDB_TIME_PRECISION_MILLI; } - if (type == TSDB_DATA_TYPE_NCHAR) { - result = result * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; - } else if (type == TSDB_DATA_TYPE_BINARY) { - result = result + VARSTR_HEADER_SIZE; - } - return result; + return convertTimePrecision(tsInt64, fromPrecision, toPrecision); } -static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, - ESchemaAction *action, bool isTag) { - int32_t code = TSDB_CODE_SUCCESS; - for (int j = 0; j < taosArrayGetSize(cols); ++j) { - if (j == 0 && !isTag) continue; - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j); - code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info); - if (code != TSDB_CODE_SUCCESS) { - return code; - } +int8_t smlGetTsTypeByLen(int32_t len) { + if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { + return TSDB_TIME_PRECISION_SECONDS; + } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { + return TSDB_TIME_PRECISION_MILLI; + } else { + return -1; } - return TSDB_CODE_SUCCESS; } -static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { - SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - int32_t i = 0; - for (; i < length; i++) { - taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); +SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen) { + SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); + if (!tag) { + return NULL; } - if (isTag) { - i = 0; - } else { - i = 1; - } - for (; i < taosArrayGetSize(cols); i++) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); - if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { - taosHashCleanup(hashTmp); - return -1; - } - } - taosHashCleanup(hashTmp); - return 0; -} + tag->sTableName = measure; + tag->sTableNameLen = measureLen; -static int32_t getBytes(uint8_t type, int32_t length) { - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - return smlFindNearestPowerOf2(length, type); - } else { - return tDataTypes[type].bytes; + tag->cols = taosArrayInit(numRows, POINTER_BYTES); + if (tag->cols == NULL) { + uError("SML:smlBuildTableInfo failed to allocate memory"); + goto cleanup; } + + // tag->tags = taosArrayInit(16, sizeof(SSmlKv)); + // if (tag->tags == NULL) { + // uError("SML:smlBuildTableInfo failed to allocate memory"); + // goto cleanup; + // } + return tag; + +cleanup: + taosMemoryFree(tag); + return NULL; } -static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, - SArray *results, int32_t numOfCols, bool isTag) { - for (int j = 0; j < taosArrayGetSize(cols); ++j) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j); - ESchemaAction action = SCHEMA_ACTION_NULL; - smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); - if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) { - SField field = {0}; - field.type = kv->type; - field.bytes = getBytes(kv->type, kv->length); - memcpy(field.name, kv->key, kv->keyLen); - taosArrayPush(results, &field); - } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { - uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); - uint16_t newIndex = *index; - if (isTag) newIndex -= numOfCols; - SField *field = (SField *)taosArrayGet(results, newIndex); - field->bytes = getBytes(kv->type, kv->length); +static int32_t smlParseTableName(SArray *tags, char *childTableName) { + size_t childTableNameLen = strlen(tsSmlChildTableName); + if (childTableNameLen <= 0) return TSDB_CODE_SUCCESS; + + for (int i = 0; i < taosArrayGetSize(tags); i++) { + SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); + // handle child table name + if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) { + memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); + break; } } + return TSDB_CODE_SUCCESS; } -// static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData, -// int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){ -static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta, - ESchemaAction action) { - SRequestObj *pRequest = NULL; - SMCreateStbReq pReq = {0}; - int32_t code = TSDB_CODE_SUCCESS; - SCmdMsgInfo pCmdMsg = {0}; +int32_t smlSetCTableName(SSmlTableInfo *oneTable) { + smlParseTableName(oneTable->tags, oneTable->childTableName); - // put front for free - pReq.numOfColumns = taosArrayGetSize(pColumns); - pReq.pColumns = pColumns; - pReq.numOfTags = taosArrayGetSize(pTags); - pReq.pTags = pTags; + if (strlen(oneTable->childTableName) == 0) { + SArray *dst = taosArrayDup(oneTable->tags, NULL); + RandTableName rName = {dst, oneTable->sTableName, (uint8_t)oneTable->sTableNameLen, oneTable->childTableName, 0}; - code = buildRequest(info->taos->id, "", 0, NULL, false, &pRequest, 0); - if (code != TSDB_CODE_SUCCESS) { - goto end; + buildChildTableName(&rName); + taosArrayDestroy(dst); + oneTable->uid = rName.uid; + } else { + oneTable->uid = *(uint64_t *)(oneTable->childTableName); } + return TSDB_CODE_SUCCESS; +} - pRequest->syncQuery = true; - if (!pRequest->pDb) { - code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; - goto end; +SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { + SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); + if (!meta) { + return NULL; } - if (action == SCHEMA_ACTION_CREATE_STABLE) { - pReq.colVer = 1; - pReq.tagVer = 1; - pReq.suid = 0; - pReq.source = TD_REQ_FROM_APP; - } else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { - pReq.colVer = pTableMeta->sversion; - pReq.tagVer = pTableMeta->tversion + 1; - pReq.suid = pTableMeta->uid; - pReq.source = TD_REQ_FROM_TAOX; - } else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE) { - pReq.colVer = pTableMeta->sversion + 1; - pReq.tagVer = pTableMeta->tversion; - pReq.suid = pTableMeta->uid; - pReq.source = TD_REQ_FROM_TAOX; - } + if (unlikely(!isDataFormat)) { + meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (meta->tagHash == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } - if (pReq.numOfTags == 0) { - pReq.numOfTags = 1; - SField field = {0}; - field.type = TSDB_DATA_TYPE_NCHAR; - field.bytes = 1; - strcpy(field.name, tsSmlTagName); - taosArrayPush(pReq.pTags, &field); + meta->colHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (meta->colHash == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } } - pReq.commentLen = -1; - pReq.igExists = true; - tNameExtractFullName(pName, pReq.name); - - pCmdMsg.epSet = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - pCmdMsg.msgType = TDMT_MND_CREATE_STB; - pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq); - pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); - if (NULL == pCmdMsg.pMsg) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; + meta->tags = taosArrayInit(32, sizeof(SSmlKv)); + if (meta->tags == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; } - tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); - - SQuery pQuery; - memset(&pQuery, 0, sizeof(pQuery)); - pQuery.execMode = QUERY_EXEC_MODE_RPC; - pQuery.pCmdMsg = &pCmdMsg; - pQuery.msgType = pQuery.pCmdMsg->msgType; - pQuery.stableQuery = true; - launchQueryImpl(pRequest, &pQuery, true, NULL); - - if (pRequest->code == TSDB_CODE_SUCCESS) { - catalogRemoveTableMeta(info->pCatalog, pName); + meta->cols = taosArrayInit(32, sizeof(SSmlKv)); + if (meta->cols == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; } - code = pRequest->code; - taosMemoryFree(pCmdMsg.pMsg); + return meta; -end: - destroyRequest(pRequest); - tFreeSMCreateStbReq(&pReq); - return code; +cleanup: + taosMemoryFree(meta); + return NULL; } -static int32_t smlModifyDBSchemas(SSmlHandle *info) { - int32_t code = 0; - SHashObj *hashTmp = NULL; - STableMeta *pTableMeta = NULL; +// uint16_t smlCalTypeSum(char* endptr, int32_t left){ +// uint16_t sum = 0; +// for(int i = 0; i < left; i++){ +// sum += endptr[i]; +// } +// return sum; +// } - SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; - tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); +#define RETURN_FALSE \ + smlBuildInvalidDataMsg(msg, "invalid data", pVal); \ + return false; - SRequestConnInfo conn = {0}; - conn.pTrans = info->taos->pAppInfo->pTransporter; - conn.requestId = info->pRequest->requestId; - conn.requestObjRefId = info->pRequest->self; - conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - - SSmlSTableMeta **tableMetaSml = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); - while (tableMetaSml) { - SSmlSTableMeta *sTableData = *tableMetaSml; - bool needCheckMeta = false; // for multi thread - - size_t superTableLen = 0; - void *superTable = taosHashGetKey(tableMetaSml, &superTableLen); - memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - memcpy(pName.tname, superTable, superTableLen); - - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - - if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) { - SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); - SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); - smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true); - smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false); - - code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); - goto end; - } - info->cost.numOfCreateSTables++; - taosMemoryFreeClear(pTableMeta); - - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); - goto end; - } - } else if (code == TSDB_CODE_SUCCESS) { - hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, - HASH_NO_LOCK); - for (uint16_t i = pTableMeta->tableInfo.numOfColumns; - i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); - } - - ESchemaAction action = SCHEMA_ACTION_NULL; - code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->tags, &action, true); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - if (action != SCHEMA_ACTION_NULL) { - SArray *pColumns = - taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); - SArray *pTags = - taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - SField field = {0}; - field.type = pTableMeta->schema[i].type; - field.bytes = pTableMeta->schema[i].bytes; - strcpy(field.name, pTableMeta->schema[i].name); - if (i < pTableMeta->tableInfo.numOfColumns) { - taosArrayPush(pColumns, &field); - } else { - taosArrayPush(pTags, &field); - } - } - smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, - pTableMeta->tableInfo.numOfColumns, true); - - code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); - goto end; - } - - info->cost.numOfAlterTagSTables++; - taosMemoryFreeClear(pTableMeta); - code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - } +#define SET_DOUBLE \ + kvVal->type = TSDB_DATA_TYPE_DOUBLE; \ + kvVal->d = result; + +#define SET_FLOAT \ + if (!IS_VALID_FLOAT(result)) { \ + smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_FLOAT; \ + kvVal->f = (float)result; + +#define SET_BIGINT \ + if (smlDoubleToInt64OverFlow(result)) { \ + errno = 0; \ + int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \ + if (errno == ERANGE) { \ + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_BIGINT; \ + kvVal->i = tmp; \ + return true; \ + } \ + kvVal->type = TSDB_DATA_TYPE_BIGINT; \ + kvVal->i = (int64_t)result; + +#define SET_INT \ + if (!IS_VALID_INT(result)) { \ + smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_INT; \ + kvVal->i = result; + +#define SET_SMALL_INT \ + if (!IS_VALID_SMALLINT(result)) { \ + smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_SMALLINT; \ + kvVal->i = result; + +#define SET_UBIGINT \ + if (result >= (double)UINT64_MAX || result < 0) { \ + errno = 0; \ + uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \ + if (errno == ERANGE || result < 0) { \ + smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ + kvVal->u = tmp; \ + return true; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ + kvVal->u = result; + +#define SET_UINT \ + if (!IS_VALID_UINT(result)) { \ + smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UINT; \ + kvVal->u = result; + +#define SET_USMALL_INT \ + if (!IS_VALID_USMALLINT(result)) { \ + smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_USMALLINT; \ + kvVal->u = result; + +#define SET_TINYINT \ + if (!IS_VALID_TINYINT(result)) { \ + smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_TINYINT; \ + kvVal->i = result; + +#define SET_UTINYINT \ + if (!IS_VALID_UTINYINT(result)) { \ + smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UTINYINT; \ + kvVal->u = result; + +bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->length; + char *endptr = NULL; + double result = taosStr2Double(pVal, &endptr); + if (pVal == endptr) { + RETURN_FALSE + } - taosHashClear(hashTmp); - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + int32_t left = len - (endptr - pVal); + if (left == 0) { + SET_DOUBLE + } else if (left == 3) { + if (endptr[0] == 'f' || endptr[0] == 'F') { + if (endptr[1] == '6' && endptr[2] == '4') { + SET_DOUBLE + } else if (endptr[1] == '3' && endptr[2] == '2') { + SET_FLOAT + } else { + RETURN_FALSE } - action = SCHEMA_ACTION_NULL; - code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, &action, false); - if (code != TSDB_CODE_SUCCESS) { - goto end; + } else if (endptr[0] == 'i' || endptr[0] == 'I') { + if (endptr[1] == '6' && endptr[2] == '4') { + SET_BIGINT + } else if (endptr[1] == '3' && endptr[2] == '2') { + SET_INT + } else if (endptr[1] == '1' && endptr[2] == '6') { + SET_SMALL_INT + } else { + RETURN_FALSE } - if (action != SCHEMA_ACTION_NULL) { - SArray *pColumns = - taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); - SArray *pTags = - taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - SField field = {0}; - field.type = pTableMeta->schema[i].type; - field.bytes = pTableMeta->schema[i].bytes; - strcpy(field.name, pTableMeta->schema[i].name); - if (i < pTableMeta->tableInfo.numOfColumns) { - taosArrayPush(pColumns, &field); - } else { - taosArrayPush(pTags, &field); - } - } - - smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns, - pTableMeta->tableInfo.numOfColumns, false); - - code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); - goto end; - } - - info->cost.numOfAlterColSTables++; - taosMemoryFreeClear(pTableMeta); - code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); - goto end; - } + } else if (endptr[0] == 'u' || endptr[0] == 'U') { + if (endptr[1] == '6' && endptr[2] == '4') { + SET_UBIGINT + } else if (endptr[1] == '3' && endptr[2] == '2') { + SET_UINT + } else if (endptr[1] == '1' && endptr[2] == '6') { + SET_USMALL_INT + } else { + RETURN_FALSE } - - needCheckMeta = true; - taosHashCleanup(hashTmp); - hashTmp = NULL; } else { - uError("SML:0x%" PRIx64 " load table meta error: %s", info->id, tstrerror(code)); - goto end; + RETURN_FALSE } - - if (needCheckMeta) { - code = smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags, - sTableData->tags, true); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " check tag failed. super table name %s", info->id, pName.tname); - goto end; + } else if (left == 2) { + if (endptr[0] == 'i' || endptr[0] == 'I') { + if (endptr[1] == '8') { + SET_TINYINT + } else { + RETURN_FALSE } - code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols, false); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " check cols failed. super table name %s", info->id, pName.tname); - goto end; + } else if (endptr[0] == 'u' || endptr[0] == 'U') { + if (endptr[1] == '8') { + SET_UTINYINT + } else { + RETURN_FALSE } + } else { + RETURN_FALSE } - - sTableData->tableMeta = pTableMeta; - - tableMetaSml = (SSmlSTableMeta **)taosHashIterate(info->superTables, tableMetaSml); + } else if (left == 1) { + if (endptr[0] == 'i' || endptr[0] == 'I') { + SET_BIGINT + } else if (endptr[0] == 'u' || endptr[0] == 'U') { + SET_UBIGINT + } else { + RETURN_FALSE + } + } else { + RETURN_FALSE; } - return 0; - -end: - taosHashCleanup(hashTmp); - taosMemoryFreeClear(pTableMeta); - // catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); - return code; + return true; } -static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { +bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->length; char *endptr = NULL; @@ -708,764 +509,459 @@ static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { return true; } -static bool smlParseBool(SSmlKv *kvVal) { - const char *pVal = kvVal->value; - int32_t len = kvVal->length; - if ((len == 1) && (pVal[0] == 't' || pVal[0] == 'T')) { - kvVal->i = true; - return true; - } +STableMeta *smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen) { + STableMeta *pTableMeta = NULL; - if ((len == 1) && (pVal[0] == 'f' || pVal[0] == 'F')) { - kvVal->i = false; - return true; - } + SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; + tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); - if ((len == 4) && !strncasecmp(pVal, "true", len)) { - kvVal->i = true; - return true; - } - if ((len == 5) && !strncasecmp(pVal, "false", len)) { - kvVal->i = false; - return true; - } - return false; -} + SRequestConnInfo conn = {0}; + conn.pTrans = info->taos->pAppInfo->pTransporter; + conn.requestId = info->pRequest->requestId; + conn.requestObjRefId = info->pRequest->self; + conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + memcpy(pName.tname, measure, measureLen); -static bool smlIsBinary(const char *pVal, uint16_t len) { - // binary: "abc" - if (len < 2) { - return false; - } - if (pVal[0] == '"' && pVal[len - 1] == '"') { - return true; - } - return false; + catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + return pTableMeta; } -static bool smlIsNchar(const char *pVal, uint16_t len) { - // nchar: L"abc" - if (len < 3) { - return false; - } - if ((pVal[0] == 'l' || pVal[0] == 'L') && pVal[1] == '"' && pVal[len - 1] == '"') { - return true; - } - return false; -} +static int64_t smlGenId() { + static volatile int64_t linesSmlHandleId = 0; -static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { - char *endPtr = NULL; - int64_t tsInt64 = taosStr2Int64(value, &endPtr, 10); - if (value + len != endPtr) { - return -1; - } - double ts = tsInt64; - switch (type) { - case TSDB_TIME_PRECISION_HOURS: - ts *= NANOSECOND_PER_HOUR; - tsInt64 *= NANOSECOND_PER_HOUR; - break; - case TSDB_TIME_PRECISION_MINUTES: - ts *= NANOSECOND_PER_MINUTE; - tsInt64 *= NANOSECOND_PER_MINUTE; - break; - case TSDB_TIME_PRECISION_SECONDS: - ts *= NANOSECOND_PER_SEC; - tsInt64 *= NANOSECOND_PER_SEC; - break; - case TSDB_TIME_PRECISION_MILLI: - ts *= NANOSECOND_PER_MSEC; - tsInt64 *= NANOSECOND_PER_MSEC; - break; - case TSDB_TIME_PRECISION_MICRO: - ts *= NANOSECOND_PER_USEC; - tsInt64 *= NANOSECOND_PER_USEC; - break; - case TSDB_TIME_PRECISION_NANO: - break; - default: - ASSERT(0); - } - if (ts >= (double)INT64_MAX || ts < 0) { - return -1; - } + int64_t id = 0; + do { + id = atomic_add_fetch_64(&linesSmlHandleId, 1); + } while (id == 0); - return tsInt64; + return id; } -static int8_t smlGetTsTypeByLen(int32_t len) { - if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { - return TSDB_TIME_PRECISION_SECONDS; - } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { - return TSDB_TIME_PRECISION_MILLI; +static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSmlKv *kv, bool isTag, + ESchemaAction *action, SSmlHandle *info) { + uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL; + if (index) { + if (colField[*index].type != kv->type) { + uError("SML:0x%" PRIx64 " point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, + kv->key, colField[*index].type, kv->type); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if ((colField[*index].type == TSDB_DATA_TYPE_VARCHAR && + (colField[*index].bytes - VARSTR_HEADER_SIZE) < kv->length) || + (colField[*index].type == TSDB_DATA_TYPE_NCHAR && + ((colField[*index].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE < kv->length))) { + if (isTag) { + *action = SCHEMA_ACTION_CHANGE_TAG_SIZE; + } else { + *action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE; + } + } } else { - return -1; + if (isTag) { + *action = SCHEMA_ACTION_ADD_TAG; + } else { + *action = SCHEMA_ACTION_ADD_COLUMN; + } } + return 0; } -static int8_t smlGetTsTypeByPrecision(int8_t precision) { - switch (precision) { - case TSDB_SML_TIMESTAMP_HOURS: - return TSDB_TIME_PRECISION_HOURS; - case TSDB_SML_TIMESTAMP_MILLI_SECONDS: - return TSDB_TIME_PRECISION_MILLI; - case TSDB_SML_TIMESTAMP_NANO_SECONDS: - case TSDB_SML_TIMESTAMP_NOT_CONFIGURED: - return TSDB_TIME_PRECISION_NANO; - case TSDB_SML_TIMESTAMP_MICRO_SECONDS: - return TSDB_TIME_PRECISION_MICRO; - case TSDB_SML_TIMESTAMP_SECONDS: - return TSDB_TIME_PRECISION_SECONDS; - case TSDB_SML_TIMESTAMP_MINUTES: - return TSDB_TIME_PRECISION_MINUTES; - default: - return -1; +static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { + int32_t result = 1; + while (result <= length) { + result *= 2; } -} - -static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t len) { - if (len == 0 || (len == 1 && data[0] == '0')) { - return taosGetTimestampNs(); + if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { + result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE; + } else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + result = (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; } - int8_t tsType = smlGetTsTypeByPrecision(info->precision); - if (tsType == -1) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp precision", NULL); - return -1; + if (type == TSDB_DATA_TYPE_NCHAR) { + result = result * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; + } else if (type == TSDB_DATA_TYPE_BINARY) { + result = result + VARSTR_HEADER_SIZE; } + return result; +} - int64_t ts = smlGetTimeValue(data, len, tsType); - if (ts == -1) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); - return -1; +static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, + ESchemaAction *action, bool isTag) { + int32_t code = TSDB_CODE_SUCCESS; + for (int j = 0; j < taosArrayGetSize(cols); ++j) { + if (j == 0 && !isTag) continue; + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); + code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } - return ts; + return TSDB_CODE_SUCCESS; } -static int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len) { - if (!data) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL); - return -1; - } - if (len == 1 && data[0] == '0') { - return taosGetTimestampNs(); - } - int8_t tsType = smlGetTsTypeByLen(len); - if (tsType == -1) { - smlBuildInvalidDataMsg(&info->msgBuf, - "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data); - return -1; - } - int64_t ts = smlGetTimeValue(data, len, tsType); - if (ts == -1) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); - return -1; +static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { + SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + int32_t i = 0; + for (; i < length; i++) { + taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); } - return ts; -} -static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArray *cols) { - int64_t ts = 0; - if (info->protocol == TSDB_SML_LINE_PROTOCOL) { - // uError("SML:data:%s,len:%d", data, len); - ts = smlParseInfluxTime(info, data, len); - } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - ts = smlParseOpenTsdbTime(info, data, len); + if (isTag) { + i = 0; } else { - ASSERT(0); + i = 1; } - uDebug("SML:0x%" PRIx64 " smlParseTS:%" PRId64, info->id, ts); - - if (ts <= 0) { - uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); - return TSDB_CODE_INVALID_TIMESTAMP; + for (; i < taosArrayGetSize(cols); i++) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { + taosHashCleanup(hashTmp); + return -1; + } } + taosHashCleanup(hashTmp); + return 0; +} - // add ts to - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) { - return TSDB_CODE_OUT_OF_MEMORY; +static int32_t getBytes(uint8_t type, int32_t length) { + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + return smlFindNearestPowerOf2(length, type); + } else { + return tDataTypes[type].bytes; } - - kv->key = TS; - kv->keyLen = TS_LEN; - kv->i = ts; - kv->type = TSDB_DATA_TYPE_TIMESTAMP; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - taosArrayPush(cols, &kv); - - return TSDB_CODE_SUCCESS; } -static int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { - // binary - if (smlIsBinary(pVal->value, pVal->length)) { - pVal->type = TSDB_DATA_TYPE_BINARY; - pVal->length -= BINARY_ADD_LEN; - if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - pVal->value += (BINARY_ADD_LEN - 1); - return TSDB_CODE_SUCCESS; - } - // nchar - if (smlIsNchar(pVal->value, pVal->length)) { - pVal->type = TSDB_DATA_TYPE_NCHAR; - pVal->length -= NCHAR_ADD_LEN; - if (pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; +static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, + SArray *results, int32_t numOfCols, bool isTag) { + for (int j = 0; j < taosArrayGetSize(cols); ++j) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); + ESchemaAction action = SCHEMA_ACTION_NULL; + smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); + if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) { + SField field = {0}; + field.type = kv->type; + field.bytes = getBytes(kv->type, kv->length); + memcpy(field.name, kv->key, kv->keyLen); + taosArrayPush(results, &field); + } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { + uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); + uint16_t newIndex = *index; + if (isTag) newIndex -= numOfCols; + SField *field = (SField *)taosArrayGet(results, newIndex); + field->bytes = getBytes(kv->type, kv->length); } - pVal->value += (NCHAR_ADD_LEN - 1); - return TSDB_CODE_SUCCESS; } - - // bool - if (smlParseBool(pVal)) { - pVal->type = TSDB_DATA_TYPE_BOOL; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - return TSDB_CODE_SUCCESS; - } - // number - if (smlParseNumber(pVal, msg)) { - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - return TSDB_CODE_SUCCESS; - } - - return TSDB_CODE_TSC_INVALID_VALUE; + return TSDB_CODE_SUCCESS; } -static int32_t smlParseInfluxString(const char *sql, const char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) { - if (!sql) return TSDB_CODE_SML_INVALID_DATA; - JUMP_SPACE(sql, sqlEnd) - if (*sql == COMMA) return TSDB_CODE_SML_INVALID_DATA; - elements->measure = sql; - - // parse measure - while (sql < sqlEnd) { - if ((sql != elements->measure) && IS_SLASH_LETTER(sql)) { - MOVE_FORWARD_ONE(sql, sqlEnd - sql); - sqlEnd--; - continue; - } - if (IS_COMMA(sql)) { - break; - } +// static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData, +// int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){ +static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta, + ESchemaAction action) { + SRequestObj *pRequest = NULL; + SMCreateStbReq pReq = {0}; + int32_t code = TSDB_CODE_SUCCESS; + SCmdMsgInfo pCmdMsg = {0}; - if (IS_SPACE(sql)) { - break; - } - sql++; - } - elements->measureLen = sql - elements->measure; - if (IS_INVALID_TABLE_LEN(elements->measureLen)) { - smlBuildInvalidDataMsg(msg, "measure is empty or too large than 192", NULL); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } + // put front for free + pReq.numOfColumns = taosArrayGetSize(pColumns); + pReq.pColumns = pColumns; + pReq.numOfTags = taosArrayGetSize(pTags); + pReq.pTags = pTags; - // parse tag - if (*sql == SPACE) { - elements->tagsLen = 0; - } else { - if (*sql == COMMA) sql++; - elements->tags = sql; - while (sql < sqlEnd) { - if (IS_SPACE(sql)) { - break; - } - sql++; - } - elements->tagsLen = sql - elements->tags; + code = buildRequest(info->taos->id, "", 0, NULL, false, &pRequest, 0); + if (code != TSDB_CODE_SUCCESS) { + goto end; } - elements->measureTagsLen = sql - elements->measure; - // parse cols - JUMP_SPACE(sql, sqlEnd) - elements->cols = sql; - bool isInQuote = false; - while (sql < sqlEnd) { - if (IS_QUOTE(sql)) { - isInQuote = !isInQuote; - } - if (!isInQuote && IS_SPACE(sql)) { - break; - } - sql++; - } - if (isInQuote) { - smlBuildInvalidDataMsg(msg, "only one quote", elements->cols); - return TSDB_CODE_SML_INVALID_DATA; - } - elements->colsLen = sql - elements->cols; - if (elements->colsLen == 0) { - smlBuildInvalidDataMsg(msg, "cols is empty", NULL); - return TSDB_CODE_SML_INVALID_DATA; + pRequest->syncQuery = true; + if (!pRequest->pDb) { + code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + goto end; } - // parse timestamp - JUMP_SPACE(sql, sqlEnd) - elements->timestamp = sql; - while (sql < sqlEnd) { - if (isspace(*sql)) { - break; - } - sql++; + if (action == SCHEMA_ACTION_CREATE_STABLE) { + pReq.colVer = 1; + pReq.tagVer = 1; + pReq.suid = 0; + pReq.source = TD_REQ_FROM_APP; + } else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { + pReq.colVer = pTableMeta->sversion; + pReq.tagVer = pTableMeta->tversion + 1; + pReq.suid = pTableMeta->uid; + pReq.source = TD_REQ_FROM_TAOX; + } else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE) { + pReq.colVer = pTableMeta->sversion + 1; + pReq.tagVer = pTableMeta->tversion; + pReq.suid = pTableMeta->uid; + pReq.source = TD_REQ_FROM_TAOX; } - elements->timestampLen = sql - elements->timestamp; - - return TSDB_CODE_SUCCESS; -} -static void smlParseTelnetElement(const char **sql, const char *sqlEnd, const char **data, int32_t *len) { - while (*sql < sqlEnd) { - if (**sql != SPACE && !(*data)) { - *data = *sql; - } else if (**sql == SPACE && *data) { - *len = *sql - *data; - break; - } - (*sql)++; + if (pReq.numOfTags == 0) { + pReq.numOfTags = 1; + SField field = {0}; + field.type = TSDB_DATA_TYPE_NCHAR; + field.bytes = 1; + strcpy(field.name, tsSmlTagName); + taosArrayPush(pReq.pTags, &field); } -} - -static int32_t smlParseTelnetTags(const char *data, const char *sqlEnd, SArray *cols, char *childTableName, - SHashObj *dumplicateKey, SSmlMsgBuf *msg) { - if (!cols) return TSDB_CODE_OUT_OF_MEMORY; - const char *sql = data; - size_t childTableNameLen = strlen(tsSmlChildTableName); - while (sql < sqlEnd) { - JUMP_SPACE(sql, sqlEnd) - if (*sql == '\0') break; - - const char *key = sql; - int32_t keyLen = 0; - - // parse key - while (sql < sqlEnd) { - if (*sql == SPACE) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; - } - if (*sql == EQUAL) { - keyLen = sql - key; - sql++; - break; - } - sql++; - } - - if (IS_INVALID_COL_LEN(keyLen)) { - smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { - smlBuildInvalidDataMsg(msg, "dumplicate key", key); - return TSDB_CODE_TSC_DUP_NAMES; - } - - // parse value - const char *value = sql; - int32_t valueLen = 0; - while (sql < sqlEnd) { - // parse value - if (*sql == SPACE) { - break; - } - if (*sql == EQUAL) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; - } - sql++; - } - valueLen = sql - value; - - if (valueLen == 0) { - smlBuildInvalidDataMsg(msg, "invalid value", value); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - // handle child table name - if (childTableNameLen != 0 && strncmp(key, tsSmlChildTableName, keyLen) == 0) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, value, (valueLen < TSDB_TABLE_NAME_LEN ? valueLen : TSDB_TABLE_NAME_LEN)); - continue; - } - - if (valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - // add kv to SSmlKv - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - kv->key = key; - kv->keyLen = keyLen; - kv->value = value; - kv->length = valueLen; - kv->type = TSDB_DATA_TYPE_NCHAR; + pReq.commentLen = -1; + pReq.igExists = true; + tNameExtractFullName(pName, pReq.name); - taosArrayPush(cols, &kv); + pCmdMsg.epSet = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + pCmdMsg.msgType = TDMT_MND_CREATE_STB; + pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq); + pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); + if (NULL == pCmdMsg.pMsg) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; } + tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); - return TSDB_CODE_SUCCESS; -} + SQuery pQuery; + memset(&pQuery, 0, sizeof(pQuery)); + pQuery.execMode = QUERY_EXEC_MODE_RPC; + pQuery.pCmdMsg = &pCmdMsg; + pQuery.msgType = pQuery.pCmdMsg->msgType; + pQuery.stableQuery = true; -// format: =[ =] -static int32_t smlParseTelnetString(SSmlHandle *info, const char *sql, const char *sqlEnd, SSmlTableInfo *tinfo, - SArray *cols) { - if (!sql) return TSDB_CODE_SML_INVALID_DATA; + launchQueryImpl(pRequest, &pQuery, true, NULL); - // parse metric - smlParseTelnetElement(&sql, sqlEnd, &tinfo->sTableName, &tinfo->sTableNameLen); - if (!(tinfo->sTableName) || IS_INVALID_TABLE_LEN(tinfo->sTableNameLen)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + if (pRequest->code == TSDB_CODE_SUCCESS) { + catalogRemoveTableMeta(info->pCatalog, pName); } + code = pRequest->code; + taosMemoryFree(pCmdMsg.pMsg); - // parse timestamp - const char *timestamp = NULL; - int32_t tLen = 0; - smlParseTelnetElement(&sql, sqlEnd, ×tamp, &tLen); - if (!timestamp || tLen == 0) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); - return TSDB_CODE_SML_INVALID_DATA; - } +end: + destroyRequest(pRequest); + tFreeSMCreateStbReq(&pReq); + return code; +} - int32_t ret = smlParseTS(info, timestamp, tLen, cols); - if (ret != TSDB_CODE_SUCCESS) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); - return ret; +static int32_t smlModifyDBSchemas(SSmlHandle *info) { + if (info->dataFormat && !info->needModifySchema) { + return TSDB_CODE_SUCCESS; } + int32_t code = 0; + SHashObj *hashTmp = NULL; + STableMeta *pTableMeta = NULL; - // parse value - const char *value = NULL; - int32_t valueLen = 0; - smlParseTelnetElement(&sql, sqlEnd, &value, &valueLen); - if (!value || valueLen == 0) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); - return TSDB_CODE_TSC_INVALID_VALUE; - } + SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; + tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - taosArrayPush(cols, &kv); - kv->key = VALUE; - kv->keyLen = VALUE_LEN; - kv->value = value; - kv->length = valueLen; - if ((ret = smlParseValue(kv, &info->msgBuf)) != TSDB_CODE_SUCCESS) { - return ret; - } + SRequestConnInfo conn = {0}; + conn.pTrans = info->taos->pAppInfo->pTransporter; + conn.requestId = info->pRequest->requestId; + conn.requestObjRefId = info->pRequest->self; + conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - // parse tags - ret = smlParseTelnetTags(sql, sqlEnd, tinfo->tags, tinfo->childTableName, info->dumplicateKey, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); - return ret; - } + NodeList *tmp = info->superTables; + while (tmp) { + SSmlSTableMeta *sTableData = (SSmlSTableMeta *)tmp->data.value; + bool needCheckMeta = false; // for multi thread - return TSDB_CODE_SUCCESS; -} + size_t superTableLen = (size_t)tmp->data.keyLen; + const void *superTable = tmp->data.key; + memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + memcpy(pName.tname, superTable, superTableLen); -static int32_t smlParseCols(const char *data, int32_t len, SArray *cols, char *childTableName, bool isTag, - SHashObj *dumplicateKey, SSmlMsgBuf *msg) { - if (len == 0) { - return TSDB_CODE_SUCCESS; - } + code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - size_t childTableNameLen = strlen(tsSmlChildTableName); - const char *sql = data; - while (sql < data + len) { - const char *key = sql; - int32_t keyLen = 0; + if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) { + SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); + SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); + smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true); + smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false); - while (sql < data + len) { - // parse key - if (IS_COMMA(sql)) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; - } - if (IS_EQUAL(sql)) { - keyLen = sql - key; - sql++; - break; + code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); + goto end; } - sql++; - } - - if (IS_INVALID_COL_LEN(keyLen)) { - smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { - smlBuildInvalidDataMsg(msg, "dumplicate key", key); - return TSDB_CODE_TSC_DUP_NAMES; - } + info->cost.numOfCreateSTables++; + taosMemoryFreeClear(pTableMeta); - // parse value - const char *value = sql; - int32_t valueLen = 0; - bool isInQuote = false; - while (sql < data + len) { - // parse value - if (!isTag && IS_QUOTE(sql)) { - isInQuote = !isInQuote; - sql++; - continue; - } - if (!isInQuote && IS_COMMA(sql)) { - break; + code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); + goto end; } - if (!isInQuote && IS_EQUAL(sql)) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; + } else if (code == TSDB_CODE_SUCCESS) { + hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, + HASH_NO_LOCK); + for (uint16_t i = pTableMeta->tableInfo.numOfColumns; + i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { + taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); } - sql++; - } - valueLen = sql - value; - sql++; - if (isInQuote) { - smlBuildInvalidDataMsg(msg, "only one quote", value); - return TSDB_CODE_SML_INVALID_DATA; - } - if (valueLen == 0) { - smlBuildInvalidDataMsg(msg, "invalid value", value); - return TSDB_CODE_SML_INVALID_DATA; - } - PROCESS_SLASH(key, keyLen) - PROCESS_SLASH(value, valueLen) - - // handle child table name - if (childTableName && childTableNameLen != 0 && strncmp(key, tsSmlChildTableName, keyLen) == 0) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, value, (valueLen < TSDB_TABLE_NAME_LEN ? valueLen : TSDB_TABLE_NAME_LEN)); - continue; - } - - // add kv to SSmlKv - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - if (cols) taosArrayPush(cols, &kv); - - kv->key = key; - kv->keyLen = keyLen; - kv->value = value; - kv->length = valueLen; - if (isTag) { - if (valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - kv->type = TSDB_DATA_TYPE_NCHAR; - } else { - int32_t ret = smlParseValue(kv, msg); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + ESchemaAction action = SCHEMA_ACTION_NULL; + code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->tags, &action, true); + if (code != TSDB_CODE_SUCCESS) { + goto end; } - } - } + if (action != SCHEMA_ACTION_NULL) { + SArray *pColumns = + taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); + SArray *pTags = + taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - return TSDB_CODE_SUCCESS; -} + for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { + SField field = {0}; + field.type = pTableMeta->schema[i].type; + field.bytes = pTableMeta->schema[i].bytes; + strcpy(field.name, pTableMeta->schema[i].name); + if (i < pTableMeta->tableInfo.numOfColumns) { + taosArrayPush(pColumns, &field); + } else { + taosArrayPush(pTags, &field); + } + } + smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, + pTableMeta->tableInfo.numOfColumns, true); -static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, SSmlMsgBuf *msg) { - for (int i = 0; i < taosArrayGetSize(cols); ++i) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); + code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); + goto end; + } - int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); - if (index) { - SSmlKv **value = (SSmlKv **)taosArrayGet(metaArray, *index); - if (kv->type != (*value)->type) { - smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); - return TSDB_CODE_SML_NOT_SAME_TYPE; - } else { - if (IS_VAR_DATA_TYPE(kv->type)) { // update string len, if bigger - if (kv->length > (*value)->length) { - *value = kv; - } + info->cost.numOfAlterTagSTables++; + taosMemoryFreeClear(pTableMeta); + code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + goto end; } } - } else { - size_t tmp = taosArrayGetSize(metaArray); - ASSERT(tmp <= INT16_MAX); - int16_t size = tmp; - taosArrayPush(metaArray, &kv); - taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); - } - } - return TSDB_CODE_SUCCESS; -} + taosHashClear(hashTmp); + for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { + taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + } + action = SCHEMA_ACTION_NULL; + code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, &action, false); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + if (action != SCHEMA_ACTION_NULL) { + SArray *pColumns = + taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); + SArray *pTags = + taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); -static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) { - for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); - taosArrayPush(metaArray, &kv); - taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); - } -} + for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { + SField field = {0}; + field.type = pTableMeta->schema[i].type; + field.bytes = pTableMeta->schema[i].bytes; + strcpy(field.name, pTableMeta->schema[i].name); + if (i < pTableMeta->tableInfo.numOfColumns) { + taosArrayPush(pColumns, &field); + } else { + taosArrayPush(pTags, &field); + } + } -static SSmlTableInfo *smlBuildTableInfo() { - SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); - if (!tag) { - return NULL; - } + smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns, + pTableMeta->tableInfo.numOfColumns, false); - tag->cols = taosArrayInit(16, POINTER_BYTES); - if (tag->cols == NULL) { - uError("SML:smlBuildTableInfo failed to allocate memory"); - goto cleanup; - } + code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); + goto end; + } - tag->tags = taosArrayInit(16, POINTER_BYTES); - if (tag->tags == NULL) { - uError("SML:smlBuildTableInfo failed to allocate memory"); - goto cleanup; - } - return tag; + info->cost.numOfAlterColSTables++; + taosMemoryFreeClear(pTableMeta); + code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); + goto end; + } + } -cleanup: - taosMemoryFree(tag); - return NULL; -} + needCheckMeta = true; + taosHashCleanup(hashTmp); + hashTmp = NULL; + } else { + uError("SML:0x%" PRIx64 " load table meta error: %s", info->id, tstrerror(code)); + goto end; + } -static void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) { - if (info->dataFormat) { - for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { - SArray *kvArray = (SArray *)taosArrayGetP(tag->cols, i); - for (int j = 0; j < taosArrayGetSize(kvArray); ++j) { - SSmlKv *p = (SSmlKv *)taosArrayGetP(kvArray, j); - taosMemoryFree(p); + if (needCheckMeta) { + code = smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags, + sTableData->tags, true); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " check tag failed. super table name %s", info->id, pName.tname); + goto end; } - taosArrayDestroy(kvArray); - } - } else { - for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { - SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i); - void **p1 = (void **)taosHashIterate(kvHash, NULL); - while (p1) { - taosMemoryFree(*p1); - p1 = (void **)taosHashIterate(kvHash, p1); + code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols, false); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " check cols failed. super table name %s", info->id, pName.tname); + goto end; } - taosHashCleanup(kvHash); } - } - for (size_t i = 0; i < taosArrayGetSize(tag->tags); i++) { - SSmlKv *p = (SSmlKv *)taosArrayGetP(tag->tags, i); - taosMemoryFree(p); - } - taosArrayDestroy(tag->cols); - taosArrayDestroy(tag->tags); - taosMemoryFree(tag); -} -static int32_t smlKvTimeArrayCompare(const void *key1, const void *key2) { - SArray *s1 = *(SArray **)key1; - SArray *s2 = *(SArray **)key2; - SSmlKv *kv1 = (SSmlKv *)taosArrayGetP(s1, 0); - SSmlKv *kv2 = (SSmlKv *)taosArrayGetP(s2, 0); - ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP); - if (kv1->i < kv2->i) { - return -1; - } else if (kv1->i > kv2->i) { - return 1; - } else { - return 0; - } -} + sTableData->tableMeta = pTableMeta; -static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) { - SHashObj *s1 = *(SHashObj **)key1; - SHashObj *s2 = *(SHashObj **)key2; - SSmlKv **kv1pp = (SSmlKv **)taosHashGet(s1, TS, TS_LEN); - SSmlKv **kv2pp = (SSmlKv **)taosHashGet(s2, TS, TS_LEN); - if (!kv1pp || !kv2pp) { - uError("smlKvTimeHashCompare kv is null"); - return -1; - } - SSmlKv *kv1 = *kv1pp; - SSmlKv *kv2 = *kv2pp; - if (!kv1 || kv1->type != TSDB_DATA_TYPE_TIMESTAMP) { - uError("smlKvTimeHashCompare kv1"); - return -1; - } - if (!kv2 || kv2->type != TSDB_DATA_TYPE_TIMESTAMP) { - uError("smlKvTimeHashCompare kv2"); - return -1; - } - if (kv1->i < kv2->i) { - return -1; - } else if (kv1->i > kv2->i) { - return 1; - } else { - return 0; + tmp = tmp->next; } + return 0; + +end: + taosHashCleanup(hashTmp); + taosMemoryFreeClear(pTableMeta); + // catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); + return code; } -static int32_t smlDealCols(SSmlTableInfo *oneTable, bool dataFormat, SArray *cols) { - if (dataFormat) { - void *p = taosArraySearch(oneTable->cols, &cols, smlKvTimeArrayCompare, TD_GT); - if (p == NULL) { - taosArrayPush(oneTable->cols, &cols); - } else { - taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols); +/* +static int32_t smlCheckDupUnit(SHashObj *dumplicateKey, SArray *tags, SSmlMsgBuf *msg){ + for(int i = 0; i < taosArrayGetSize(tags); i++) { + SSmlKv *tag = taosArrayGet(tags, i); + if (smlCheckDuplicateKey(tag->key, tag->keyLen, dumplicateKey)) { + smlBuildInvalidDataMsg(msg, "dumplicate key", tag->key); + return TSDB_CODE_TSC_DUP_NAMES; } - return TSDB_CODE_SUCCESS; - } - - SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (!kvHash) { - uError("SML:smlDealCols failed to allocate memory"); - return TSDB_CODE_OUT_OF_MEMORY; - } - for (size_t i = 0; i < taosArrayGetSize(cols); i++) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); - taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); - } - - void *p = taosArraySearch(oneTable->cols, &kvHash, smlKvTimeHashCompare, TD_GT); - if (p == NULL) { - taosArrayPush(oneTable->cols, &kvHash); - } else { - taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &kvHash); } return TSDB_CODE_SUCCESS; } -static SSmlSTableMeta *smlBuildSTableMeta() { - SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); - if (!meta) { - return NULL; - } - meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (meta->tagHash == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; +static int32_t smlJudgeDupColName(SArray *cols, SArray *tags, SSmlMsgBuf *msg) { + SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + int ret = smlCheckDupUnit(dumplicateKey, cols, msg); + if(ret != TSDB_CODE_SUCCESS){ + goto end; } - - meta->colHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (meta->colHash == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; + ret = smlCheckDupUnit(dumplicateKey, tags, msg); + if(ret != TSDB_CODE_SUCCESS){ + goto end; } - meta->tags = taosArrayInit(32, POINTER_BYTES); - if (meta->tags == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; - } + end: + taosHashCleanup(dumplicateKey); + return ret; +} +*/ - meta->cols = taosArrayInit(32, POINTER_BYTES); - if (meta->cols == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; +static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) { + for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); + if (ret == 0) { + taosArrayPush(metaArray, kv); + } } - return meta; - -cleanup: - taosMemoryFree(meta); - return NULL; } static void smlDestroySTableMeta(SSmlSTableMeta *meta) { @@ -1477,840 +973,214 @@ static void smlDestroySTableMeta(SSmlSTableMeta *meta) { taosMemoryFree(meta); } -static void smlDestroyCols(SArray *cols) { - if (!cols) return; +static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg) { for (int i = 0; i < taosArrayGetSize(cols); ++i) { - void *kv = taosArrayGetP(cols, i); - taosMemoryFree(kv); - } -} - -static void smlDestroyInfo(SSmlHandle *info) { - if (!info) return; - qDestroyQuery(info->pQuery); - smlDestroyHandle(info->exec); - - // destroy info->childTables - void **p1 = (void **)taosHashIterate(info->childTables, NULL); - while (p1) { - smlDestroyTableInfo(info, (SSmlTableInfo *)(*p1)); - p1 = (void **)taosHashIterate(info->childTables, p1); - } - taosHashCleanup(info->childTables); - - // destroy info->superTables - p1 = (void **)taosHashIterate(info->superTables, NULL); - while (p1) { - smlDestroySTableMeta((SSmlSTableMeta *)(*p1)); - p1 = (void **)taosHashIterate(info->superTables, p1); - } - taosHashCleanup(info->superTables); - - // destroy info->pVgHash - taosHashCleanup(info->pVgHash); - taosHashCleanup(info->dumplicateKey); - if (!info->dataFormat) { - taosArrayDestroy(info->colsContainer); - } - destroyRequest(info->pRequest); - - cJSON_Delete(info->root); - taosMemoryFreeClear(info); -} - -static SSmlHandle *smlBuildSmlInfo(STscObj *pTscObj, SRequestObj *request, SMLProtocolType protocol, int8_t precision) { - int32_t code = TSDB_CODE_SUCCESS; - SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle)); - if (NULL == info) { - return NULL; - } - info->id = smlGenId(); - - info->pQuery = (SQuery *)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == info->pQuery) { - uError("SML:0x%" PRIx64 " create info->pQuery error", info->id); - goto cleanup; - } - info->pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - info->pQuery->haveResultSet = false; - info->pQuery->msgType = TDMT_VND_SUBMIT; - info->pQuery->pRoot = (SNode *)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT); - if (NULL == info->pQuery->pRoot) { - uError("SML:0x%" PRIx64 " create info->pQuery->pRoot error", info->id); - goto cleanup; - } - - if (pTscObj) { - info->taos = pTscObj; - code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); - goto cleanup; - } - } - - info->precision = precision; - info->protocol = protocol; - if (protocol == TSDB_SML_LINE_PROTOCOL) { - info->dataFormat = tsSmlDataFormat; - } else { - info->dataFormat = true; - } - - if (request) { - info->pRequest = request; - info->msgBuf.buf = info->pRequest->msgBuf; - info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; - info->pRequest->stmtType = info->pQuery->pRoot->type; - } - - info->exec = smlInitHandle(info->pQuery); - info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - - info->dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (!info->dataFormat) { - info->colsContainer = taosArrayInit(32, POINTER_BYTES); - if (NULL == info->colsContainer) { - uError("SML:0x%" PRIx64 " create info failed", info->id); - goto cleanup; - } - } - if (NULL == info->exec || NULL == info->childTables || NULL == info->superTables || NULL == info->pVgHash || - NULL == info->dumplicateKey) { - uError("SML:0x%" PRIx64 " create info failed", info->id); - goto cleanup; - } - - return info; -cleanup: - smlDestroyInfo(info); - return NULL; -} - -/************* TSDB_SML_JSON_PROTOCOL function start **************/ -static int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *root, SSmlTableInfo *tinfo) { - cJSON *metric = cJSON_GetObjectItem(root, "metric"); - if (!cJSON_IsString(metric)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - tinfo->sTableNameLen = strlen(metric->valuestring); - if (IS_INVALID_TABLE_LEN(tinfo->sTableNameLen)) { - uError("OTD:0x%" PRIx64 " Metric lenght is 0 or large than 192", info->id); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - - tinfo->sTableName = metric->valuestring; - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsVal) { - int32_t size = cJSON_GetArraySize(root); - if (size != OTD_JSON_SUB_FIELDS_NUM) { - return TSDB_CODE_TSC_INVALID_JSON; - } + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - cJSON *value = cJSON_GetObjectItem(root, "value"); - if (!cJSON_IsNumber(value)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *type = cJSON_GetObjectItem(root, "type"); - if (!cJSON_IsString(type)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - double timeDouble = value->valuedouble; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - - if (timeDouble == 0) { - *tsVal = taosGetTimestampNs(); - return TSDB_CODE_SUCCESS; - } - - if (timeDouble < 0) { - return TSDB_CODE_INVALID_TIMESTAMP; - } - - *tsVal = timeDouble; - size_t typeLen = strlen(type->valuestring); - if (typeLen == 1 && (type->valuestring[0] == 's' || type->valuestring[0] == 'S')) { - // seconds - *tsVal = *tsVal * NANOSECOND_PER_SEC; - timeDouble = timeDouble * NANOSECOND_PER_SEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - } else if (typeLen == 2 && (type->valuestring[1] == 's' || type->valuestring[1] == 'S')) { - switch (type->valuestring[0]) { - case 'm': - case 'M': - // milliseconds - *tsVal = *tsVal * NANOSECOND_PER_MSEC; - timeDouble = timeDouble * NANOSECOND_PER_MSEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - break; - case 'u': - case 'U': - // microseconds - *tsVal = *tsVal * NANOSECOND_PER_USEC; - timeDouble = timeDouble * NANOSECOND_PER_USEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; + int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); + if (index) { + SSmlKv *value = (SSmlKv *)taosArrayGet(metaArray, *index); + if (isTag) { + if (kv->length > value->length) { + value->length = kv->length; } - break; - case 'n': - case 'N': - break; - default: - return TSDB_CODE_TSC_INVALID_JSON; - } - } else { - return TSDB_CODE_TSC_INVALID_JSON; - } - - return TSDB_CODE_SUCCESS; -} - -static uint8_t smlGetTimestampLen(int64_t num) { - uint8_t len = 0; - while ((num /= 10) != 0) { - len++; - } - len++; - return len; -} - -static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) { - // Timestamp must be the first KV to parse - int64_t tsVal = 0; - - cJSON *timestamp = cJSON_GetObjectItem(root, "timestamp"); - if (cJSON_IsNumber(timestamp)) { - // timestamp value 0 indicates current system time - double timeDouble = timestamp->valuedouble; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - - if (timeDouble < 0) { - return TSDB_CODE_INVALID_TIMESTAMP; - } - - uint8_t tsLen = smlGetTimestampLen((int64_t)timeDouble); - tsVal = (int64_t)timeDouble; - if (tsLen == TSDB_TIME_PRECISION_SEC_DIGITS) { - tsVal = tsVal * NANOSECOND_PER_SEC; - timeDouble = timeDouble * NANOSECOND_PER_SEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - } else if (tsLen == TSDB_TIME_PRECISION_MILLI_DIGITS) { - tsVal = tsVal * NANOSECOND_PER_MSEC; - timeDouble = timeDouble * NANOSECOND_PER_MSEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - } else if (timeDouble == 0) { - tsVal = taosGetTimestampNs(); - } else { - return TSDB_CODE_INVALID_TIMESTAMP; - } - } else if (cJSON_IsObject(timestamp)) { - int32_t ret = smlParseTSFromJSONObj(info, timestamp, &tsVal); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " Failed to parse timestamp from JSON Obj", info->id); - return ret; - } - } else { - return TSDB_CODE_TSC_INVALID_JSON; - } - - // add ts to - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) { - return TSDB_CODE_OUT_OF_MEMORY; - } - kv->key = TS; - kv->keyLen = TS_LEN; - kv->i = tsVal; - kv->type = TSDB_DATA_TYPE_TIMESTAMP; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - taosArrayPush(cols, &kv); - return TSDB_CODE_SUCCESS; -} - -static int32_t smlConvertJSONBool(SSmlKv *pVal, char *typeStr, cJSON *value) { - if (strcasecmp(typeStr, "bool") != 0) { - uError("OTD:invalid type(%s) for JSON Bool", typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - pVal->type = TSDB_DATA_TYPE_BOOL; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valueint; - - return TSDB_CODE_SUCCESS; -} - -static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { - // tinyint - if (strcasecmp(typeStr, "i8") == 0 || strcasecmp(typeStr, "tinyint") == 0) { - if (!IS_VALID_TINYINT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(tinyint)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_TINYINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // smallint - if (strcasecmp(typeStr, "i16") == 0 || strcasecmp(typeStr, "smallint") == 0) { - if (!IS_VALID_SMALLINT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(smallint)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_SMALLINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // int - if (strcasecmp(typeStr, "i32") == 0 || strcasecmp(typeStr, "int") == 0) { - if (!IS_VALID_INT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(int)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_INT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // bigint - if (strcasecmp(typeStr, "i64") == 0 || strcasecmp(typeStr, "bigint") == 0) { - pVal->type = TSDB_DATA_TYPE_BIGINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - if (value->valuedouble >= (double)INT64_MAX) { - pVal->i = INT64_MAX; - } else if (value->valuedouble <= (double)INT64_MIN) { - pVal->i = INT64_MIN; - } else { - pVal->i = value->valuedouble; - } - return TSDB_CODE_SUCCESS; - } - // float - if (strcasecmp(typeStr, "f32") == 0 || strcasecmp(typeStr, "float") == 0) { - if (!IS_VALID_FLOAT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(float)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_FLOAT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->f = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // double - if (strcasecmp(typeStr, "f64") == 0 || strcasecmp(typeStr, "double") == 0) { - pVal->type = TSDB_DATA_TYPE_DOUBLE; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->d = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - - // if reach here means type is unsupported - uError("OTD:invalid type(%s) for JSON Number", typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; -} - -static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) { - if (strcasecmp(typeStr, "binary") == 0) { - pVal->type = TSDB_DATA_TYPE_BINARY; - } else if (strcasecmp(typeStr, "nchar") == 0) { - pVal->type = TSDB_DATA_TYPE_NCHAR; - } else { - uError("OTD:invalid type(%s) for JSON String", typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - pVal->length = (int16_t)strlen(value->valuestring); - - if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - if (pVal->type == TSDB_DATA_TYPE_NCHAR && - pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - - pVal->value = value->valuestring; - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseValueFromJSONObj(cJSON *root, SSmlKv *kv) { - int32_t ret = TSDB_CODE_SUCCESS; - int32_t size = cJSON_GetArraySize(root); - - if (size != OTD_JSON_SUB_FIELDS_NUM) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *value = cJSON_GetObjectItem(root, "value"); - if (value == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *type = cJSON_GetObjectItem(root, "type"); - if (!cJSON_IsString(type)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - switch (value->type) { - case cJSON_True: - case cJSON_False: { - ret = smlConvertJSONBool(kv, type->valuestring, value); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - break; - } - case cJSON_Number: { - ret = smlConvertJSONNumber(kv, type->valuestring, value); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - break; - } - case cJSON_String: { - ret = smlConvertJSONString(kv, type->valuestring, value); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - break; - } - default: - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { - switch (root->type) { - case cJSON_True: - case cJSON_False: { - kv->type = TSDB_DATA_TYPE_BOOL; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - kv->i = root->valueint; - break; - } - case cJSON_Number: { - kv->type = TSDB_DATA_TYPE_DOUBLE; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - kv->d = root->valuedouble; - break; - } - case cJSON_String: { - /* set default JSON type to binary/nchar according to - * user configured parameter tsDefaultJSONStrType - */ + continue; + } + if (kv->type != value->type) { + smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); + return TSDB_CODE_SML_NOT_SAME_TYPE; + } - char *tsDefaultJSONStrType = "nchar"; // todo - smlConvertJSONString(kv, tsDefaultJSONStrType, root); - break; - } - case cJSON_Object: { - int32_t ret = smlParseValueFromJSONObj(root, kv); - if (ret != TSDB_CODE_SUCCESS) { - uError("OTD:Failed to parse value from JSON Obj"); - return ret; + if (IS_VAR_DATA_TYPE(kv->type) && (kv->length > value->length)) { // update string len, if bigger + value->length = kv->length; + } + } else { + size_t tmp = taosArrayGetSize(metaArray); + ASSERT(tmp <= INT16_MAX); + int16_t size = tmp; + int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); + if (ret == 0) { + taosArrayPush(metaArray, kv); } - break; } - default: - return TSDB_CODE_TSC_INVALID_JSON; } return TSDB_CODE_SUCCESS; } -static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { - if (!cols) return TSDB_CODE_OUT_OF_MEMORY; - cJSON *metricVal = cJSON_GetObjectItem(root, "value"); - if (metricVal == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) { - return TSDB_CODE_OUT_OF_MEMORY; +static void smlDestroyTableInfo(SSmlTableInfo *tag) { + for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { + SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i); + taosHashCleanup(kvHash); } - taosArrayPush(cols, &kv); - kv->key = VALUE; - kv->keyLen = VALUE_LEN; - int32_t ret = smlParseValueFromJSON(metricVal, kv); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - return TSDB_CODE_SUCCESS; + taosMemoryFree(tag->key); + taosArrayDestroy(tag->cols); + taosArrayDestroy(tag->tags); + taosMemoryFree(tag); } -static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableName, SHashObj *dumplicateKey, - SSmlMsgBuf *msg) { - int32_t ret = TSDB_CODE_SUCCESS; - if (!pKVs) { - return TSDB_CODE_OUT_OF_MEMORY; - } - cJSON *tags = cJSON_GetObjectItem(root, "tags"); - if (tags == NULL || tags->type != cJSON_Object) { - return TSDB_CODE_TSC_INVALID_JSON; - } +void smlDestroyInfo(SSmlHandle *info) { + if (!info) return; + qDestroyQuery(info->pQuery); - size_t childTableNameLen = strlen(tsSmlChildTableName); - int32_t tagNum = cJSON_GetArraySize(tags); - for (int32_t i = 0; i < tagNum; ++i) { - cJSON *tag = cJSON_GetArrayItem(tags, i); - if (tag == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; - } - size_t keyLen = strlen(tag->string); - if (IS_INVALID_COL_LEN(keyLen)) { - uError("OTD:Tag key length is 0 or too large than 64"); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - // check duplicate keys - if (smlCheckDuplicateKey(tag->string, keyLen, dumplicateKey)) { - return TSDB_CODE_TSC_DUP_NAMES; + // destroy info->childTables + NodeList *tmp = info->childTables; + while (tmp) { + if (tmp->data.used) { + smlDestroyTableInfo((SSmlTableInfo *)tmp->data.value); } + NodeList *t = tmp->next; + taosMemoryFree(tmp); + tmp = t; + } - // handle child table name - if (childTableNameLen != 0 && strcmp(tag->string, tsSmlChildTableName) == 0) { - if (!cJSON_IsString(tag)) { - uError("OTD:ID must be JSON string"); - return TSDB_CODE_TSC_INVALID_JSON; - } - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - tstrncpy(childTableName, tag->valuestring, TSDB_TABLE_NAME_LEN); - continue; + // destroy info->superTables + tmp = info->superTables; + while (tmp) { + if (tmp->data.used) { + smlDestroySTableMeta((SSmlSTableMeta *)tmp->data.value); } + NodeList *t = tmp->next; + taosMemoryFree(tmp); + tmp = t; + } - // add kv to SSmlKv - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - taosArrayPush(pKVs, &kv); + // destroy info->pVgHash + taosHashCleanup(info->pVgHash); - // key - kv->keyLen = keyLen; - kv->key = tag->string; + taosArrayDestroy(info->preLineTagKV); + taosArrayDestroy(info->preLineColKV); - // value - ret = smlParseValueFromJSON(tag, kv); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + if (!info->dataFormat) { + for (int i = 0; i < info->lineNum; i++) { + taosArrayDestroy(info->lines[i].colArray); } + taosMemoryFree(info->lines); } - return ret; + taosMemoryFreeClear(info); } -static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo *tinfo, SArray *cols) { - int32_t ret = TSDB_CODE_SUCCESS; - - if (!cJSON_IsObject(root)) { - uError("OTD:0x%" PRIx64 " data point needs to be JSON object", info->id); - return TSDB_CODE_TSC_INVALID_JSON; +SSmlHandle *smlBuildSmlInfo(TAOS *taos) { + int32_t code = TSDB_CODE_SUCCESS; + SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle)); + if (NULL == info) { + return NULL; } - - int32_t size = cJSON_GetArraySize(root); - // outmost json fields has to be exactly 4 - if (size != OTD_JSON_FIELDS_NUM) { - uError("OTD:0x%" PRIx64 " Invalid number of JSON fields in data point %d", info->id, size); - return TSDB_CODE_TSC_INVALID_JSON; + if (taos != NULL) { + info->taos = acquireTscObj(*(int64_t *)taos); + code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); + goto cleanup; + } } - // Parse metric - ret = smlParseMetricFromJSON(info, root, tinfo); - if (ret != TSDB_CODE_SUCCESS) { - uError("OTD:0x%" PRIx64 " Unable to parse metric from JSON payload", info->id); - return ret; - } - uDebug("OTD:0x%" PRIx64 " Parse metric from JSON payload finished", info->id); + info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + info->id = smlGenId(); + info->pQuery = smlInitHandle(); + info->dataFormat = true; - // Parse timestamp - ret = smlParseTSFromJSON(info, root, cols); - if (ret) { - uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); - return ret; - } - uDebug("OTD:0x%" PRIx64 " Parse timestamp from JSON payload finished", info->id); + info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); + info->preLineColKV = taosArrayInit(8, sizeof(SSmlKv)); - // Parse metric value - ret = smlParseColsFromJSON(root, cols); - if (ret) { - uError("OTD:0x%" PRIx64 " Unable to parse metric value from JSON payload", info->id); - return ret; + if (NULL == info->pVgHash) { + uError("create SSmlHandle failed"); + goto cleanup; } - uDebug("OTD:0x%" PRIx64 " Parse metric value from JSON payload finished", info->id); - // Parse tags - ret = smlParseTagsFromJSON(root, tinfo->tags, tinfo->childTableName, info->dumplicateKey, &info->msgBuf); - if (ret) { - uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); - return ret; + return info; + +cleanup: + smlDestroyInfo(info); + return NULL; +} + +static int32_t smlPushCols(SArray *colsArray, SArray *cols) { + SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (!kvHash) { + uError("SML:smlDealCols failed to allocate memory"); + return TSDB_CODE_OUT_OF_MEMORY; + } + for (size_t i = 0; i < taosArrayGetSize(cols); i++) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); } - uDebug("OTD:0x%" PRIx64 " Parse tags from JSON payload finished", info->id); + taosArrayPush(colsArray, &kvHash); return TSDB_CODE_SUCCESS; } -/************* TSDB_SML_JSON_PROTOCOL function end **************/ - -static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql, const int len) { - SSmlLineInfo elements = {0}; - uDebug("SML:0x%" PRIx64 " smlParseInfluxLine raw:%d, len:%d, sql:%s", info->id, info->isRawLine, len, - (info->isRawLine ? "rawdata" : sql)); - int ret = smlParseInfluxString(sql, sql + len, &elements, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseInfluxLine failed", info->id); - return ret; - } +static int32_t smlParseLineBottom(SSmlHandle *info) { + if (info->dataFormat) return TSDB_CODE_SUCCESS; - SArray *cols = NULL; - if (info->dataFormat) { // if dataFormat, cols need new memory to save data - cols = taosArrayInit(16, POINTER_BYTES); - if (cols == NULL) { - uError("SML:0x%" PRIx64 " smlParseInfluxLine failed to allocate memory", info->id); - return TSDB_CODE_OUT_OF_MEMORY; - } - } else { // if dataFormat is false, cols do not need to save data, there is another new memory to save data - cols = info->colsContainer; - } - - ret = smlParseTS(info, elements.timestamp, elements.timestampLen, cols); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseTS failed", info->id); - if (info->dataFormat) taosArrayDestroy(cols); - return ret; - } - ret = smlParseCols(elements.cols, elements.colsLen, cols, NULL, false, info->dumplicateKey, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseCols parse cloums fields failed", info->id); - smlDestroyCols(cols); - if (info->dataFormat) taosArrayDestroy(cols); - return ret; - } - - bool hasTable = true; - SSmlTableInfo *tinfo = NULL; - SSmlTableInfo **oneTable = - (SSmlTableInfo **)taosHashGet(info->childTables, elements.measure, elements.measureTagsLen); - if (!oneTable) { - tinfo = smlBuildTableInfo(); - if (!tinfo) { - smlDestroyCols(cols); - if (info->dataFormat) taosArrayDestroy(cols); - return TSDB_CODE_OUT_OF_MEMORY; + for (int32_t i = 0; i < info->lineNum; i++) { + SSmlLineInfo *elements = info->lines + i; + SSmlTableInfo *tinfo = NULL; + if (info->protocol == TSDB_SML_LINE_PROTOCOL) { + tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements->measure, elements->measureTagsLen, NULL); + } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { + tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); + } else { + tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); } - taosHashPut(info->childTables, elements.measure, elements.measureTagsLen, &tinfo, POINTER_BYTES); - oneTable = &tinfo; - hasTable = false; - } - - ret = smlDealCols(*oneTable, info->dataFormat, cols); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - if (!hasTable) { - ret = smlParseCols(elements.tags, elements.tagsLen, (*oneTable)->tags, (*oneTable)->childTableName, true, - info->dumplicateKey, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseCols parse tag fields failed", info->id); - return ret; + if (tinfo == NULL) { + uError("SML:0x%" PRIx64 "get oneTable failed, line num:%d", info->id, i); + smlBuildInvalidDataMsg(&info->msgBuf, "get oneTable failed", elements->measure); + return TSDB_CODE_SML_INVALID_DATA; } - if (taosArrayGetSize((*oneTable)->tags) > TSDB_MAX_TAGS) { + if (taosArrayGetSize(tinfo->tags) > TSDB_MAX_TAGS) { smlBuildInvalidDataMsg(&info->msgBuf, "too many tags than 128", NULL); return TSDB_CODE_PAR_INVALID_TAGS_NUM; } - if (taosArrayGetSize(cols) + taosArrayGetSize((*oneTable)->tags) > TSDB_MAX_COLUMNS) { + if (taosArrayGetSize(elements->colArray) + taosArrayGetSize(tinfo->tags) > TSDB_MAX_COLUMNS) { smlBuildInvalidDataMsg(&info->msgBuf, "too many columns than 4096", NULL); return TSDB_CODE_PAR_TOO_MANY_COLUMNS; } - (*oneTable)->sTableName = elements.measure; - (*oneTable)->sTableNameLen = elements.measureLen; - if (strlen((*oneTable)->childTableName) == 0) { - RandTableName rName = {(*oneTable)->tags, (*oneTable)->sTableName, (uint8_t)(*oneTable)->sTableNameLen, - (*oneTable)->childTableName, 0}; - - buildChildTableName(&rName); - (*oneTable)->uid = rName.uid; - } else { - (*oneTable)->uid = *(uint64_t *)((*oneTable)->childTableName); - } - } - - SSmlSTableMeta **tableMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, elements.measure, elements.measureLen); - if (tableMeta) { // update meta - ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, cols, &info->msgBuf); - if (!hasTable && ret == TSDB_CODE_SUCCESS) { - ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, (*oneTable)->tags, &info->msgBuf); - } + int ret = smlPushCols(tinfo->cols, elements->colArray); if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); return ret; } - } else { - SSmlSTableMeta *meta = smlBuildSTableMeta(); - smlInsertMeta(meta->tagHash, meta->tags, (*oneTable)->tags); - smlInsertMeta(meta->colHash, meta->cols, cols); - taosHashPut(info->superTables, elements.measure, elements.measureLen, &meta, POINTER_BYTES); - } - - if (!info->dataFormat) { - taosArrayClear(info->colsContainer); - } - taosHashClear(info->dumplicateKey); - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseTelnetLine(SSmlHandle *info, void *data, const int len) { - int ret = TSDB_CODE_SUCCESS; - SSmlTableInfo *tinfo = smlBuildTableInfo(); - if (!tinfo) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - SArray *cols = taosArrayInit(16, POINTER_BYTES); - if (cols == NULL) { - uError("SML:0x%" PRIx64 " smlParseTelnetLine failed to allocate memory", info->id); - return TSDB_CODE_OUT_OF_MEMORY; - } - - if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - ret = smlParseTelnetString(info, (const char *)data, (char *)data + len, tinfo, cols); - } else if (info->protocol == TSDB_SML_JSON_PROTOCOL) { - ret = smlParseJSONString(info, (cJSON *)data, tinfo, cols); - } else { - ASSERT(0); - } - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseTelnetLine failed", info->id); - smlDestroyTableInfo(info, tinfo); - smlDestroyCols(cols); - taosArrayDestroy(cols); - return ret; - } - - if (taosArrayGetSize(tinfo->tags) <= 0 || taosArrayGetSize(tinfo->tags) > TSDB_MAX_TAGS) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalidate tags length:[1,128]", NULL); - smlDestroyTableInfo(info, tinfo); - smlDestroyCols(cols); - taosArrayDestroy(cols); - return TSDB_CODE_PAR_INVALID_TAGS_NUM; - } - taosHashClear(info->dumplicateKey); - if (strlen(tinfo->childTableName) == 0) { - RandTableName rName = {tinfo->tags, tinfo->sTableName, (uint8_t)tinfo->sTableNameLen, tinfo->childTableName, 0}; - buildChildTableName(&rName); - tinfo->uid = rName.uid; - } else { - tinfo->uid = *(uint64_t *)(tinfo->childTableName); // generate uid by name simple - } - - bool hasTable = true; - SSmlTableInfo **oneTable = - (SSmlTableInfo **)taosHashGet(info->childTables, tinfo->childTableName, strlen(tinfo->childTableName)); - if (!oneTable) { - taosHashPut(info->childTables, tinfo->childTableName, strlen(tinfo->childTableName), &tinfo, POINTER_BYTES); - oneTable = &tinfo; - hasTable = false; - } else { - smlDestroyTableInfo(info, tinfo); - } + SSmlSTableMeta *tableMeta = + (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + if (tableMeta) { // update meta + ret = smlUpdateMeta(tableMeta->colHash, tableMeta->cols, elements->colArray, false, &info->msgBuf); + if (ret == TSDB_CODE_SUCCESS) { + ret = smlUpdateMeta(tableMeta->tagHash, tableMeta->tags, tinfo->tags, true, &info->msgBuf); + } + if (ret != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); + return ret; + } + } else { + // ret = smlJudgeDupColName(elements->colArray, tinfo->tags, &info->msgBuf); + // if (ret != TSDB_CODE_SUCCESS) { + // uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); + // return ret; + // } - taosArrayPush((*oneTable)->cols, &cols); - SSmlSTableMeta **tableMeta = - (SSmlSTableMeta **)taosHashGet(info->superTables, (*oneTable)->sTableName, (*oneTable)->sTableNameLen); - if (tableMeta) { // update meta - ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, cols, &info->msgBuf); - if (!hasTable && ret == TSDB_CODE_SUCCESS) { - ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, (*oneTable)->tags, &info->msgBuf); - } - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); - return ret; + SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); + smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); + smlInsertMeta(meta->colHash, meta->cols, elements->colArray); + nodeListSet(&info->superTables, elements->measure, elements->measureLen, meta, NULL); } - } else { - SSmlSTableMeta *meta = smlBuildSTableMeta(); - smlInsertMeta(meta->tagHash, meta->tags, (*oneTable)->tags); - smlInsertMeta(meta->colHash, meta->cols, cols); - taosHashPut(info->superTables, (*oneTable)->sTableName, (*oneTable)->sTableNameLen, &meta, POINTER_BYTES); } return TSDB_CODE_SUCCESS; } -static int32_t smlParseJSON(SSmlHandle *info, char *payload) { - int32_t payloadNum = 0; - int32_t ret = TSDB_CODE_SUCCESS; - - if (payload == NULL) { - uError("SML:0x%" PRIx64 " empty JSON Payload", info->id); - return TSDB_CODE_TSC_INVALID_JSON; - } - - info->root = cJSON_Parse(payload); - if (info->root == NULL) { - uError("SML:0x%" PRIx64 " parse json failed:%s", info->id, payload); - return TSDB_CODE_TSC_INVALID_JSON; - } - // multiple data points must be sent in JSON array - if (cJSON_IsObject(info->root)) { - payloadNum = 1; - } else if (cJSON_IsArray(info->root)) { - payloadNum = cJSON_GetArraySize(info->root); - } else { - uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); - ret = TSDB_CODE_TSC_INVALID_JSON; - goto end; - } - - for (int32_t i = 0; i < payloadNum; ++i) { - cJSON *dataPoint = (payloadNum == 1 && cJSON_IsObject(info->root)) ? info->root : cJSON_GetArrayItem(info->root, i); - ret = smlParseTelnetLine(info, dataPoint, -1); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); - goto end; - } - } - -end: - return ret; -} - static int32_t smlInsertData(SSmlHandle *info) { int32_t code = TSDB_CODE_SUCCESS; - SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); - while (oneTable) { - SSmlTableInfo *tableData = *oneTable; + NodeList *tmp = info->childTables; + while (tmp) { + SSmlTableInfo *tableData = (SSmlTableInfo *)tmp->data.value; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); @@ -2330,45 +1200,36 @@ static int32_t smlInsertData(SSmlHandle *info) { } taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg)); - SSmlSTableMeta **pMeta = - (SSmlSTableMeta **)taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); - ASSERT(NULL != pMeta && NULL != *pMeta); + SSmlSTableMeta *pMeta = + (SSmlSTableMeta *)nodeListGet(info->superTables, tableData->sTableName, tableData->sTableNameLen, NULL); + ASSERT(NULL != pMeta); // use tablemeta of stable to save vgid and uid of child table - (*pMeta)->tableMeta->vgId = vg.vgId; - (*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid + pMeta->tableMeta->vgId = vg.vgId; + pMeta->tableMeta->uid = tableData->uid; // one table merge data block together according uid - code = smlBindData(info->exec, tableData->tags, (*pMeta)->cols, tableData->cols, info->dataFormat, - (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, - info->ttl, info->msgBuf.buf, info->msgBuf.len); + code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, pMeta->cols, tableData->cols, pMeta->tableMeta, + tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, info->ttl, + info->msgBuf.buf, info->msgBuf.len); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBindData failed", info->id); return code; } - oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); + tmp = tmp->next; } - code = smlBuildOutput(info->exec, info->pVgHash); + code = smlBuildOutput(info->pQuery, info->pVgHash); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildOutput failed", info->id); return code; } info->cost.insertRpcTime = taosGetTimestampUs(); - // launchQueryImpl(info->pRequest, info->pQuery, false, NULL); - // info->affectedRows = taos_affected_rows(info->pRequest); - // return info->pRequest->code; - SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary; atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); - SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper)); - if (pWrapper == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pWrapper->pRequest = info->pRequest; - launchAsyncQuery(info->pRequest, info->pQuery, NULL, pWrapper); - return TSDB_CODE_SUCCESS; + launchQueryImpl(info->pRequest, info->pQuery, true, NULL); + return info->pRequest->code; } static void smlPrintStatisticInfo(SSmlHandle *info) { @@ -2384,6 +1245,44 @@ static void smlPrintStatisticInfo(SSmlHandle *info) { info->cost.endTime - info->cost.parseTime); } +int32_t smlClearForRerun(SSmlHandle *info) { + info->reRun = false; + // clear info->childTables + NodeList *pList = info->childTables; + while (pList) { + if (pList->data.used) { + smlDestroyTableInfo((SSmlTableInfo *)pList->data.value); + pList->data.used = false; + } + pList = pList->next; + } + + // clear info->superTables + pList = info->superTables; + while (pList) { + if (pList->data.used) { + smlDestroySTableMeta((SSmlSTableMeta *)pList->data.value); + pList->data.used = false; + } + pList = pList->next; + } + + if (unlikely(info->lines != NULL)) { + uError("SML:0x%" PRIx64 " info->lines != NULL", info->id); + return TSDB_CODE_SML_INVALID_DATA; + } + info->lines = (SSmlLineInfo *)taosMemoryCalloc(info->lineNum, sizeof(SSmlLineInfo)); + + memset(&info->preLine, 0, sizeof(SSmlLineInfo)); + info->currSTableMeta = NULL; + info->currTableDataCtx = NULL; + + SVnodeModifyOpStmt *stmt = (SVnodeModifyOpStmt *)(info->pQuery->pRoot); + stmt->freeHashFunc(stmt->pTableBlockHashObj); + stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + return TSDB_CODE_SUCCESS; +} + static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { int32_t code = TSDB_CODE_SUCCESS; if (info->protocol == TSDB_SML_JSON_PROTOCOL) { @@ -2399,7 +1298,9 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char return code; } - for (int32_t i = 0; i < numLines; ++i) { + char *oldRaw = rawLine; + int32_t i = 0; + while (i < numLines) { char *tmp = NULL; int len = 0; if (lines) { @@ -2418,10 +1319,24 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char } } + uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, len:%d, sql:%s", info->id, info->isRawLine, len, + (info->isRawLine ? "rawdata" : tmp)); + if (info->protocol == TSDB_SML_LINE_PROTOCOL) { - code = smlParseInfluxLine(info, tmp, len); + if (info->dataFormat) { + SSmlLineInfo element = {0}; + code = smlParseInfluxString(info, tmp, tmp + len, &element); + } else { + code = smlParseInfluxString(info, tmp, tmp + len, info->lines + i); + } } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - code = smlParseTelnetLine(info, tmp, len); + if (info->dataFormat) { + SSmlLineInfo element = {0}; + code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, &element); + } else { + code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, info->lines + i); + } + } else { ASSERT(0); } @@ -2429,7 +1344,18 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); return code; } + if (info->reRun) { + i = 0; + rawLine = oldRaw; + code = smlClearForRerun(info); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + continue; + } + i++; } + return code; } @@ -2444,17 +1370,22 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL uError("SML:0x%" PRIx64 " smlParseLine error : %s", info->id, tstrerror(code)); return code; } + code = smlParseLineBottom(info); + if (code != 0) { + uError("SML:0x%" PRIx64 " smlParseLineBottom error : %s", info->id, tstrerror(code)); + return code; + } - info->cost.lineNum = numLines; - info->cost.numOfSTables = taosHashGetSize(info->superTables); - info->cost.numOfCTables = taosHashGetSize(info->childTables); + info->cost.lineNum = info->lineNum; + info->cost.numOfSTables = nodeListSize(info->superTables); + info->cost.numOfCTables = nodeListSize(info->childTables); info->cost.schemaTime = taosGetTimestampUs(); do { code = smlModifyDBSchemas(info); if (code == 0) break; - } while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES); + } while (retryNum++ < nodeListSize(info->superTables) * MAX_RETRY_TIMES); if (code != 0) { uError("SML:0x%" PRIx64 " smlModifyDBSchemas error : %s", info->id, tstrerror(code)); @@ -2471,92 +1402,42 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL return code; } -static int32_t isSchemalessDb(STscObj *taos, SRequestObj *request) { - // SCatalog *catalog = NULL; - // int32_t code = catalogGetHandle(((STscObj *)taos)->pAppInfo->clusterId, &catalog); - // if (code != TSDB_CODE_SUCCESS) { - // uError("SML get catalog error %d", code); - // return code; - // } - // - // SName name; - // tNameSetDbName(&name, taos->acctId, taos->db, strlen(taos->db)); - // char dbFname[TSDB_DB_FNAME_LEN] = {0}; - // tNameGetFullDbName(&name, dbFname); - // SDbCfgInfo pInfo = {0}; - // - // SRequestConnInfo conn = {0}; - // conn.pTrans = taos->pAppInfo->pTransporter; - // conn.requestId = request->requestId; - // conn.requestObjRefId = request->self; - // conn.mgmtEps = getEpSet_s(&taos->pAppInfo->mgmtEp); - // - // code = catalogGetDBCfg(catalog, &conn, dbFname, &pInfo); - // if (code != TSDB_CODE_SUCCESS) { - // return code; - // } - // taosArrayDestroy(pInfo.pRetensions); - // - // if (!pInfo.schemaless) { - // return TSDB_CODE_SML_INVALID_DB_CONF; - // } - return TSDB_CODE_SUCCESS; -} - -static void smlInsertCallback(void *param, void *res, int32_t code) { - SRequestObj *pRequest = (SRequestObj *)res; - SSmlHandle *info = (SSmlHandle *)param; - int32_t rows = taos_affected_rows(pRequest); - - uDebug("SML:0x%" PRIx64 " result. code:%d, msg:%s", info->id, pRequest->code, pRequest->msgBuf); - Params *pParam = info->params; - // lock - taosThreadSpinLock(&pParam->lock); - pParam->cnt++; - if (code != TSDB_CODE_SUCCESS) { - pParam->request->code = code; - pParam->request->body.resInfo.numOfRows += rows; - } else { - pParam->request->body.resInfo.numOfRows += info->affectedRows; +TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, char *rawLineEnd, int numLines, + int protocol, int precision, int32_t ttl, int64_t reqid) { + int32_t code = TSDB_CODE_SUCCESS; + if (NULL == taos) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return NULL; } - // unlock - taosThreadSpinUnlock(&pParam->lock); - if (pParam->cnt == pParam->total) { - tsem_post(&pParam->sem); + SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); + if (request == NULL) { + uError("SML:taos_schemaless_insert error request is null"); + return NULL; } - uDebug("SML:0x%" PRIx64 " insert finished, code: %d, rows: %d, total: %d", info->id, code, rows, info->affectedRows); - info->cost.endTime = taosGetTimestampUs(); - info->cost.code = code; - smlPrintStatisticInfo(info); - smlDestroyInfo(info); -} -TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char *rawLine, char *rawLineEnd, - int numLines, int protocol, int precision, int32_t ttl) { - int batchs = 0; - STscObj *pTscObj = request->pTscObj; + SSmlHandle *info = smlBuildSmlInfo(taos); + if (info == NULL) { + request->code = TSDB_CODE_OUT_OF_MEMORY; + uError("SML:taos_schemaless_insert error SSmlHandle is null"); + return (TAOS_RES *)request; + } + info->pRequest = request; + info->isRawLine = rawLine != NULL; + info->ttl = ttl; + info->precision = precision; + info->protocol = (TSDB_SML_PROTOCOL_TYPE)protocol; + info->msgBuf.buf = info->pRequest->msgBuf; + info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; + info->lineNum = numLines; - pTscObj->schemalessType = 1; SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - - Params params = {0}; - params.request = request; - tsem_init(¶ms.sem, 0, 0); - taosThreadSpinInit(&(params.lock), 0); - if (request->pDb == NULL) { request->code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; smlBuildInvalidDataMsg(&msg, "Database not specified", NULL); goto end; } - if (isSchemalessDb(pTscObj, request) != TSDB_CODE_SUCCESS) { - request->code = TSDB_CODE_SML_INVALID_DB_CONF; - smlBuildInvalidDataMsg(&msg, "Cannot write data to a non schemaless database", NULL); - goto end; - } - if (protocol < TSDB_SML_LINE_PROTOCOL || protocol > TSDB_SML_JSON_PROTOCOL) { request->code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; smlBuildInvalidDataMsg(&msg, "protocol invalidate", NULL); @@ -2578,65 +1459,14 @@ TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char goto end; } - batchs = ceil(((double)numLines) / tsSmlBatchSize); - params.total = batchs; - for (int i = 0; i < batchs; ++i) { - SRequestObj *req = (SRequestObj *)createRequest(pTscObj->id, TSDB_SQL_INSERT, 0); - if (!req) { - request->code = TSDB_CODE_OUT_OF_MEMORY; - uError("SML:taos_schemaless_insert error request is null"); - goto end; - } - SSmlHandle *info = smlBuildSmlInfo(pTscObj, req, (SMLProtocolType)protocol, precision); - if (!info) { - request->code = TSDB_CODE_OUT_OF_MEMORY; - uError("SML:taos_schemaless_insert error SSmlHandle is null"); - goto end; - } - - info->isRawLine = (rawLine == NULL); - info->ttl = ttl; - - int32_t perBatch = tsSmlBatchSize; - - if (numLines > perBatch) { - numLines -= perBatch; - } else { - perBatch = numLines; - numLines = 0; - } - - info->params = ¶ms; - info->affectedRows = perBatch; - info->pRequest->body.queryFp = smlInsertCallback; - info->pRequest->body.param = info; - int32_t code = smlProcess(info, lines, rawLine, rawLineEnd, perBatch); - if (lines) { - lines += perBatch; - } - if (rawLine) { - int num = 0; - while (rawLine < rawLineEnd) { - if (*(rawLine++) == '\n') { - num++; - } - if (num == perBatch) { - break; - } - } - } - if (code != TSDB_CODE_SUCCESS) { - info->pRequest->body.queryFp(info, req, code); - } - } - tsem_wait(¶ms.sem); + code = smlProcess(info, lines, rawLine, rawLineEnd, numLines); + request->code = code; + info->cost.endTime = taosGetTimestampUs(); + info->cost.code = code; + // smlPrintStatisticInfo(info); end: - taosThreadSpinDestroy(¶ms.lock); - tsem_destroy(¶ms.sem); - // ((STscObj *)taos)->schemalessType = 0; - pTscObj->schemalessType = 1; - uDebug("resultend:%s", request->msgBuf); + smlDestroyInfo(info); return (TAOS_RES *)request; } @@ -2661,25 +1491,7 @@ end: TAOS_RES *taos_schemaless_insert_ttl_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int32_t ttl, int64_t reqid) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); - if (!request) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } - - if (!lines) { - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&msg, "lines is null", NULL); - return (TAOS_RES *)request; - } - - return taos_schemaless_insert_inner(request, lines, NULL, NULL, numLines, protocol, precision, ttl); + return taos_schemaless_insert_inner(taos, lines, NULL, NULL, numLines, protocol, precision, ttl, reqid); } TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) { @@ -2699,24 +1511,6 @@ TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLi TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int32_t ttl, int64_t reqid) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); - if (!request) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } - - if (!lines || len <= 0) { - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&msg, "lines is null", NULL); - return (TAOS_RES *)request; - } - int numLines = 0; *totalRows = 0; char *tmp = lines; @@ -2729,7 +1523,7 @@ TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int tmp = lines + i + 1; } } - return taos_schemaless_insert_inner(request, NULL, lines, lines + len, numLines, protocol, precision, ttl); + return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, *totalRows, protocol, precision, ttl, reqid); } TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c new file mode 100644 index 0000000000000000000000000000000000000000..3de1397f72533c2ee35df832116e5e81e45686e2 --- /dev/null +++ b/source/client/src/clientSmlJson.c @@ -0,0 +1,501 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include "clientSml.h" + +#define JUMP_JSON_SPACE(start) \ +while(*(start)){\ + if(unlikely(*(start) > 32))\ + break;\ + else\ + (start)++;\ + } + +SArray *smlJsonParseTags(char *start, char *end){ + SArray *tags = taosArrayInit(4, sizeof(SSmlKv)); + while(start < end){ + SSmlKv kv = {0}; + kv.type = TSDB_DATA_TYPE_NCHAR; + bool isInQuote = false; + while(start < end){ + if(unlikely(!isInQuote && *start == '"')){ + start++; + kv.key = start; + isInQuote = true; + continue; + } + if(unlikely(isInQuote && *start == '"')){ + kv.keyLen = start - kv.key; + start++; + break; + } + start++; + } + bool hasColon = false; + while(start < end){ + if(unlikely(!hasColon && *start == ':')){ + start++; + hasColon = true; + continue; + } + if(unlikely(hasColon && kv.value == NULL && (*start > 32 && *start != '"'))){ + kv.value = start; + start++; + continue; + } + + if(unlikely(hasColon && kv.value != NULL && (*start == '"' || *start == ',' || *start == '}'))){ + kv.length = start - kv.value; + taosArrayPush(tags, &kv); + start++; + break; + } + start++; + } + } + return tags; +} + +static int32_t smlParseTagsFromJSON(SSmlHandle *info, SSmlLineInfo *elements) { + int32_t ret = TSDB_CODE_SUCCESS; + + if(is_same_child_table_telnet(elements, &info->preLine) == 0){ + return TSDB_CODE_SUCCESS; + } + + bool isSameMeasure = IS_SAME_SUPER_TABLE; + + int cnt = 0; + SArray *preLineKV = info->preLineTagKV; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if(info->dataFormat){ + if(unlikely(!isSameMeasure)){ + SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + + if(unlikely(sMeta == NULL)){ + sMeta = smlBuildSTableMeta(info->dataFormat); + STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); + sMeta->tableMeta = pTableMeta; + if(pTableMeta == NULL){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL); + } + info->currSTableMeta = sMeta->tableMeta; + superKV = sMeta->tags; + + if(unlikely(taosArrayGetSize(superKV) == 0)){ + isSuperKVInit = false; + } + taosArraySetSize(preLineKV, 0); + } + }else{ + taosArraySetSize(preLineKV, 0); + } + + SArray *tags = smlJsonParseTags(elements->tags, elements->tags + elements->tagsLen); + int32_t tagNum = taosArrayGetSize(tags); + for (int32_t i = 0; i < tagNum; ++i) { + SSmlKv kv = *(SSmlKv*)taosArrayGet(tags, i); + + if(info->dataFormat){ + if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ + info->dataFormat = false; + info->reRun = true; + taosArrayDestroy(tags); + return TSDB_CODE_SUCCESS; + } + + if(isSameMeasure){ + if(unlikely(cnt >= taosArrayGetSize(preLineKV))) { + info->dataFormat = false; + info->reRun = true; + taosArrayDestroy(tags); + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); + if(unlikely(kv.length > preKV->length)){ + preKV->length = kv.length; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + ASSERT(tableMeta != NULL); + + SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + taosArrayDestroy(tags); + return TSDB_CODE_SUCCESS; + } + }else{ + if(isSuperKVInit){ + if(unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + taosArrayDestroy(tags); + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = (SSmlKv *)taosArrayGet(superKV, cnt); + if(unlikely(kv.length > preKV->length)) { + preKV->length = kv.length; + }else{ + kv.length = preKV->length; + } + info->needModifySchema = true; + + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + taosArrayDestroy(tags); + return TSDB_CODE_SUCCESS; + } + }else{ + taosArrayPush(superKV, &kv); + } + taosArrayPush(preLineKV, &kv); + } + }else{ + taosArrayPush(preLineKV, &kv); + } + cnt++; + } + taosArrayDestroy(tags); + + SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); + if (unlikely(tinfo == NULL)) { + tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); + if (unlikely(!tinfo)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tinfo->tags = taosArrayDup(preLineKV, NULL); + + smlSetCTableName(tinfo); + if (info->dataFormat) { + info->currSTableMeta->uid = tinfo->uid; + tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); + if (tinfo->tableDataCtx == NULL) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } + } + + SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); + *key = *elements; + tinfo->key = key; + nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet); + } + if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; + + return ret; +} + +static char* smlJsonGetObj(char *payload){ + int leftBracketCnt = 0; + while(*payload) { + if (unlikely(*payload == '{')) { + leftBracketCnt++; + payload++; + continue; + } + if (unlikely(*payload == '}')) { + leftBracketCnt--; + payload++; + if (leftBracketCnt == 0) { + return payload; + } else if (leftBracketCnt < 0) { + return NULL; + } + continue; + } + payload++; + } + return NULL; +} + +void smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset){ + int index = 0; + while(*(*start)){ + if((*start)[0] != '"'){ + (*start)++; + continue; + } + + if(unlikely(index >= 4)) { + uError("index >= 4, %s", *start) + break; + } + char *sTmp = *start; + if((*start)[1] == 'm' && (*start)[2] == 'e' && (*start)[3] == 't' + && (*start)[4] == 'r' && (*start)[5] == 'i' && (*start)[6] == 'c' && (*start)[7] == '"'){ + + (*start) += 8; + bool isInQuote = false; + while(*(*start)){ + if(unlikely(!isInQuote && *(*start) == '"')){ + (*start)++; + offset[index++] = *start - sTmp; + element->measure = (*start); + isInQuote = true; + continue; + } + if(unlikely(isInQuote && *(*start) == '"')){ + element->measureLen = (*start) - element->measure; + break; + } + (*start)++; + } + }else if((*start)[1] == 't' && (*start)[2] == 'i' && (*start)[3] == 'm' + && (*start)[4] == 'e' && (*start)[5] == 's' && (*start)[6] == 't' + && (*start)[7] == 'a' && (*start)[8] == 'm' && (*start)[9] == 'p' && (*start)[10] == '"'){ + + (*start) += 11; + bool hasColon = false; + while(*(*start)){ + if(unlikely(!hasColon && *(*start) == ':')){ + (*start)++; + JUMP_JSON_SPACE((*start)) + offset[index++] = *start - sTmp; + element->timestamp = (*start); + hasColon = true; + continue; + } + if(unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32))){ + element->timestampLen = (*start) - element->timestamp; + break; + } + (*start)++; + } + }else if((*start)[1] == 'v' && (*start)[2] == 'a' && (*start)[3] == 'l' + && (*start)[4] == 'u' && (*start)[5] == 'e' && (*start)[6] == '"'){ + + (*start) += 7; + + bool hasColon = false; + while(*(*start)){ + if(unlikely(!hasColon && *(*start) == ':')){ + (*start)++; + JUMP_JSON_SPACE((*start)) + offset[index++] = *start - sTmp; + element->cols = (*start); + hasColon = true; + continue; + } + if(unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32))){ + element->colsLen = (*start) - element->cols; + break; + } + (*start)++; + } + }else if((*start)[1] == 't' && (*start)[2] == 'a' && (*start)[3] == 'g' + && (*start)[4] == 's' && (*start)[5] == '"'){ + (*start) += 6; + + while(*(*start)){ + if(unlikely(*(*start) == ':')){ + (*start)++; + JUMP_JSON_SPACE((*start)) + offset[index++] = *start - sTmp; + element->tags = (*start); + char* tmp = smlJsonGetObj((*start)); + if(tmp){ + element->tagsLen = tmp - (*start); + *start = tmp; + } + break; + } + (*start)++; + } + } + if(*(*start) == '}'){ + (*start)++; + break; + } + (*start)++; + } +} + +void smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset){ + int index = 0; + while(*(*start)){ + if((*start)[0] != '"'){ + (*start)++; + continue; + } + + if(unlikely(index >= 4)) { + uError("index >= 4, %s", *start) + break; + } + if((*start)[1] == 'm'){ + (*start) += offset[index++]; + element->measure = *start; + while(*(*start)){ + if(unlikely(*(*start) == '"')){ + element->measureLen = (*start) - element->measure; + break; + } + (*start)++; + } + }else if((*start)[1] == 't' && (*start)[2] == 'i'){ + (*start) += offset[index++]; + element->timestamp = *start; + while(*(*start)){ + if(unlikely(*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)){ + element->timestampLen = (*start) - element->timestamp; + break; + } + (*start)++; + } + }else if((*start)[1] == 'v'){ + (*start) += offset[index++]; + element->cols = *start; + while(*(*start)){ + if(unlikely( *(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)){ + element->colsLen = (*start) - element->cols; + break; + } + (*start)++; + } + }else if((*start)[1] == 't' && (*start)[2] == 'a'){ + (*start) += offset[index++]; + element->tags = (*start); + char* tmp = smlJsonGetObj((*start)); + if(tmp){ + element->tagsLen = tmp - (*start); + *start = tmp; + } + break; + } + if(*(*start) == '}'){ + (*start)++; + break; + } + (*start)++; + } +} + +static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *elements) { + int32_t ret = TSDB_CODE_SUCCESS; + + if(info->offset[0] == 0){ + smlJsonParseObjFirst(start, elements, info->offset); + }else{ + smlJsonParseObj(start, elements, info->offset); + } + if(**start == '\0') return TSDB_CODE_SUCCESS; + + SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; + if (smlParseNumber(&kv, &info->msgBuf)) { + kv.length = (int16_t)tDataTypes[kv.type].bytes; + }else{ + return TSDB_CODE_TSC_INVALID_VALUE; + } + + // Parse tags + ret = smlParseTagsFromJSON(info, elements); + if (unlikely(ret)) { + uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); + return ret; + } + + if(unlikely(info->reRun)){ + return TSDB_CODE_SUCCESS; + } + + // Parse timestamp + // notice!!! put ts back to tag to ensure get meta->precision + int64_t ts = smlParseOpenTsdbTime(info, elements->timestamp, elements->timestampLen); + if (unlikely(ts < 0)) { + uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); + return TSDB_CODE_INVALID_TIMESTAMP; + } + SSmlKv kvTs = { .key = TS, .keyLen = TS_LEN, .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + + if(info->dataFormat){ + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); + } + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildRow(info->currTableDataCtx); + } + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + return ret; + } + }else{ + if(elements->colArray == NULL){ + elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + } + taosArrayPush(elements->colArray, &kvTs); + taosArrayPush(elements->colArray, &kv); + } + info->preLine = *elements; + + return TSDB_CODE_SUCCESS; +} + +int32_t smlParseJSON(SSmlHandle *info, char *payload) { + int32_t payloadNum = 1 << 15; + int32_t ret = TSDB_CODE_SUCCESS; + + int cnt = 0; + char *dataPointStart = payload; + while (1) { + if(info->dataFormat) { + SSmlLineInfo element = {0}; + ret = smlParseJSONString(info, &dataPointStart, &element); + }else{ + if(cnt >= payloadNum){ + payloadNum = payloadNum << 1; + void* tmp = taosMemoryRealloc(info->lines, payloadNum * sizeof(SSmlLineInfo)); + if(tmp != NULL){ + info->lines = (SSmlLineInfo*)tmp; + } + } + ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt); + } + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); + return ret; + } + + if(*dataPointStart == '\0') break; + + if(unlikely(info->reRun)){ + cnt = 0; + dataPointStart = payload; + info->lineNum = payloadNum; + ret = smlClearForRerun(info); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } + continue; + } + cnt++; + } + info->lineNum = cnt; + + return TSDB_CODE_SUCCESS; +} diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c new file mode 100644 index 0000000000000000000000000000000000000000..f56ad34fc52bc1687ce42aa38256b01eb4724f68 --- /dev/null +++ b/source/client/src/clientSmlLine.c @@ -0,0 +1,700 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include + +#include "clientSml.h" + +// comma , +//#define IS_SLASH_COMMA(sql) (*(sql) == COMMA && *((sql)-1) == SLASH) +#define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH) +// space +//#define IS_SLASH_SPACE(sql) (*(sql) == SPACE && *((sql)-1) == SLASH) +#define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH) +// equal = +//#define IS_SLASH_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) == SLASH) +#define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH) +// quote " +//#define IS_SLASH_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) == SLASH) +#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH) +// SLASH +//#define IS_SLASH_SLASH(sql) (*(sql) == SLASH && *((sql)-1) == SLASH) + +#define IS_SLASH_LETTER(sql) \ + (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL || *(sql) == QUOTE || *(sql) == SLASH)) \ +// (IS_SLASH_COMMA(sql) || IS_SLASH_SPACE(sql) || IS_SLASH_EQUAL(sql) || IS_SLASH_QUOTE(sql) || IS_SLASH_SLASH(sql)) + +#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) + +#define PROCESS_SLASH(key, keyLen) \ + for (int i = 1; i < keyLen; ++i) { \ + if (IS_SLASH_LETTER(key + i)) { \ + MOVE_FORWARD_ONE(key + i, keyLen - i); \ + i--; \ + keyLen--; \ + } \ + } + +#define BINARY_ADD_LEN 2 // "binary" 2 means " " +#define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" " + +uint8_t smlPrecisionConvert[7] = {TSDB_TIME_PRECISION_NANO, TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES, + TSDB_TIME_PRECISION_SECONDS, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO, + TSDB_TIME_PRECISION_NANO}; + +static bool smlParseBool(SSmlKv *kvVal) { + const char *pVal = kvVal->value; + int32_t len = kvVal->length; + if ((len == 1) && (pVal[0] == 't' || pVal[0] == 'T')) { + kvVal->i = TSDB_TRUE; + return true; + } + + if ((len == 1) && (pVal[0] == 'f' || pVal[0] == 'F')) { + kvVal->i = TSDB_FALSE; + return true; + } + + if ((len == 4) && !strncasecmp(pVal, "true", len)) { + kvVal->i = TSDB_TRUE; + return true; + } + if ((len == 5) && !strncasecmp(pVal, "false", len)) { + kvVal->i = TSDB_FALSE; + return true; + } + return false; +} + +static bool smlIsBinary(const char *pVal, uint16_t len) { + // binary: "abc" + if (len < 2) { + return false; + } + if (pVal[0] == '"' && pVal[len - 1] == '"') { + return true; + } + return false; +} + +static bool smlIsNchar(const char *pVal, uint16_t len) { + // nchar: L"abc" + if (len < 3) { + return false; + } + if (pVal[1] == '"' && pVal[len - 1] == '"' && (pVal[0] == 'l' || pVal[0] == 'L')) { + return true; + } + return false; +} + +static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t len) { + uint8_t toPrecision = info->currSTableMeta ? info->currSTableMeta->tableInfo.precision : TSDB_TIME_PRECISION_NANO; + + if(unlikely(len == 0 || (len == 1 && data[0] == '0'))){ + return taosGetTimestampNs()/smlFactorNS[toPrecision]; + } + + uint8_t fromPrecision = smlPrecisionConvert[info->precision]; + + int64_t ts = smlGetTimeValue(data, len, fromPrecision, toPrecision); + if (unlikely(ts == -1)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); + return -1; + } + return ts; +} + +int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { + if (pVal->value[0] == '"'){ // binary + if (pVal->length >= 2 && pVal->value[pVal->length - 1] == '"') { + pVal->type = TSDB_DATA_TYPE_BINARY; + pVal->length -= BINARY_ADD_LEN; + if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + pVal->value += (BINARY_ADD_LEN - 1); + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if(pVal->value[0] == 'l' || pVal->value[0] == 'L'){ // nchar + if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= 3){ + pVal->type = TSDB_DATA_TYPE_NCHAR; + pVal->length -= NCHAR_ADD_LEN; + if (pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + pVal->value += (NCHAR_ADD_LEN - 1); + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pVal->value[0] == 't' || pVal->value[0] == 'T'){ + if(pVal->length == 1 || (pVal->length == 4 && (pVal->value[1] == 'r' || pVal->value[1] == 'R') + && (pVal->value[2] == 'u' || pVal->value[2] == 'U') + && (pVal->value[3] == 'e' || pVal->value[3] == 'E'))){ + pVal->i = TSDB_TRUE; + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pVal->value[0] == 'f' || pVal->value[0] == 'F'){ + if(pVal->length == 1 || (pVal->length == 5 && (pVal->value[1] == 'a' || pVal->value[1] == 'A') + && (pVal->value[2] == 'l' || pVal->value[2] == 'L') + && (pVal->value[3] == 's' || pVal->value[3] == 'S') + && (pVal->value[4] == 'e' || pVal->value[4] == 'E'))){ + pVal->i = TSDB_FALSE; + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + + // number + if (smlParseNumber(pVal, msg)) { + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return TSDB_CODE_SUCCESS; + } + + return TSDB_CODE_TSC_INVALID_VALUE; +} + +static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, + SSmlLineInfo* currElement, bool isSameMeasure, bool isSameCTable){ + if(isSameCTable){ + return TSDB_CODE_SUCCESS; + } + + int cnt = 0; + SArray *preLineKV = info->preLineTagKV; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if(info->dataFormat){ + if(unlikely(!isSameMeasure)){ + SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); + + if(unlikely(sMeta == NULL)){ + sMeta = smlBuildSTableMeta(info->dataFormat); + STableMeta * pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); + sMeta->tableMeta = pTableMeta; + if(pTableMeta == NULL){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + nodeListSet(&info->superTables, currElement->measure, currElement->measureLen, sMeta, NULL); + } + info->currSTableMeta = sMeta->tableMeta; + superKV = sMeta->tags; + + if(unlikely(taosArrayGetSize(superKV) == 0)){ + isSuperKVInit = false; + } + taosArraySetSize(preLineKV, 0); + } + }else{ + taosArraySetSize(preLineKV, 0); + } + + + while (*sql < sqlEnd) { + if (unlikely(IS_SPACE(*sql))) { + break; + } + + bool hasSlash = false; + // parse key + const char *key = *sql; + size_t keyLen = 0; + while (*sql < sqlEnd) { + if (unlikely(IS_COMMA(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(IS_EQUAL(*sql))) { + keyLen = *sql - key; + (*sql)++; + break; + } + if(!hasSlash){ + hasSlash = (*(*sql) == SLASH); + } + (*sql)++; + } + if(unlikely(hasSlash)) { + PROCESS_SLASH(key, keyLen) + } + + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } + + // parse value + const char *value = *sql; + size_t valueLen = 0; + hasSlash = false; + while (*sql < sqlEnd) { + // parse value + if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { + break; + }else if (unlikely(IS_EQUAL(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + + if(!hasSlash){ + hasSlash = (*(*sql) == SLASH); + } + + (*sql)++; + } + valueLen = *sql - value; + + if (unlikely(valueLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); + return TSDB_CODE_SML_INVALID_DATA; + } + + if(unlikely(hasSlash)) { + PROCESS_SLASH(value, valueLen) + } + + if (unlikely(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + + SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen}; + if(info->dataFormat){ + if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(isSameMeasure){ + if(unlikely(cnt >= taosArrayGetSize(preLineKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); + if(unlikely(kv.length > preKV->length)){ + preKV->length = kv.length; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); + ASSERT(tableMeta != NULL); + + SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + if(isSuperKVInit){ + if(unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = (SSmlKv *)taosArrayGet(superKV, cnt); + if(unlikely(kv.length > preKV->length)) { + preKV->length = kv.length; + }else{ + kv.length = preKV->length; + } + info->needModifySchema = true; + + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + taosArrayPush(superKV, &kv); + } + taosArrayPush(preLineKV, &kv); + } + }else{ + taosArrayPush(preLineKV, &kv); + } + + cnt++; + if(IS_SPACE(*sql)){ + break; + } + (*sql)++; + } + + void* oneTable = nodeListGet(info->childTables, currElement->measure, currElement->measureTagsLen, NULL); + if ((oneTable != NULL)) { + return TSDB_CODE_SUCCESS; + } + + SSmlTableInfo *tinfo = smlBuildTableInfo(1, currElement->measure, currElement->measureLen); + if (unlikely(!tinfo)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tinfo->tags = taosArrayDup(preLineKV, NULL); + + smlSetCTableName(tinfo); + if(info->dataFormat) { + info->currSTableMeta->uid = tinfo->uid; + tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); + if(tinfo->tableDataCtx == NULL){ + smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } + } + + nodeListSet(&info->childTables, currElement->measure, currElement->measureTagsLen, tinfo, NULL); + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, + SSmlLineInfo* currElement, bool isSameMeasure, bool isSameCTable){ + int cnt = 0; + SArray *preLineKV = info->preLineColKV; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if(info->dataFormat){ + if(unlikely(!isSameCTable)){ + SSmlTableInfo *oneTable = (SSmlTableInfo *)nodeListGet(info->childTables, currElement->measure, currElement->measureTagsLen, NULL); + if (unlikely(oneTable == NULL)) { + smlBuildInvalidDataMsg(&info->msgBuf, "child table should inside", currElement->measure); + return TSDB_CODE_SML_INVALID_DATA; + } + info->currTableDataCtx = oneTable->tableDataCtx; + } + + if(unlikely(!isSameMeasure)){ + SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); + + if(unlikely(sMeta == NULL)){ + sMeta = smlBuildSTableMeta(info->dataFormat); + STableMeta * pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); + sMeta->tableMeta = pTableMeta; + if(pTableMeta == NULL){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + nodeListSet(&info->superTables, currElement->measure, currElement->measureLen, sMeta, NULL); + } + info->currSTableMeta = sMeta->tableMeta; + superKV = sMeta->cols; + if(unlikely(taosArrayGetSize(superKV) == 0)){ + isSuperKVInit = false; + } + taosArraySetSize(preLineKV, 0); + } + } + + while (*sql < sqlEnd) { + if (unlikely(IS_SPACE(*sql))) { + break; + } + + bool hasSlash = false; + // parse key + const char *key = *sql; + size_t keyLen = 0; + while (*sql < sqlEnd) { + if (unlikely(IS_COMMA(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(IS_EQUAL(*sql))) { + keyLen = *sql - key; + (*sql)++; + break; + } + if(!hasSlash){ + hasSlash = (*(*sql) == SLASH); + } + (*sql)++; + } + if(unlikely(hasSlash)) { + PROCESS_SLASH(key, keyLen) + } + + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } + + // parse value + const char *value = *sql; + size_t valueLen = 0; + hasSlash = false; + bool isInQuote = false; + while (*sql < sqlEnd) { + // parse value + if (unlikely(IS_QUOTE(*sql))) { + isInQuote = !isInQuote; + (*sql)++; + continue; + } + if (!isInQuote){ + if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { + break; + } else if (unlikely(IS_EQUAL(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + } + if(!hasSlash){ + hasSlash = (*(*sql) == SLASH); + } + + (*sql)++; + } + valueLen = *sql - value; + + if (unlikely(isInQuote)) { + smlBuildInvalidDataMsg(&info->msgBuf, "only one quote", value); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(valueLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); + return TSDB_CODE_SML_INVALID_DATA; + } + if(unlikely(hasSlash)) { + PROCESS_SLASH(value, valueLen) + } + + SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen}; + int32_t ret = smlParseValue(&kv, &info->msgBuf); + if (ret != TSDB_CODE_SUCCESS) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlParseValue error", value); + return ret; + } + + if(info->dataFormat){ + //cnt begin 0, add ts so + 2 + if(unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + // bind data + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, cnt + 1); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uError("smlBuildCol error, retry"); + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(isSameMeasure){ + if(cnt >= taosArrayGetSize(preLineKV)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); + if(kv.type != preKV->type){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > preKV->length)){ + preKV->length = kv.length; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); + ASSERT(tableMeta != NULL); + + SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->cols, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + if(isSuperKVInit){ + if(unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = (SSmlKv *)taosArrayGet(superKV, cnt); + if(unlikely(kv.type != preKV->type)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(IS_VAR_DATA_TYPE(kv.type)){ + if(kv.length > preKV->length) { + preKV->length = kv.length; + }else{ + kv.length = preKV->length; + } + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + taosArrayPush(superKV, &kv); + } + taosArrayPush(preLineKV, &kv); + } + }else{ + if(currElement->colArray == NULL){ + currElement->colArray = taosArrayInit(16, sizeof(SSmlKv)); + taosArraySetSize(currElement->colArray, 1); + } + taosArrayPush(currElement->colArray, &kv); //reserve for timestamp + } + + cnt++; + if(IS_SPACE(*sql)){ + break; + } + (*sql)++; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements) { + if (!sql) return TSDB_CODE_SML_INVALID_DATA; + JUMP_SPACE(sql, sqlEnd) + if (unlikely(*sql == COMMA)) return TSDB_CODE_SML_INVALID_DATA; + elements->measure = sql; + + // parse measure + while (sql < sqlEnd) { + if (unlikely((sql != elements->measure) && IS_SLASH_LETTER(sql))) { + MOVE_FORWARD_ONE(sql, sqlEnd - sql); + sqlEnd--; + continue; + } + if (unlikely(IS_COMMA(sql))) { + break; + } + + if (unlikely(IS_SPACE(sql))) { + break; + } + sql++; + } + elements->measureLen = sql - elements->measure; + if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL); + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + + // to get measureTagsLen before + const char* tmp = sql; + while (tmp < sqlEnd){ + if (unlikely(IS_SPACE(tmp))) { + break; + } + tmp++; + } + elements->measureTagsLen = tmp - elements->measure; + + bool isSameCTable = false; + bool isSameMeasure = false; + if(IS_SAME_CHILD_TABLE){ + isSameCTable = true; + isSameMeasure = true; + }else if(info->dataFormat) { + isSameMeasure = IS_SAME_SUPER_TABLE; + } + // parse tag + if (*sql == COMMA) sql++; + elements->tags = sql; + + int ret = smlParseTagKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable); + if(unlikely(ret != TSDB_CODE_SUCCESS)){ + return ret; + } + if(unlikely(info->reRun)){ + return TSDB_CODE_SUCCESS; + } + + sql = elements->measure + elements->measureTagsLen; + elements->tagsLen = sql - elements->tags; + + // parse cols + JUMP_SPACE(sql, sqlEnd) + elements->cols = sql; + + ret = smlParseColKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable); + if(unlikely(ret != TSDB_CODE_SUCCESS)){ + return ret; + } + + if(unlikely(info->reRun)){ + return TSDB_CODE_SUCCESS; + } + + elements->colsLen = sql - elements->cols; + if (unlikely(elements->colsLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "cols is empty", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } + + // parse timestamp + JUMP_SPACE(sql, sqlEnd) + elements->timestamp = sql; + while (sql < sqlEnd) { + if (unlikely(isspace(*sql))) { + break; + } + sql++; + } + elements->timestampLen = sql - elements->timestamp; + + int64_t ts = smlParseInfluxTime(info, elements->timestamp, elements->timestampLen); + if (unlikely(ts <= 0)) { + uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); + return TSDB_CODE_INVALID_TIMESTAMP; + } + // add ts to + SSmlKv kv = { .key = TS, .keyLen = TS_LEN, .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + if(info->dataFormat){ + smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0); + smlBuildRow(info->currTableDataCtx); + }else{ + taosArraySet(elements->colArray, 0, &kv); + } + info->preLine = *elements; + + return ret; +} + diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c new file mode 100644 index 0000000000000000000000000000000000000000..53a7e3e81e99b8d3a5107964fe7a15c41a2a2420 --- /dev/null +++ b/source/client/src/clientSmlTelnet.c @@ -0,0 +1,333 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include + +#include "clientSml.h" + +int32_t is_same_child_table_telnet(const void *a, const void *b){ + SSmlLineInfo *t1 = (SSmlLineInfo *)a; + SSmlLineInfo *t2 = (SSmlLineInfo *)b; + return (((t1->measureLen == t2->measureLen) && memcmp(t1->measure, t2->measure, t1->measureLen) == 0) + && ((t1->tagsLen == t2->tagsLen) && memcmp(t1->tags, t2->tags, t1->tagsLen) == 0)) ? 0 : 1; +} + +int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len) { + uint8_t toPrecision = info->currSTableMeta ? info->currSTableMeta->tableInfo.precision : TSDB_TIME_PRECISION_NANO; + + if (unlikely(!data)) { + smlBuildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL); + return -1; + } + if (unlikely(len == 1 && data[0] == '0')) { + return taosGetTimestampNs()/smlFactorNS[toPrecision]; + } + int8_t fromPrecision = smlGetTsTypeByLen(len); + if (unlikely(fromPrecision == -1)) { + smlBuildInvalidDataMsg(&info->msgBuf, + "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data); + return -1; + } + int64_t ts = smlGetTimeValue(data, len, fromPrecision, toPrecision); + if (unlikely(ts == -1)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); + return -1; + } + return ts; +} + + +static void smlParseTelnetElement(char **sql, char *sqlEnd, char **data, int32_t *len) { + while (*sql < sqlEnd) { + if (unlikely((**sql != SPACE && !(*data)))) { + *data = *sql; + } else if (unlikely(**sql == SPACE && *data)) { + *len = *sql - *data; + break; + } + (*sql)++; + } +} + +static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) { + if(is_same_child_table_telnet(elements, &info->preLine) == 0){ + return TSDB_CODE_SUCCESS; + } + + bool isSameMeasure = IS_SAME_SUPER_TABLE; + + int cnt = 0; + SArray *preLineKV = info->preLineTagKV; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if(info->dataFormat){ + if(!isSameMeasure){ + SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + + if(unlikely(sMeta == NULL)){ + sMeta = smlBuildSTableMeta(info->dataFormat); + STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); + sMeta->tableMeta = pTableMeta; + if(pTableMeta == NULL){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL); + } + info->currSTableMeta = sMeta->tableMeta; + superKV = sMeta->tags; + + if(unlikely(taosArrayGetSize(superKV) == 0)){ + isSuperKVInit = false; + } + taosArraySetSize(preLineKV, 0); + } + }else{ + taosArraySetSize(preLineKV, 0); + } + + const char *sql = data; + while (sql < sqlEnd) { + JUMP_SPACE(sql, sqlEnd) + if (unlikely(*sql == '\0')) break; + + const char *key = sql; + size_t keyLen = 0; + + // parse key + while (sql < sqlEnd) { + if (unlikely(*sql == SPACE)) { + smlBuildInvalidDataMsg(msg, "invalid data", sql); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(*sql == EQUAL)) { + keyLen = sql - key; + sql++; + break; + } + sql++; + } + + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } +// if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { +// smlBuildInvalidDataMsg(msg, "dumplicate key", key); +// return TSDB_CODE_TSC_DUP_NAMES; +// } + + // parse value + const char *value = sql; + size_t valueLen = 0; + while (sql < sqlEnd) { + // parse value + if (unlikely(*sql == SPACE)) { + break; + } + if (unlikely(*sql == EQUAL)) { + smlBuildInvalidDataMsg(msg, "invalid data", sql); + return TSDB_CODE_SML_INVALID_DATA; + } + sql++; + } + valueLen = sql - value; + + if (unlikely(valueLen == 0)) { + smlBuildInvalidDataMsg(msg, "invalid value", value); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (unlikely(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + + SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen}; + + if(info->dataFormat){ + if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(isSameMeasure){ + if(unlikely(cnt >= taosArrayGetSize(preLineKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); + if(unlikely(kv.length > preKV->length)){ + preKV->length = kv.length; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + ASSERT(tableMeta != NULL); + + SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + if(isSuperKVInit){ + if(unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = (SSmlKv *)taosArrayGet(superKV, cnt); + if(unlikely(kv.length > preKV->length)) { + preKV->length = kv.length; + }else{ + kv.length = preKV->length; + } + info->needModifySchema = true; + + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + taosArrayPush(superKV, &kv); + } + taosArrayPush(preLineKV, &kv); + } + }else{ + taosArrayPush(preLineKV, &kv); + } + cnt++; + } + SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); + if (unlikely(tinfo == NULL)) { + tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); + if (!tinfo) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tinfo->tags = taosArrayDup(preLineKV, NULL); + + smlSetCTableName(tinfo); + if (info->dataFormat) { + info->currSTableMeta->uid = tinfo->uid; + tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); + if (tinfo->tableDataCtx == NULL) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } + } + + SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); + *key = *elements; + tinfo->key = key; + nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet); + } + if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; + + return TSDB_CODE_SUCCESS; +} + +// format: =[ =] +int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements) { + if (!sql) return TSDB_CODE_SML_INVALID_DATA; + + // parse metric + smlParseTelnetElement(&sql, sqlEnd, &elements->measure, &elements->measureLen); + if (unlikely((!(elements->measure) || IS_INVALID_TABLE_LEN(elements->measureLen)))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + + // parse timestamp + smlParseTelnetElement(&sql, sqlEnd, &elements->timestamp, &elements->timestampLen); + if (unlikely(!elements->timestamp || elements->timestampLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); + return TSDB_CODE_SML_INVALID_DATA; + } + + bool needConverTime = false; // get TS before parse tag(get meta), so need conver time + if(info->dataFormat && info->currSTableMeta == NULL){ + needConverTime = true; + } + int64_t ts = smlParseOpenTsdbTime(info, elements->timestamp, elements->timestampLen); + if (unlikely(ts < 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); + return TSDB_CODE_INVALID_TIMESTAMP; + } + SSmlKv kvTs = { .key = TS, .keyLen = TS_LEN, .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + + // parse value + smlParseTelnetElement(&sql, sqlEnd, &elements->cols, &elements->colsLen); + if (unlikely(!elements->cols || elements->colsLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; + if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_VALUE; + } + + JUMP_SPACE(sql, sqlEnd) + + elements->tags = sql; + elements->tagsLen = sqlEnd - sql; + if (unlikely(!elements->tags || elements->tagsLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + int ret = smlParseTelnetTags(info, sql, sqlEnd, elements, &info->msgBuf); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + return ret; + } + + if(unlikely(info->reRun)){ + return TSDB_CODE_SUCCESS; + } + + if(info->dataFormat){ + if(needConverTime) { + kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision); + } + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); + } + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildRow(info->currTableDataCtx); + } + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + return ret; + } + }else{ + if(elements->colArray == NULL){ + elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + } + taosArrayPush(elements->colArray, &kvTs); + taosArrayPush(elements->colArray, &kv); + } + info->preLine = *elements; + + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 82ea9e0d8f8f6e70829567ce34e9b9d1ae444da4..7bc99c65e8662685a32a24779389619f0c2ae422 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -152,9 +152,10 @@ int32_t stmtRestoreQueryFields(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } -int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, SName* tbName, const char* sTableName, bool autoCreateTbl) { +int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, SName* tbName, const char* sTableName, + bool autoCreateTbl) { STscStmt* pStmt = (STscStmt*)stmt; - char tbFName[TSDB_TABLE_FNAME_LEN]; + char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(tbName, tbFName); memcpy(&pStmt->bInfo.sname, tbName, sizeof(*tbName)); @@ -171,12 +172,11 @@ int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, return TSDB_CODE_SUCCESS; } -int32_t stmtUpdateExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockHash, bool autoCreateTbl) { +int32_t stmtUpdateExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockHash) { STscStmt* pStmt = (STscStmt*)stmt; pStmt->sql.pVgHash = pVgHash; pStmt->exec.pBlockHash = pBlockHash; - pStmt->exec.autoCreateTbl = autoCreateTbl; return TSDB_CODE_SUCCESS; } @@ -186,7 +186,7 @@ int32_t stmtUpdateInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, SNam STscStmt* pStmt = (STscStmt*)stmt; STMT_ERR_RET(stmtUpdateBindInfo(stmt, pTableMeta, tags, tbName, sTableName, autoCreateTbl)); - STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash, autoCreateTbl)); + STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash)); pStmt->sql.autoCreateTbl = autoCreateTbl; @@ -214,16 +214,16 @@ int32_t stmtCacheBlock(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } - STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + STableDataCxt** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (!pSrc) { return TSDB_CODE_OUT_OF_MEMORY; } - STableDataBlocks* pDst = NULL; + STableDataCxt* pDst = NULL; - STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc)); + STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc, true)); SStmtTableCache cache = { - .pDataBlock = pDst, + .pDataCtx = pDst, .boundTags = pStmt->bInfo.boundTags, }; @@ -241,6 +241,8 @@ int32_t stmtCacheBlock(STscStmt* pStmt) { } int32_t stmtParseSql(STscStmt* pStmt) { + pStmt->exec.pCurrBlock = NULL; + SStmtCallback stmtCb = { .pStmt = pStmt, .getTbNameFn = stmtGetTbName, @@ -273,7 +275,7 @@ int32_t stmtCleanBindInfo(STscStmt* pStmt) { pStmt->bInfo.tbName[0] = 0; pStmt->bInfo.tbFName[0] = 0; if (!pStmt->bInfo.tagsCached) { - destroyBoundColumnInfo(pStmt->bInfo.boundTags); + qDestroyBoundColInfo(pStmt->bInfo.boundTags); taosMemoryFreeClear(pStmt->bInfo.boundTags); } memset(pStmt->bInfo.stbFName, 0, TSDB_TABLE_FNAME_LEN); @@ -289,29 +291,25 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) { size_t keyLen = 0; void* pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); while (pIter) { - STableDataBlocks* pBlocks = *(STableDataBlocks**)pIter; - char* key = taosHashGetKey(pIter, &keyLen); - STableMeta* pMeta = qGetTableMetaInDataBlock(pBlocks); + STableDataCxt* pBlocks = *(STableDataCxt**)pIter; + char* key = taosHashGetKey(pIter, &keyLen); + STableMeta* pMeta = qGetTableMetaInDataBlock(pBlocks); - if (keepTable && (strlen(pStmt->bInfo.tbFName) == keyLen) && strncmp(pStmt->bInfo.tbFName, key, keyLen) == 0) { - STMT_ERR_RET(qResetStmtDataBlock(pBlocks, true)); + if (keepTable && pBlocks == pStmt->exec.pCurrBlock) { + ASSERT(NULL == pBlocks->pData); + TSWAP(pBlocks->pData, pStmt->exec.pCurrTbData); + STMT_ERR_RET(qResetStmtDataBlock(pBlocks, false)); pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); continue; } - if (STMT_TYPE_MULTI_INSERT == pStmt->sql.type) { - qFreeStmtDataBlock(pBlocks); - } else { - qDestroyStmtDataBlock(pBlocks); - } + qDestroyStmtDataBlock(pBlocks); taosHashRemove(pStmt->exec.pBlockHash, key, keyLen); pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); } - pStmt->exec.autoCreateTbl = false; - if (keepTable) { return TSDB_CODE_SUCCESS; } @@ -319,6 +317,9 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) { taosHashCleanup(pStmt->exec.pBlockHash); pStmt->exec.pBlockHash = NULL; + tDestroySSubmitTbData(pStmt->exec.pCurrTbData, TSDB_MSG_FLG_ENCODE); + taosMemoryFreeClear(pStmt->exec.pCurrTbData); + STMT_ERR_RET(stmtCleanBindInfo(pStmt)); return TSDB_CODE_SUCCESS; @@ -337,8 +338,8 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { while (pIter) { SStmtTableCache* pCache = (SStmtTableCache*)pIter; - qDestroyStmtDataBlock(pCache->pDataBlock); - destroyBoundColumnInfo(pCache->boundTags); + qDestroyStmtDataBlock(pCache->pDataCtx); + qDestroyBoundColInfo(pCache->boundTags); taosMemoryFreeClear(pCache->boundTags); pIter = taosHashIterate(pStmt->sql.pTableCache, pIter); @@ -354,7 +355,8 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } -int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataBlocks* pDataBlock, STableDataBlocks** newBlock, uint64_t uid) { +int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataCxt* pDataBlock, STableDataCxt** newBlock, uint64_t uid, + uint64_t suid) { SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); SVgroupInfo vgInfo = {0}; SRequestConnInfo conn = {.pTrans = pStmt->taos->pAppInfo->pTransporter, @@ -366,7 +368,9 @@ int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataBlocks* pDataBlock, STab STMT_ERR_RET( taosHashPut(pStmt->sql.pVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo))); - STMT_ERR_RET(qRebuildStmtDataBlock(newBlock, pDataBlock, uid, vgInfo.vgId)); + STMT_ERR_RET(qRebuildStmtDataBlock(newBlock, pDataBlock, uid, suid, vgInfo.vgId, pStmt->sql.autoCreateTbl)); + + STMT_DLOG("tableDataCxt rebuilt, uid:%" PRId64 ", vgId:%d", uid, vgInfo.vgId); return TSDB_CODE_SUCCESS; } @@ -375,12 +379,13 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.needParse = true; pStmt->bInfo.inExecCache = false; - STableDataBlocks* pBlockInExec = - taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); - if (pBlockInExec) { + STableDataCxt** pCxtInExec = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + if (pCxtInExec) { pStmt->bInfo.needParse = false; pStmt->bInfo.inExecCache = true; + pStmt->exec.pCurrBlock = *pCxtInExec; + if (pStmt->sql.autoCreateTbl) { tscDebug("reuse stmt block for tb %s in execBlock", pStmt->bInfo.tbFName); return TSDB_CODE_SUCCESS; @@ -407,18 +412,18 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pStmt->bInfo.tbSuid, sizeof(pStmt->bInfo.tbSuid)); if (pCache) { pStmt->bInfo.needParse = false; - pStmt->exec.autoCreateTbl = true; - pStmt->bInfo.tbUid = 0; - STableDataBlocks* pNewBlock = NULL; - STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataBlock, &pNewBlock, 0)); + STableDataCxt* pNewBlock = NULL; + STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataCtx, &pNewBlock, 0, pStmt->bInfo.tbSuid)); if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + pStmt->exec.pCurrBlock = pNewBlock; + tscDebug("reuse stmt block for tb %s in sqlBlock, suid:0x%" PRIx64, pStmt->bInfo.tbFName, pStmt->bInfo.tbSuid); return TSDB_CODE_SUCCESS; @@ -489,14 +494,16 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.boundTags = pCache->boundTags; pStmt->bInfo.tagsCached = true; - STableDataBlocks* pNewBlock = NULL; - STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataBlock, &pNewBlock, uid)); + STableDataCxt* pNewBlock = NULL; + STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataCtx, &pNewBlock, uid, suid)); if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + pStmt->exec.pCurrBlock = pNewBlock; + tscDebug("tb %s in sqlBlock list, set to current", pStmt->bInfo.tbFName); return TSDB_CODE_SUCCESS; @@ -614,8 +621,8 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { return TSDB_CODE_SUCCESS; } - STableDataBlocks** pDataBlock = - (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + STableDataCxt** pDataBlock = + (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_APP_ERROR); @@ -626,8 +633,6 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); - pStmt->exec.autoCreateTbl = true; - return TSDB_CODE_SUCCESS; } @@ -637,8 +642,8 @@ int stmtFetchTagFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); } - STableDataBlocks** pDataBlock = - (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + STableDataCxt** pDataBlock = + (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_APP_ERROR); @@ -655,8 +660,8 @@ int stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); } - STableDataBlocks** pDataBlock = - (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + STableDataCxt** pDataBlock = + (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_APP_ERROR); @@ -729,11 +734,18 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { return TSDB_CODE_SUCCESS; } - STableDataBlocks** pDataBlock = - (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); - if (NULL == pDataBlock) { - tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); - STMT_ERR_RET(TSDB_CODE_APP_ERROR); + STableDataCxt** pDataBlock = NULL; + + if (pStmt->exec.pCurrBlock) { + pDataBlock = &pStmt->exec.pCurrBlock; + } else { + pDataBlock = + (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + if (NULL == pDataBlock) { + tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); + STMT_ERR_RET(TSDB_CODE_TSC_STMT_CACHE_ERROR); + } + pStmt->exec.pCurrBlock = *pDataBlock; } if (colIdx < 0) { @@ -779,10 +791,10 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) { int32_t code = 0; int32_t finalCode = 0; size_t keyLen = 0; - STableDataBlocks** pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); + void* pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); while (pIter) { - STableDataBlocks* pBlock = *pIter; - char* key = taosHashGetKey(pIter, &keyLen); + STableDataCxt* pBlock = *(STableDataCxt**)pIter; + char* key = taosHashGetKey(pIter, &keyLen); STableMeta* pMeta = qGetTableMetaInDataBlock(pBlock); if (pMeta->uid) { @@ -848,7 +860,7 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) { pMeta->uid = pTableMeta->uid; pStmt->bInfo.tbUid = pTableMeta->uid; - taosMemoryFree(pTableMeta); + taosMemoryFree(pTableMeta); } pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); @@ -861,7 +873,6 @@ int stmtExec(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; int32_t code = 0; SSubmitRsp* pRsp = NULL; - bool autoCreateTbl = pStmt->exec.autoCreateTbl; STMT_DLOG_E("start to exec"); @@ -870,8 +881,13 @@ int stmtExec(TAOS_STMT* stmt) { if (STMT_TYPE_QUERY == pStmt->sql.type) { launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); } else { + tDestroySSubmitTbData(pStmt->exec.pCurrTbData, TSDB_MSG_FLG_ENCODE); + taosMemoryFreeClear(pStmt->exec.pCurrTbData); + + STMT_ERR_RET(qCloneCurrentTbData(pStmt->exec.pCurrBlock, &pStmt->exec.pCurrTbData)); + STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, pStmt->exec.pBlockHash)); - launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, (autoCreateTbl ? (void**)&pRsp : NULL)); + launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); } if (pStmt->exec.pRequest->code && NEED_CLIENT_HANDLE_ERROR(pStmt->exec.pRequest->code)) { @@ -894,15 +910,6 @@ _return: stmtCleanExecInfo(pStmt, (code ? false : true), false); - if (TSDB_CODE_SUCCESS == code && autoCreateTbl) { - if (NULL == pRsp) { - tscError("no submit resp got for auto create table"); - code = TSDB_CODE_APP_ERROR; - } else { - code = stmtUpdateTableUid(pStmt, pRsp); - } - } - tFreeSSubmitRsp(pRsp); ++pStmt->sql.runTimes; diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 54778e87a788190c55fafc0e4d0db732677d795d..d608447506778cad1e964cdd79183d87be0a88e8 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -25,7 +25,7 @@ #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" -#include "../src/clientSml.c" +#include "../inc/clientSml.h" #include "taos.h" int main(int argc, char **argv) { @@ -40,11 +40,14 @@ TEST(testCase, smlParseInfluxString_Test) { msgBuf.len = 256; SSmlLineInfo elements = {0}; + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->protocol = TSDB_SML_LINE_PROTOCOL; + info->dataFormat = false; // case 1 char *tmp = "\\,st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; char *sql = (char *)taosMemoryCalloc(256, 1); memcpy(sql, tmp, strlen(tmp) + 1); - int ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measureLen, strlen(",st")); @@ -58,28 +61,23 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 2 false tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); - - // case 3 false - tmp = "st, t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); - ASSERT_EQ(ret, 0); - ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 1); - ASSERT_EQ(elements.colsLen, strlen("t1=3,t2=4,t3=t3")); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 4 tag is null tmp = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measureLen, strlen("st")); @@ -93,12 +91,14 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 5 tag is null tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql + 1); ASSERT_EQ(elements.measureLen, strlen("st")); @@ -111,91 +111,104 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + 1 + elements.measureTagsLen + 3 + elements.colsLen + 2); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 6 tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; + smlClearForRerun(info); // case 7 tmp = " st , "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); - ASSERT_EQ(ret, 0); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); + ASSERT_NE(ret, 0); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 8 false tmp = ", st , "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; + taosMemoryFree(sql); + smlDestroyInfo(info); + } TEST(testCase, smlParseCols_Error_Test) { - const char *data[] = {"c=\"89sd", // binary, nchar - "c=j\"89sd\"", - "c=\"89sd\"k", - "c=u", // bool - "c=truet", - "c=f64", // double - "c=8f64f", - "c=8ef64", - "c=f32", // float - "c=8f32f", - "c=8wef32", - "c=-3.402823466e+39f32", - "c=", // double - "c=8f", - "c=8we", - "c=i8", // tiny int - "c=-8i8f", - "c=8wei8", - "c=-999i8", - "c=u8", // u tiny int - "c=8fu8", - "c=8weu8", - "c=999u8", - "c=-8u8", - "c=i16", // small int - "c=8fi16u", - "c=8wei16", - "c=-67787i16", - "c=u16", // u small int - "c=8u16f", - "c=8weu16", - "c=-9u16", - "c=67787u16", - "c=i32", // int - "c=8i32f", - "c=8wei32", - "c=2147483649i32", - "c=u32", // u int - "c=8u32f", - "c=8weu32", - "c=-4u32", - "c=42949672958u32", - "c=i64", // big int - "c=8i64i", - "c=8wei64", - "c=-9223372036854775809i64", - "c=i", // big int - "c=8fi", - "c=8wei", - "c=9223372036854775808i", - "c=u64", // u big int - "c=8u64f", - "c=8weu64", - "c=-3.402823466e+39u64", - "c=-339u64", - "c=18446744073709551616u64", - "c=1,c=2", - "c=1=2"}; - - SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + const char *data[] = {"st,t=1 c=\"89sd 1626006833639000000", // binary, nchar + "st,t=1 c=j\"89sd\" 1626006833639000000", + "st,t=1 c=\"89sd\"k 1626006833639000000", + "st,t=1 c=u 1626006833639000000", // bool + "st,t=1 c=truet 1626006833639000000", + "st,t=1 c=f64 1626006833639000000", // double + "st,t=1 c=8f64f 1626006833639000000", + "st,t=1 c=8ef64 1626006833639000000", + "st,t=1 c=f32 1626006833639000000", // float + "st,t=1 c=8f32f 1626006833639000000", + "st,t=1 c=8wef32 1626006833639000000", + "st,t=1 c=-3.402823466e+39f32 1626006833639000000", + "st,t=1 c= 1626006833639000000", // double + "st,t=1 c=8f 1626006833639000000", + "st,t=1 c=8we 1626006833639000000", + "st,t=1 c=i8 1626006833639000000", // tiny int + "st,t=1 c=-8i8f 1626006833639000000", + "st,t=1 c=8wei8 1626006833639000000", + "st,t=1 c=-999i8 1626006833639000000", + "st,t=1 c=u8 1626006833639000000", // u tiny int + "st,t=1 c=8fu8 1626006833639000000", + "st,t=1 c=8weu8 1626006833639000000", + "st,t=1 c=999u8 1626006833639000000", + "st,t=1 c=-8u8 1626006833639000000", + "st,t=1 c=i16 1626006833639000000", // small int + "st,t=1 c=8fi16u 1626006833639000000", + "st,t=1 c=8wei16 1626006833639000000", + "st,t=1 c=-67787i16 1626006833639000000", + "st,t=1 c=u16 1626006833639000000", // u small int + "st,t=1 c=8u16f 1626006833639000000", + "st,t=1 c=8weu16 1626006833639000000", + "st,t=1 c=-9u16 1626006833639000000", + "st,t=1 c=67787u16 1626006833639000000", + "st,t=1 c=i32 1626006833639000000", // int + "st,t=1 c=8i32f 1626006833639000000", + "st,t=1 c=8wei32 1626006833639000000", + "st,t=1 c=2147483649i32 1626006833639000000", + "st,t=1 c=u32 1626006833639000000", // u int + "st,t=1 c=8u32f 1626006833639000000", + "st,t=1 c=8weu32 1626006833639000000", + "st,t=1 c=-4u32 1626006833639000000", + "st,t=1 c=42949672958u32 1626006833639000000", + "st,t=1 c=i64 1626006833639000000", // big int + "st,t=1 c=8i64i 1626006833639000000", + "st,t=1 c=8wei64 1626006833639000000", + "st,t=1 c=-9223372036854775809i64 1626006833639000000", + "st,t=1 c=i 1626006833639000000", // big int + "st,t=1 c=8fi 1626006833639000000", + "st,t=1 c=8wei 1626006833639000000", + "st,t=1 c=9223372036854775808i 1626006833639000000", + "st,t=1 c=u64 1626006833639000000", // u big int + "st,t=1 c=8u64f 1626006833639000000", + "st,t=1 c=8weu64 1626006833639000000", + "st,t=1 c=-3.402823466e+39u64 1626006833639000000", + "st,t=1 c=-339u64 1626006833639000000", + "st,t=1 c=18446744073709551616u64 1626006833639000000", + "st,t=1 c=1=2 1626006833639000000"}; + + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->protocol = TSDB_SML_LINE_PROTOCOL; + info->dataFormat = false; for (int i = 0; i < sizeof(data) / sizeof(data[0]); i++) { char msg[256] = {0}; SSmlMsgBuf msgBuf; @@ -204,76 +217,14 @@ TEST(testCase, smlParseCols_Error_Test) { int32_t len = strlen(data[i]); char *sql = (char *)taosMemoryCalloc(256, 1); memcpy(sql, data[i], len + 1); - SArray *cols = taosArrayInit(8, POINTER_BYTES); - int32_t ret = smlParseCols(sql, len, cols, NULL, false, dumplicateKey, &msgBuf); - printf("i:%d\n", i); + SSmlLineInfo elements = {0}; + int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); +// printf("i:%d\n", i); ASSERT_NE(ret, TSDB_CODE_SUCCESS); - taosHashClear(dumplicateKey); taosMemoryFree(sql); - for (int j = 0; j < taosArrayGetSize(cols); j++) { - void *kv = taosArrayGetP(cols, j); - taosMemoryFree(kv); - } - taosArrayDestroy(cols); + taosArrayDestroy(elements.colArray); } - taosHashCleanup(dumplicateKey); -} - -TEST(testCase, smlParseCols_tag_Test) { - char msg[256] = {0}; - SSmlMsgBuf msgBuf; - msgBuf.buf = msg; - msgBuf.len = 256; - - SArray *cols = taosArrayInit(16, POINTER_BYTES); - ASSERT_NE(cols, nullptr); - SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - - const char *data = - "cbin=\"passit " - "helloc\",cnch=L\"iisdfsf\",cbool=false,cf64=4.31f64,cf64_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=" - "898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_" - "=l\"iuwq\""; - int32_t len = strlen(data); - int32_t ret = smlParseCols(data, len, cols, NULL, true, dumplicateKey, &msgBuf); - ASSERT_EQ(ret, TSDB_CODE_SUCCESS); - int32_t size = taosArrayGetSize(cols); - ASSERT_EQ(size, 19); - - // nchar - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, 0); - ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0); - ASSERT_EQ(kv->keyLen, 4); - ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); - ASSERT_EQ(kv->length, 15); - ASSERT_EQ(strncasecmp(kv->value, "\"passit", 7), 0); - - // nchar - kv = (SSmlKv *)taosArrayGetP(cols, 3); - ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); - ASSERT_EQ(kv->keyLen, 4); - ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); - ASSERT_EQ(kv->length, 7); - ASSERT_EQ(strncasecmp(kv->value, "4.31f64", 7), 0); - - for (int i = 0; i < size; i++) { - void *tmp = taosArrayGetP(cols, i); - taosMemoryFree(tmp); - } - taosArrayClear(cols); - - // test tag is null - data = "t=3e"; - len = 0; - memset(msgBuf.buf, 0, msgBuf.len); - taosHashClear(dumplicateKey); - ret = smlParseCols(data, len, cols, NULL, true, dumplicateKey, &msgBuf); - ASSERT_EQ(ret, TSDB_CODE_SUCCESS); - size = taosArrayGetSize(cols); - ASSERT_EQ(size, 0); - - taosArrayDestroy(cols); - taosHashCleanup(dumplicateKey); + smlDestroyInfo(info); } TEST(testCase, smlParseCols_Test) { @@ -281,226 +232,207 @@ TEST(testCase, smlParseCols_Test) { SSmlMsgBuf msgBuf; msgBuf.buf = msg; msgBuf.len = 256; - - SArray *cols = taosArrayInit(16, POINTER_BYTES); - ASSERT_NE(cols, nullptr); - - SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->protocol = TSDB_SML_LINE_PROTOCOL; + info->dataFormat = false; + SSmlLineInfo elements = {0}; + info->msgBuf = msgBuf; const char *data = - "cb\\=in=\"pass\\,it " + "st,t=1 cb\\=in=\"pass\\,it " "hello,c=2\",cnch=L\"ii\\=sdfsf\",cbool=false,cf64=4.31f64,cf64_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=" "233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t," - "cboolf=f,cnch_=l\"iuwq\""; + "cboolf=f,cnch_=l\"iuwq\" 1626006833639000000"; int32_t len = strlen(data); char *sql = (char *)taosMemoryCalloc(1024, 1); memcpy(sql, data, len + 1); - int32_t ret = smlParseCols(sql, len, cols, NULL, false, dumplicateKey, &msgBuf); + int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); ASSERT_EQ(ret, TSDB_CODE_SUCCESS); - int32_t size = taosArrayGetSize(cols); - ASSERT_EQ(size, 19); + int32_t size = taosArrayGetSize(elements.colArray); + ASSERT_EQ(size, 20); // binary - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, 0); + SSmlKv *kv = (SSmlKv *)taosArrayGet(elements.colArray, 1); ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY); ASSERT_EQ(kv->length, 17); ASSERT_EQ(strncasecmp(kv->value, "pass,it ", 8), 0); - taosMemoryFree(kv); // nchar - kv = (SSmlKv *)taosArrayGetP(cols, 1); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 2); ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->length, 8); ASSERT_EQ(strncasecmp(kv->value, "ii=sd", 5), 0); - taosMemoryFree(kv); // bool - kv = (SSmlKv *)taosArrayGetP(cols, 2); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 3); ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, false); - taosMemoryFree(kv); // double - kv = (SSmlKv *)taosArrayGetP(cols, 3); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 4); ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); ASSERT_EQ(kv->length, 8); // ASSERT_EQ(kv->d, 4.31); printf("4.31 = kv->d:%f\n", kv->d); - taosMemoryFree(kv); // float - kv = (SSmlKv *)taosArrayGetP(cols, 4); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 5); ASSERT_EQ(strncasecmp(kv->key, "cf64_", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); ASSERT_EQ(kv->length, 8); // ASSERT_EQ(kv->f, 8.32); printf("8.32 = kv->d:%f\n", kv->d); - taosMemoryFree(kv); // float - kv = (SSmlKv *)taosArrayGetP(cols, 5); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 6); ASSERT_EQ(strncasecmp(kv->key, "cf32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); ASSERT_EQ(kv->length, 4); // ASSERT_EQ(kv->f, 8.23); printf("8.23 = kv->f:%f\n", kv->f); - taosMemoryFree(kv); // tiny int - kv = (SSmlKv *)taosArrayGetP(cols, 6); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 7); ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0); ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, -34); - taosMemoryFree(kv); // unsigned tiny int - kv = (SSmlKv *)taosArrayGetP(cols, 7); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 8); ASSERT_EQ(strncasecmp(kv->key, "cu8", 3), 0); ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UTINYINT); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->u, 89); - taosMemoryFree(kv); // small int - kv = (SSmlKv *)taosArrayGetP(cols, 8); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 9); ASSERT_EQ(strncasecmp(kv->key, "ci16", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_SMALLINT); ASSERT_EQ(kv->length, 2); ASSERT_EQ(kv->u, 233); - taosMemoryFree(kv); // unsigned smallint - kv = (SSmlKv *)taosArrayGetP(cols, 9); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 10); ASSERT_EQ(strncasecmp(kv->key, "cu16", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_USMALLINT); ASSERT_EQ(kv->length, 2); ASSERT_EQ(kv->u, 898); - taosMemoryFree(kv); // int - kv = (SSmlKv *)taosArrayGetP(cols, 10); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 11); ASSERT_EQ(strncasecmp(kv->key, "ci32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_INT); ASSERT_EQ(kv->length, 4); ASSERT_EQ(kv->u, 98289); - taosMemoryFree(kv); // unsigned int - kv = (SSmlKv *)taosArrayGetP(cols, 11); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 12); ASSERT_EQ(strncasecmp(kv->key, "cu32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UINT); ASSERT_EQ(kv->length, 4); ASSERT_EQ(kv->u, 12323); - taosMemoryFree(kv); // bigint - kv = (SSmlKv *)taosArrayGetP(cols, 12); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 13); ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(kv->length, 8); ASSERT_EQ(kv->i, -89238); - taosMemoryFree(kv); // bigint - kv = (SSmlKv *)taosArrayGetP(cols, 13); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 14); ASSERT_EQ(strncasecmp(kv->key, "ci", 2), 0); ASSERT_EQ(kv->keyLen, 2); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(kv->length, 8); ASSERT_EQ(kv->i, 989); - taosMemoryFree(kv); // unsigned bigint - kv = (SSmlKv *)taosArrayGetP(cols, 14); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 15); ASSERT_EQ(strncasecmp(kv->key, "cu64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UBIGINT); ASSERT_EQ(kv->length, 8); ASSERT_EQ(kv->u, 8989323); - taosMemoryFree(kv); // bool - kv = (SSmlKv *)taosArrayGetP(cols, 15); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 16); ASSERT_EQ(strncasecmp(kv->key, "cbooltrue", 9), 0); ASSERT_EQ(kv->keyLen, 9); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, true); - taosMemoryFree(kv); // bool - kv = (SSmlKv *)taosArrayGetP(cols, 16); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 17); ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, true); - taosMemoryFree(kv); // bool - kv = (SSmlKv *)taosArrayGetP(cols, 17); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 18); ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, false); - taosMemoryFree(kv); // nchar - kv = (SSmlKv *)taosArrayGetP(cols, 18); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 19); ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->length, 4); ASSERT_EQ(strncasecmp(kv->value, "iuwq", 4), 0); - taosMemoryFree(kv); - taosArrayDestroy(cols); - taosHashCleanup(dumplicateKey); + taosArrayDestroy(elements.colArray); taosMemoryFree(sql); + smlDestroyInfo(info); } -TEST(testCase, smlGetTimestampLen_Test) { - uint8_t len = smlGetTimestampLen(0); - ASSERT_EQ(len, 1); - - len = smlGetTimestampLen(1); - ASSERT_EQ(len, 1); - - len = smlGetTimestampLen(10); - ASSERT_EQ(len, 2); - - len = smlGetTimestampLen(390); - ASSERT_EQ(len, 3); - - len = smlGetTimestampLen(-1); - ASSERT_EQ(len, 1); - - len = smlGetTimestampLen(-10); - ASSERT_EQ(len, 2); - - len = smlGetTimestampLen(-390); - ASSERT_EQ(len, 3); -} +//TEST(testCase, smlGetTimestampLen_Test) { +// uint8_t len = smlGetTimestampLen(0); +// ASSERT_EQ(len, 1); +// +// len = smlGetTimestampLen(1); +// ASSERT_EQ(len, 1); +// +// len = smlGetTimestampLen(10); +// ASSERT_EQ(len, 2); +// +// len = smlGetTimestampLen(390); +// ASSERT_EQ(len, 3); +// +// len = smlGetTimestampLen(-1); +// ASSERT_EQ(len, 1); +// +// len = smlGetTimestampLen(-10); +// ASSERT_EQ(len, 2); +// +// len = smlGetTimestampLen(-390); +// ASSERT_EQ(len, 3); +//} TEST(testCase, smlParseNumber_Test) { SSmlKv kv = {0}; @@ -515,7 +447,9 @@ TEST(testCase, smlParseNumber_Test) { } TEST(testCase, smlParseTelnetLine_error_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->dataFormat = false; + info->protocol = TSDB_SML_TELNET_PROTOCOL; ASSERT_NE(info, nullptr); const char *sql[] = { @@ -532,479 +466,95 @@ TEST(testCase, smlParseTelnetLine_error_Test) { "sys.procs.running 1479496100 42 ", "sys.procs.running 1479496100 42 host= ", "sys.procs.running 1479496100 42or host=web01", - "sys.procs.running 1479496100 true host=web01", - "sys.procs.running 1479496100 \"binary\" host=web01", - "sys.procs.running 1479496100 L\"rfr\" host=web01", +// "sys.procs.running 1479496100 true host=web01", +// "sys.procs.running 1479496100 \"binary\" host=web01", +// "sys.procs.running 1479496100 L\"rfr\" host=web01", "sys.procs.running 1479496100 42 host=web01 cpu= ", - "sys.procs.running 1479496100 42 host=web01 host=w2", "sys.procs.running 1479496100 42 host=web01 host", "sys.procs.running 1479496100 42 host=web01=er", "sys.procs.running 1479496100 42 host= web01", }; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - int ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); + SSmlLineInfo elements = {0}; + int ret = smlParseTelnetString(info, (char*)sql[i], (char*)(sql[i] + strlen(sql[i])), &elements); +// printf("i:%d\n", i); ASSERT_NE(ret, 0); } smlDestroyInfo(info); } -TEST(testCase, smlParseTelnetLine_diff_type_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - - const char *sql[] = {"sys.procs.running 1479496104000 42 host=web01", - "sys.procs.running 1479496104000 42u8 host=web01", - "appywjnuct 1626006833641 True id=\"appywjnuct_40601_49808_1\" t0=t t1=127i8 " - "id=\"appywjnuct_40601_49808_2\" t2=32767i16 t3=2147483647i32 t4=9223372036854775807i64 " - "t5=11.12345f32 t6=22.123456789f64 t7=\"binaryTagValue\" t8=L\"ncharTagValue\""}; - - int ret = TSDB_CODE_SUCCESS; - for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); - if (ret != TSDB_CODE_SUCCESS) break; - } - ASSERT_NE(ret, 0); - smlDestroyInfo(info); -} - -TEST(testCase, smlParseTelnetLine_json_error_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); +TEST(testCase, smlParseTelnetLine_Test) { + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->dataFormat = false; + info->protocol = TSDB_SML_TELNET_PROTOCOL; ASSERT_NE(info, nullptr); const char *sql[] = { - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 13468464009999333322222223,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"web01\",\n" - " \"dc\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400i,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"web01\",\n" - " \"dc\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"groupid\": { \n" - " \"value\" : 2,\n" - " \"type\" : \"nchar\"\n" - " },\n" - " \"location\": { \n" - " \"value\" : \"北京\",\n" - " \"type\" : \"binary\"\n" - " },\n" - " \"id\": \"d1001\"\n" - " }\n" - " },\n" - "]", + "twudyr 1626006833641 \"abcd`~!@#$%^&*()_-{[}]|:;<.>?lfjal\" id=twudyr_17102_17825 t0=t t1=127i8 t2=32767i16 t3=2147483647i32 t4=9223372036854775807i64 t5=11.12345f32 t6=22.123456789f64 t7=\"abcd`~!@#$%^&*()_-{[}]|:;<.>?lfjal\" t8=L\"abcd`~!@#$%^&*()_-{[}]|:;<.>?lfjal\"", }; - - int ret = TSDB_CODE_SUCCESS; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); - ASSERT_NE(ret, 0); + SSmlLineInfo elements = {0}; + int ret = smlParseTelnetString(info, (char*)sql[i], (char*)(sql[i] + strlen(sql[i])), &elements); +// printf("i:%d\n", i); + ASSERT_EQ(ret, 0); } smlDestroyInfo(info); } -TEST(testCase, smlParseTelnetLine_diff_json_type1_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - - const char *sql[] = { - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": 8\n" - " }\n" - " },\n" - "]", - }; - - int ret = TSDB_CODE_SUCCESS; - for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); - if (ret != TSDB_CODE_SUCCESS) break; - } - ASSERT_NE(ret, 0); - smlDestroyInfo(info); -} - TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->protocol = TSDB_SML_JSON_PROTOCOL; ASSERT_NE(info, nullptr); const char *sql[] = { - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": \"18\",\n" - " \"tags\": {\n" - " \"host\": \"fff\"\n" - " }\n" - " },\n" - "]", + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\": 1346846400,\"value\": 18,\"tags\": {\"host\": \"lga\"}},{\"metric\": \"sys.sdfa\",\"timestamp\": 1346846400,\"value\": \"18\",\"tags\": {\"host\": 8932}},]", }; - int ret = TSDB_CODE_SUCCESS; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); - if (ret != TSDB_CODE_SUCCESS) break; + char *dataPointStart = (char *)sql[i]; + int8_t offset[4] = {0}; + while (1) { + SSmlLineInfo elements = {0}; + if(offset[0] == 0){ + smlJsonParseObjFirst(&dataPointStart, &elements, offset); + }else{ + smlJsonParseObj(&dataPointStart, &elements, offset); + } + if(*dataPointStart == '\0') break; + + SArray *tags = smlJsonParseTags(elements.tags, elements.tags + elements.tagsLen); + size_t num = taosArrayGetSize(tags); + ASSERT_EQ(num, 1); + + taosArrayDestroy(tags); + } } - ASSERT_NE(ret, 0); smlDestroyInfo(info); } -TEST(testCase, sml_col_4096_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - - const char *sql[] = { - "spgwgvldxv,id=spgwgvldxv_1,t0=f " - "c0=t,c1=t,c2=t,c3=t,c4=t,c5=t,c6=t,c7=t,c8=t,c9=t,c10=t,c11=t,c12=t,c13=t,c14=t,c15=t,c16=t,c17=t,c18=t,c19=t," - "c20=t,c21=t,c22=t,c23=t,c24=t,c25=t,c26=t,c27=t,c28=t,c29=t,c30=t,c31=t,c32=t,c33=t,c34=t,c35=t,c36=t,c37=t,c38=" - "t,c39=t,c40=t,c41=t,c42=t,c43=t,c44=t,c45=t,c46=t,c47=t,c48=t,c49=t,c50=t,c51=t,c52=t,c53=t,c54=t,c55=t,c56=t," - "c57=t,c58=t,c59=t,c60=t,c61=t,c62=t,c63=t,c64=t,c65=t,c66=t,c67=t,c68=t,c69=t,c70=t,c71=t,c72=t,c73=t,c74=t,c75=" - "t,c76=t,c77=t,c78=t,c79=t,c80=t,c81=t,c82=t,c83=t,c84=t,c85=t,c86=t,c87=t,c88=t,c89=t,c90=t,c91=t,c92=t,c93=t," - "c94=t,c95=t,c96=t,c97=t,c98=t,c99=t,c100=t," - "c101=t,c102=t,c103=t,c104=t,c105=t,c106=t,c107=t,c108=t,c109=t,c110=t,c111=t,c112=t,c113=t,c114=t,c115=t,c116=t," - "c117=t,c118=t,c119=t,c120=t,c121=t,c122=t,c123=t,c124=t,c125=t,c126=t,c127=t,c128=t,c129=t,c130=t,c131=t,c132=t," - "c133=t,c134=t,c135=t,c136=t,c137=t,c138=t,c139=t,c140=t,c141=t,c142=t,c143=t,c144=t,c145=t,c146=t,c147=t,c148=t," - "c149=t,c150=t,c151=t,c152=t,c153=t,c154=t,c155=t,c156=t,c157=t,c158=t,c159=t,c160=t,c161=t,c162=t,c163=t,c164=t," - "c165=t,c166=t,c167=t,c168=t,c169=t,c170=t,c171=t,c172=t,c173=t,c174=t,c175=t,c176=t,c177=t,c178=t,c179=t,c180=t," - "c181=t,c182=t,c183=t,c184=t,c185=t,c186=t,c187=t,c188=t,c189=t," - "c190=t,c191=t,c192=t,c193=t,c194=t,c195=t,c196=t,c197=t,c198=t,c199=t,c200=t,c201=t,c202=t,c203=t,c204=t,c205=t," - "c206=t,c207=t,c208=t,c209=t,c210=t,c211=t,c212=t,c213=t,c214=t,c215=t,c216=t,c217=t,c218=t,c219=t,c220=t,c221=t," - "c222=t,c223=t,c224=t,c225=t,c226=t,c227=t,c228=t,c229=t,c230=t,c231=t,c232=t,c233=t,c234=t,c235=t,c236=t,c237=t," - "c238=t,c239=t,c240=t,c241=t,c242=t,c243=t,c244=t,c245=t,c246=t,c247=t,c248=t,c249=t,c250=t,c251=t,c252=t,c253=t," - "c254=t,c255=t,c256=t,c257=t,c258=t,c259=t,c260=t,c261=t,c262=t,c263=t,c264=t,c265=t,c266=t,c267=t,c268=t,c269=t," - "c270=t,c271=t,c272=t,c273=t,c274=t,c275=t,c276=t,c277=t,c278=t," - "c279=t,c280=t,c281=t,c282=t,c283=t,c284=t,c285=t,c286=t,c287=t,c288=t,c289=t,c290=t,c291=t,c292=t,c293=t,c294=t," - "c295=t,c296=t,c297=t,c298=t,c299=t,c300=t,c301=t,c302=t,c303=t,c304=t,c305=t,c306=t,c307=t,c308=t,c309=t,c310=t," - "c311=t,c312=t,c313=t,c314=t,c315=t,c316=t,c317=t,c318=t,c319=t,c320=t,c321=t,c322=t,c323=t,c324=t,c325=t,c326=t," - "c327=t,c328=t,c329=t,c330=t,c331=t,c332=t,c333=t,c334=t,c335=t,c336=t,c337=t,c338=t,c339=t,c340=t,c341=t,c342=t," - "c343=t,c344=t,c345=t,c346=t,c347=t,c348=t,c349=t,c350=t,c351=t,c352=t,c353=t,c354=t,c355=t,c356=t,c357=t,c358=t," - "c359=t,c360=t,c361=t,c362=t,c363=t,c364=t,c365=t,c366=t,c367=t,c368=t,c369=t,c370=t,c371=t,c372=t,c373=t,c374=t," - "c375=t,c376=t,c377=t,c378=t,c379=t,c380=t,c381=t,c382=t,c383=t,c384=t,c385=t,c386=t,c387=t,c388=t,c389=t,c390=t," - "c391=t,c392=t,c393=t,c394=t,c395=t,c396=t,c397=t,c398=t,c399=t,c400=t,c401=t,c402=t,c403=t,c404=t,c405=t,c406=t," - "c407=t,c408=t,c409=t,c410=t,c411=t,c412=t,c413=t,c414=t,c415=t,c416=t,c417=t,c418=t,c419=t,c420=t,c421=t,c422=t," - "c423=t,c424=t,c425=t,c426=t,c427=t,c428=t,c429=t,c430=t,c431=t,c432=t,c433=t,c434=t,c435=t,c436=t,c437=t,c438=t," - "c439=t,c440=t,c441=t,c442=t,c443=t,c444=t,c445=t,c446=t," - "c447=t,c448=t,c449=t,c450=t,c451=t,c452=t,c453=t,c454=t,c455=t,c456=t,c457=t,c458=t,c459=t,c460=t,c461=t,c462=t," - "c463=t,c464=t,c465=t,c466=t,c467=t,c468=t,c469=t,c470=t,c471=t,c472=t,c473=t,c474=t,c475=t,c476=t,c477=t,c478=t," - "c479=t,c480=t,c481=t,c482=t,c483=t,c484=t,c485=t,c486=t,c487=t,c488=t,c489=t,c490=t,c491=t,c492=t,c493=t,c494=t," - "c495=t,c496=t,c497=t,c498=t,c499=t,c500=t,c501=t,c502=t,c503=t,c504=t,c505=t,c506=t,c507=t,c508=t,c509=t,c510=t," - "c511=t,c512=t,c513=t,c514=t,c515=t,c516=t,c517=t,c518=t,c519=t,c520=t,c521=t,c522=t,c523=t,c524=t,c525=t,c526=t," - "c527=t,c528=t,c529=t,c530=t,c531=t,c532=t,c533=t,c534=t,c535=t,c536=t,c537=t,c538=t,c539=t,c540=t,c541=t,c542=t," - "c543=t,c544=t,c545=t,c546=t,c547=t,c548=t,c549=t,c550=t,c551=t,c552=t,c553=t,c554=t,c555=t,c556=t,c557=t,c558=t," - "c559=t,c560=t,c561=t,c562=t,c563=t,c564=t,c565=t,c566=t,c567=t,c568=t,c569=t,c570=t,c571=t,c572=t,c573=t,c574=t," - "c575=t,c576=t,c577=t,c578=t,c579=t,c580=t,c581=t,c582=t,c583=t,c584=t,c585=t,c586=t,c587=t,c588=t,c589=t,c590=t," - "c591=t,c592=t,c593=t,c594=t,c595=t,c596=t,c597=t,c598=t,c599=t,c600=t,c601=t,c602=t,c603=t,c604=t,c605=t,c606=t," - "c607=t,c608=t,c609=t,c610=t,c611=t,c612=t,c613=t,c614=t," - "c615=t,c616=t,c617=t,c618=t,c619=t,c620=t,c621=t,c622=t,c623=t,c624=t,c625=t,c626=t,c627=t,c628=t,c629=t,c630=t," - "c631=t,c632=t,c633=t,c634=t,c635=t,c636=t,c637=t,c638=t,c639=t,c640=t,c641=t,c642=t,c643=t,c644=t,c645=t,c646=t," - "c647=t,c648=t,c649=t,c650=t,c651=t,c652=t,c653=t,c654=t,c655=t,c656=t,c657=t,c658=t,c659=t,c660=t,c661=t,c662=t," - "c663=t,c664=t,c665=t,c666=t,c667=t,c668=t,c669=t,c670=t,c671=t,c672=t,c673=t,c674=t,c675=t,c676=t,c677=t,c678=t," - "c679=t,c680=t,c681=t,c682=t,c683=t,c684=t,c685=t,c686=t,c687=t,c688=t,c689=t,c690=t,c691=t,c692=t,c693=t,c694=t," - "c695=t,c696=t,c697=t,c698=t,c699=t,c700=t,c701=t,c702=t,c703=t,c704=t,c705=t,c706=t,c707=t,c708=t,c709=t,c710=t," - "c711=t,c712=t,c713=t,c714=t,c715=t,c716=t,c717=t,c718=t,c719=t,c720=t,c721=t,c722=t,c723=t,c724=t,c725=t,c726=t," - "c727=t,c728=t,c729=t,c730=t,c731=t,c732=t,c733=t,c734=t,c735=t,c736=t,c737=t,c738=t,c739=t,c740=t,c741=t,c742=t," - "c743=t,c744=t,c745=t,c746=t,c747=t,c748=t,c749=t,c750=t,c751=t,c752=t,c753=t,c754=t,c755=t,c756=t,c757=t,c758=t," - "c759=t,c760=t,c761=t,c762=t,c763=t,c764=t,c765=t,c766=t,c767=t,c768=t,c769=t,c770=t,c771=t,c772=t,c773=t,c774=t," - "c775=t,c776=t,c777=t,c778=t,c779=t,c780=t,c781=t,c782=t," - "c783=t,c784=t,c785=t,c786=t,c787=t,c788=t,c789=t,c790=t,c791=t,c792=t,c793=t,c794=t,c795=t,c796=t,c797=t,c798=t," - "c799=t,c800=t,c801=t,c802=t,c803=t,c804=t,c805=t,c806=t,c807=t,c808=t,c809=t,c810=t,c811=t,c812=t,c813=t," - "c814=t,c815=t,c816=t,c817=t,c818=t,c819=t,c820=t,c821=t,c822=t,c823=t,c824=t,c825=t,c826=t,c827=t,c828=t,c829=t," - "c830=t,c831=t,c832=t,c833=t,c834=t,c835=t,c836=t,c837=t,c838=t,c839=t,c840=t,c841=t,c842=t,c843=t,c844=t,c845=t," - "c846=t,c847=t,c848=t,c849=t,c850=t,c851=t,c852=t,c853=t,c854=t,c855=t,c856=t,c857=t,c858=t,c859=t,c860=t,c861=t," - "c862=t," - "c863=t,c864=t,c865=t,c866=t,c867=t,c868=t,c869=t,c870=t,c871=t,c872=t,c873=t,c874=t,c875=t,c876=t,c877=t,c878=t," - "c879=t,c880=t,c881=t,c882=t,c883=t,c884=t,c885=t,c886=t,c887=t,c888=t,c889=t,c890=t,c891=t,c892=t,c893=t,c894=t," - "c895=t,c896=t,c897=t,c898=t,c899=t,c900=t,c901=t,c902=t,c903=t,c904=t,c905=t,c906=t,c907=t,c908=t,c909=t,c910=t," - "c911=t,c912=t,c913=t,c914=t,c915=t,c916=t,c917=t,c918=t,c919=t,c920=t,c921=t,c922=t,c923=t,c924=t,c925=t,c926=t," - "c927=t,c928=t,c929=t,c930=t,c931=t,c932=t,c933=t,c934=t,c935=t,c936=t,c937=t,c938=t,c939=t,c940=t,c941=t,c942=t," - "c943=t,c944=t,c945=t,c946=t,c947=t,c948=t,c949=t,c950=t,c951=t,c952=t,c953=t,c954=t,c955=t,c956=t,c957=t,c958=t," - "c959=t,c960=t,c961=t,c962=t,c963=t,c964=t,c965=t,c966=t,c967=t,c968=t,c969=t,c970=t,c971=t,c972=t,c973=t,c974=t," - "c975=t,c976=t,c977=t,c978=t,c979=t,c980=t,c981=t,c982=t,c983=t,c984=t,c985=t,c986=t,c987=t,c988=t,c989=t,c990=t," - "c991=t,c992=t,c993=t,c994=t,c995=t,c996=t,c997=t,c998=t,c999=t,c1000=t,c1001=t,c1002=t,c1003=t,c1004=t,c1005=t," - "c1006=t,c1007=t,c1008=t,c1009=t,c1010=t,c1011=t,c1012=t,c1013=t,c1014=t,c1015=t,c1016=t,c1017=t,c1018=t,c1019=t," - "c1020=t,c1021=t,c1022=t,c1023=t,c1024=t,c1025=t,c1026=t," - "c1027=t,c1028=t,c1029=t,c1030=t,c1031=t,c1032=t,c1033=t,c1034=t,c1035=t,c1036=t,c1037=t,c1038=t,c1039=t,c1040=t," - "c1041=t,c1042=t,c1043=t,c1044=t,c1045=t,c1046=t,c1047=t,c1048=t,c1049=t,c1050=t,c1051=t,c1052=t,c1053=t,c1054=t," - "c1055=t,c1056=t,c1057=t,c1058=t,c1059=t,c1060=t,c1061=t,c1062=t,c1063=t,c1064=t,c1065=t,c1066=t,c1067=t,c1068=t," - "c1069=t,c1070=t,c1071=t,c1072=t,c1073=t,c1074=t,c1075=t,c1076=t,c1077=t,c1078=t,c1079=t,c1080=t,c1081=t,c1082=t," - "c1083=t,c1084=t,c1085=t,c1086=t,c1087=t,c1088=t,c1089=t,c1090=t,c1091=t,c1092=t,c1093=t,c1094=t,c1095=t,c1096=t," - "c1097=t,c1098=t,c1099=t,c1100=t,c1101=t,c1102=t,c1103=t,c1104=t,c1105=t,c1106=t,c1107=t,c1108=t,c1109=t,c1110=t," - "c1111=t,c1112=t,c1113=t,c1114=t,c1115=t,c1116=t,c1117=t,c1118=t,c1119=t,c1120=t,c1121=t,c1122=t,c1123=t,c1124=t," - "c1125=t,c1126=t,c1127=t,c1128=t,c1129=t,c1130=t,c1131=t,c1132=t,c1133=t,c1134=t,c1135=t,c1136=t,c1137=t,c1138=t," - "c1139=t,c1140=t,c1141=t,c1142=t,c1143=t,c1144=t,c1145=t,c1146=t,c1147=t,c1148=t,c1149=t,c1150=t,c1151=t,c1152=t," - "c1153=t,c1154=t,c1155=t,c1156=t,c1157=t,c1158=t,c1159=t,c1160=t,c1161=t,c1162=t,c1163=t,c1164=t,c1165=t,c1166=t," - "c1167=t,c1168=t,c1169=t,c1170=t,c1171=t,c1172=t,c1173=t," - "c1174=t,c1175=t,c1176=t,c1177=t,c1178=t,c1179=t,c1180=t,c1181=t,c1182=t,c1183=t,c1184=t,c1185=t,c1186=t,c1187=t," - "c1188=t,c1189=t,c1190=t,c1191=t,c1192=t,c1193=t,c1194=t,c1195=t,c1196=t,c1197=t,c1198=t,c1199=t,c1200=t,c1201=t," - "c1202=t,c1203=t,c1204=t,c1205=t,c1206=t,c1207=t,c1208=t,c1209=t,c1210=t,c1211=t,c1212=t,c1213=t,c1214=t,c1215=t," - "c1216=t,c1217=t,c1218=t,c1219=t,c1220=t,c1221=t,c1222=t,c1223=t,c1224=t,c1225=t,c1226=t,c1227=t,c1228=t,c1229=t," - "c1230=t,c1231=t,c1232=t,c1233=t,c1234=t,c1235=t,c1236=t,c1237=t,c1238=t,c1239=t,c1240=t,c1241=t,c1242=t,c1243=t," - "c1244=t,c1245=t,c1246=t,c1247=t,c1248=t,c1249=t,c1250=t,c1251=t,c1252=t,c1253=t,c1254=t,c1255=t,c1256=t,c1257=t," - "c1258=t,c1259=t,c1260=t,c1261=t,c1262=t,c1263=t,c1264=t,c1265=t,c1266=t,c1267=t,c1268=t,c1269=t,c1270=t,c1271=t," - "c1272=t,c1273=t,c1274=t,c1275=t,c1276=t,c1277=t,c1278=t,c1279=t,c1280=t,c1281=t,c1282=t,c1283=t,c1284=t,c1285=t," - "c1286=t,c1287=t,c1288=t,c1289=t,c1290=t,c1291=t,c1292=t,c1293=t,c1294=t,c1295=t,c1296=t,c1297=t,c1298=t,c1299=t," - "c1300=t,c1301=t,c1302=t,c1303=t,c1304=t,c1305=t,c1306=t,c1307=t,c1308=t,c1309=t,c1310=t,c1311=t,c1312=t,c1313=t," - "c1314=t,c1315=t,c1316=t,c1317=t,c1318=t,c1319=t,c1320=t," - "c1321=t,c1322=t,c1323=t,c1324=t,c1325=t,c1326=t,c1327=t,c1328=t,c1329=t,c1330=t,c1331=t,c1332=t,c1333=t,c1334=t," - "c1335=t,c1336=t,c1337=t,c1338=t,c1339=t,c1340=t,c1341=t,c1342=t,c1343=t,c1344=t,c1345=t,c1346=t,c1347=t," - "c1348=t,c1349=t,c1350=t,c1351=t,c1352=t,c1353=t,c1354=t,c1355=t,c1356=t,c1357=t,c1358=t,c1359=t,c1360=t,c1361=t," - "c1362=t,c1363=t,c1364=t,c1365=t,c1366=t,c1367=t,c1368=t,c1369=t,c1370=t,c1371=t,c1372=t,c1373=t,c1374=t,c1375=t," - "c1376=t,c1377=t,c1378=t,c1379=t,c1380=t,c1381=t,c1382=t,c1383=t,c1384=t,c1385=t,c1386=t,c1387=t,c1388=t,c1389=t," - "c1390=t,c1391=t,c1392=t,c1393=t,c1394=t,c1395=t,c1396=t,c1397=t,c1398=t,c1399=t,c1400=t,c1401=t,c1402=t,c1403=t," - "c1404=t,c1405=t,c1406=t,c1407=t,c1408=t,c1409=t,c1410=t,c1411=t,c1412=t,c1413=t,c1414=t,c1415=t,c1416=t,c1417=t," - "c1418=t,c1419=t,c1420=t,c1421=t,c1422=t,c1423=t,c1424=t,c1425=t,c1426=t,c1427=t,c1428=t,c1429=t,c1430=t,c1431=t," - "c1432=t,c1433=t,c1434=t,c1435=t,c1436=t,c1437=t,c1438=t,c1439=t,c1440=t,c1441=t,c1442=t,c1443=t,c1444=t,c1445=t," - "c1446=t,c1447=t,c1448=t,c1449=t,c1450=t,c1451=t,c1452=t,c1453=t,c1454=t,c1455=t,c1456=t,c1457=t,c1458=t,c1459=t," - "c1460=t,c1461=t,c1462=t,c1463=t,c1464=t,c1465=t,c1466=t,c1467=t,c1468=t,c1469=t,c1470=t,c1471=t,c1472=t,c1473=t," - "c1474=t,c1475=t,c1476=t,c1477=t,c1478=t,c1479=t,c1480=t,c1481=t,c1482=t,c1483=t,c1484=t,c1485=t,c1486=t,c1487=t," - "c1488=t,c1489=t,c1490=t,c1491=t,c1492=t,c1493=t,c1494=t," - "c1495=t,c1496=t,c1497=t,c1498=t,c1499=t,c1500=t,c1501=t,c1502=t,c1503=t,c1504=t,c1505=t,c1506=t,c1507=t,c1508=t," - "c1509=t,c1510=t,c1511=t,c1512=t,c1513=t,c1514=t,c1515=t,c1516=t,c1517=t,c1518=t,c1519=t,c1520=t,c1521=t,c1522=t," - "c1523=t,c1524=t,c1525=t,c1526=t,c1527=t,c1528=t,c1529=t,c1530=t,c1531=t,c1532=t,c1533=t,c1534=t,c1535=t,c1536=t," - "c1537=t,c1538=t,c1539=t,c1540=t,c1541=t,c1542=t,c1543=t,c1544=t,c1545=t,c1546=t,c1547=t,c1548=t,c1549=t,c1550=t," - "c1551=t,c1552=t,c1553=t,c1554=t,c1555=t,c1556=t,c1557=t,c1558=t,c1559=t,c1560=t,c1561=t,c1562=t,c1563=t,c1564=t," - "c1565=t,c1566=t,c1567=t,c1568=t,c1569=t,c1570=t,c1571=t,c1572=t,c1573=t,c1574=t,c1575=t,c1576=t,c1577=t,c1578=t," - "c1579=t,c1580=t,c1581=t,c1582=t,c1583=t,c1584=t,c1585=t,c1586=t,c1587=t,c1588=t,c1589=t,c1590=t,c1591=t,c1592=t," - "c1593=t,c1594=t,c1595=t,c1596=t,c1597=t,c1598=t,c1599=t,c1600=t,c1601=t,c1602=t,c1603=t,c1604=t,c1605=t,c1606=t," - "c1607=t,c1608=t,c1609=t,c1610=t,c1611=t,c1612=t,c1613=t,c1614=t,c1615=t,c1616=t,c1617=t,c1618=t,c1619=t,c1620=t," - "c1621=t,c1622=t,c1623=t,c1624=t,c1625=t,c1626=t,c1627=t,c1628=t,c1629=t,c1630=t,c1631=t,c1632=t,c1633=t,c1634=t," - "c1635=t,c1636=t,c1637=t,c1638=t,c1639=t,c1640=t,c1641=t," - "c1642=t,c1643=t,c1644=t,c1645=t,c1646=t,c1647=t,c1648=t,c1649=t,c1650=t,c1651=t,c1652=t,c1653=t,c1654=t,c1655=t," - "c1656=t,c1657=t,c1658=t,c1659=t,c1660=t,c1661=t,c1662=t,c1663=t,c1664=t,c1665=t,c1666=t,c1667=t,c1668=t,c1669=t," - "c1670=t,c1671=t,c1672=t,c1673=t,c1674=t,c1675=t,c1676=t,c1677=t,c1678=t,c1679=t,c1680=t,c1681=t,c1682=t,c1683=t," - "c1684=t,c1685=t,c1686=t,c1687=t,c1688=t,c1689=t,c1690=t,c1691=t,c1692=t,c1693=t,c1694=t,c1695=t,c1696=t,c1697=t," - "c1698=t,c1699=t,c1700=t,c1701=t,c1702=t,c1703=t,c1704=t,c1705=t,c1706=t,c1707=t,c1708=t,c1709=t,c1710=t,c1711=t," - "c1712=t,c1713=t,c1714=t,c1715=t,c1716=t,c1717=t,c1718=t,c1719=t,c1720=t,c1721=t,c1722=t,c1723=t,c1724=t,c1725=t," - "c1726=t,c1727=t,c1728=t,c1729=t,c1730=t,c1731=t,c1732=t,c1733=t,c1734=t,c1735=t,c1736=t,c1737=t,c1738=t,c1739=t," - "c1740=t,c1741=t,c1742=t,c1743=t,c1744=t,c1745=t,c1746=t,c1747=t,c1748=t,c1749=t,c1750=t,c1751=t,c1752=t,c1753=t," - "c1754=t,c1755=t,c1756=t,c1757=t,c1758=t,c1759=t,c1760=t,c1761=t,c1762=t,c1763=t,c1764=t,c1765=t,c1766=t,c1767=t," - "c1768=t,c1769=t,c1770=t,c1771=t,c1772=t,c1773=t,c1774=t,c1775=t,c1776=t,c1777=t,c1778=t,c1779=t,c1780=t,c1781=t," - "c1782=t,c1783=t,c1784=t,c1785=t,c1786=t,c1787=t,c1788=t," - "c1789=t,c1790=t,c1791=t,c1792=t,c1793=t,c1794=t,c1795=t,c1796=t,c1797=t,c1798=t,c1799=t,c1800=t,c1801=t,c1802=t," - "c1803=t,c1804=t,c1805=t,c1806=t,c1807=t,c1808=t,c1809=t,c1810=t,c1811=t,c1812=t,c1813=t,c1814=t,c1815=t," - "c1816=t,c1817=t,c1818=t,c1819=t,c1820=t,c1821=t,c1822=t,c1823=t,c1824=t,c1825=t,c1826=t,c1827=t,c1828=t,c1829=t," - "c1830=t,c1831=t,c1832=t,c1833=t,c1834=t,c1835=t,c1836=t,c1837=t,c1838=t,c1839=t,c1840=t,c1841=t,c1842=t,c1843=t," - "c1844=t,c1845=t,c1846=t,c1847=t,c1848=t,c1849=t,c1850=t,c1851=t,c1852=t,c1853=t,c1854=t,c1855=t,c1856=t,c1857=t," - "c1858=t,c1859=t,c1860=t,c1861=t,c1862=t,c1863=t,c1864=t,c1865=t,c1866=t,c1867=t,c1868=t,c1869=t,c1870=t,c1871=t," - "c1872=t,c1873=t,c1874=t,c1875=t,c1876=t,c1877=t,c1878=t,c1879=t,c1880=t,c1881=t,c1882=t,c1883=t,c1884=t,c1885=t," - "c1886=t,c1887=t,c1888=t,c1889=t,c1890=t,c1891=t,c1892=t,c1893=t,c1894=t,c1895=t,c1896=t,c1897=t,c1898=t,c1899=t," - "c1900=t,c1901=t,c1902=t,c1903=t,c1904=t,c1905=t,c1906=t,c1907=t,c1908=t,c1909=t,c1910=t,c1911=t,c1912=t,c1913=t," - "c1914=t,c1915=t,c1916=t,c1917=t,c1918=t,c1919=t,c1920=t,c1921=t,c1922=t,c1923=t,c1924=t,c1925=t,c1926=t,c1927=t," - "c1928=t,c1929=t,c1930=t,c1931=t,c1932=t,c1933=t,c1934=t,c1935=t,c1936=t,c1937=t,c1938=t,c1939=t,c1940=t,c1941=t," - "c1942=t,c1943=t,c1944=t,c1945=t,c1946=t,c1947=t,c1948=t,c1949=t,c1950=t,c1951=t,c1952=t,c1953=t,c1954=t,c1955=t," - "c1956=t,c1957=t,c1958=t,c1959=t,c1960=t,c1961=t,c1962=t," - "c1963=t,c1964=t,c1965=t,c1966=t,c1967=t,c1968=t,c1969=t,c1970=t,c1971=t,c1972=t,c1973=t,c1974=t,c1975=t,c1976=t," - "c1977=t,c1978=t,c1979=t,c1980=t,c1981=t,c1982=t,c1983=t,c1984=t,c1985=t,c1986=t,c1987=t,c1988=t,c1989=t,c1990=t," - "c1991=t,c1992=t,c1993=t,c1994=t,c1995=t,c1996=t,c1997=t,c1998=t,c1999=t,c2000=t,c2001=t,c2002=t,c2003=t,c2004=t," - "c2005=t,c2006=t,c2007=t,c2008=t,c2009=t,c2010=t,c2011=t,c2012=t,c2013=t,c2014=t,c2015=t,c2016=t,c2017=t,c2018=t," - "c2019=t,c2020=t,c2021=t,c2022=t,c2023=t,c2024=t,c2025=t,c2026=t,c2027=t,c2028=t,c2029=t,c2030=t,c2031=t,c2032=t," - "c2033=t,c2034=t,c2035=t,c2036=t,c2037=t,c2038=t,c2039=t,c2040=t,c2041=t,c2042=t,c2043=t,c2044=t,c2045=t,c2046=t," - "c2047=t,c2048=t,c2049=t,c2050=t,c2051=t,c2052=t,c2053=t,c2054=t,c2055=t,c2056=t,c2057=t,c2058=t,c2059=t,c2060=t," - "c2061=t,c2062=t,c2063=t,c2064=t,c2065=t,c2066=t,c2067=t,c2068=t,c2069=t,c2070=t,c2071=t,c2072=t,c2073=t,c2074=t," - "c2075=t,c2076=t,c2077=t,c2078=t,c2079=t,c2080=t,c2081=t,c2082=t,c2083=t,c2084=t,c2085=t,c2086=t,c2087=t,c2088=t," - "c2089=t,c2090=t,c2091=t,c2092=t,c2093=t,c2094=t,c2095=t,c2096=t,c2097=t,c2098=t,c2099=t,c2100=t,c2101=t,c2102=t," - "c2103=t,c2104=t,c2105=t,c2106=t,c2107=t,c2108=t,c2109=t," - "c2110=t,c2111=t,c2112=t,c2113=t,c2114=t,c2115=t,c2116=t,c2117=t,c2118=t,c2119=t,c2120=t,c2121=t,c2122=t,c2123=t," - "c2124=t,c2125=t,c2126=t,c2127=t,c2128=t,c2129=t,c2130=t,c2131=t,c2132=t,c2133=t,c2134=t,c2135=t,c2136=t,c2137=t," - "c2138=t,c2139=t,c2140=t,c2141=t,c2142=t,c2143=t,c2144=t,c2145=t,c2146=t,c2147=t,c2148=t,c2149=t,c2150=t,c2151=t," - "c2152=t,c2153=t,c2154=t,c2155=t,c2156=t,c2157=t,c2158=t,c2159=t,c2160=t,c2161=t,c2162=t,c2163=t,c2164=t,c2165=t," - "c2166=t,c2167=t,c2168=t,c2169=t,c2170=t,c2171=t,c2172=t,c2173=t,c2174=t,c2175=t,c2176=t,c2177=t,c2178=t,c2179=t," - "c2180=t,c2181=t,c2182=t,c2183=t,c2184=t,c2185=t,c2186=t,c2187=t,c2188=t,c2189=t,c2190=t,c2191=t,c2192=t,c2193=t," - "c2194=t,c2195=t,c2196=t,c2197=t,c2198=t,c2199=t,c2200=t,c2201=t,c2202=t,c2203=t,c2204=t,c2205=t,c2206=t,c2207=t," - "c2208=t,c2209=t,c2210=t,c2211=t,c2212=t,c2213=t,c2214=t,c2215=t,c2216=t,c2217=t,c2218=t,c2219=t,c2220=t,c2221=t," - "c2222=t,c2223=t,c2224=t,c2225=t,c2226=t,c2227=t,c2228=t,c2229=t,c2230=t,c2231=t,c2232=t,c2233=t,c2234=t,c2235=t," - "c2236=t,c2237=t,c2238=t,c2239=t,c2240=t,c2241=t,c2242=t,c2243=t,c2244=t,c2245=t,c2246=t,c2247=t,c2248=t,c2249=t," - "c2250=t,c2251=t,c2252=t,c2253=t,c2254=t,c2255=t,c2256=t," - "c2257=t,c2258=t,c2259=t,c2260=t,c2261=t,c2262=t,c2263=t,c2264=t,c2265=t,c2266=t,c2267=t,c2268=t,c2269=t,c2270=t," - "c2271=t,c2272=t,c2273=t,c2274=t,c2275=t,c2276=t,c2277=t,c2278=t,c2279=t,c2280=t,c2281=t,c2282=t,c2283=t," - "c2284=t,c2285=t,c2286=t,c2287=t,c2288=t,c2289=t,c2290=t,c2291=t,c2292=t,c2293=t,c2294=t,c2295=t,c2296=t,c2297=t," - "c2298=t,c2299=t,c2300=t,c2301=t,c2302=t,c2303=t,c2304=t,c2305=t,c2306=t,c2307=t,c2308=t,c2309=t,c2310=t,c2311=t," - "c2312=t,c2313=t,c2314=t,c2315=t,c2316=t,c2317=t,c2318=t,c2319=t,c2320=t,c2321=t,c2322=t,c2323=t,c2324=t,c2325=t," - "c2326=t,c2327=t,c2328=t,c2329=t,c2330=t,c2331=t,c2332=t,c2333=t,c2334=t,c2335=t,c2336=t,c2337=t,c2338=t,c2339=t," - "c2340=t,c2341=t,c2342=t,c2343=t,c2344=t,c2345=t,c2346=t,c2347=t,c2348=t,c2349=t,c2350=t,c2351=t,c2352=t,c2353=t," - "c2354=t,c2355=t,c2356=t,c2357=t,c2358=t,c2359=t,c2360=t,c2361=t,c2362=t,c2363=t,c2364=t,c2365=t,c2366=t,c2367=t," - "c2368=t,c2369=t,c2370=t,c2371=t,c2372=t,c2373=t,c2374=t,c2375=t,c2376=t,c2377=t,c2378=t,c2379=t,c2380=t,c2381=t," - "c2382=t,c2383=t,c2384=t,c2385=t,c2386=t,c2387=t,c2388=t,c2389=t,c2390=t,c2391=t,c2392=t,c2393=t,c2394=t,c2395=t," - "c2396=t,c2397=t,c2398=t,c2399=t,c2400=t,c2401=t,c2402=t,c2403=t,c2404=t,c2405=t,c2406=t,c2407=t,c2408=t,c2409=t," - "c2410=t,c2411=t,c2412=t,c2413=t,c2414=t,c2415=t,c2416=t,c2417=t,c2418=t,c2419=t,c2420=t,c2421=t,c2422=t,c2423=t," - "c2424=t,c2425=t,c2426=t,c2427=t,c2428=t,c2429=t,c2430=t," - "c2431=t,c2432=t,c2433=t,c2434=t,c2435=t,c2436=t,c2437=t,c2438=t,c2439=t,c2440=t,c2441=t,c2442=t,c2443=t,c2444=t," - "c2445=t,c2446=t,c2447=t,c2448=t,c2449=t,c2450=t,c2451=t,c2452=t,c2453=t,c2454=t,c2455=t,c2456=t,c2457=t,c2458=t," - "c2459=t,c2460=t,c2461=t,c2462=t,c2463=t,c2464=t,c2465=t,c2466=t,c2467=t,c2468=t,c2469=t,c2470=t,c2471=t,c2472=t," - "c2473=t,c2474=t,c2475=t,c2476=t,c2477=t,c2478=t,c2479=t,c2480=t,c2481=t,c2482=t,c2483=t,c2484=t,c2485=t,c2486=t," - "c2487=t,c2488=t,c2489=t,c2490=t,c2491=t,c2492=t,c2493=t,c2494=t,c2495=t,c2496=t,c2497=t,c2498=t,c2499=t,c2500=t," - "c2501=t,c2502=t,c2503=t,c2504=t,c2505=t,c2506=t,c2507=t,c2508=t,c2509=t,c2510=t,c2511=t,c2512=t,c2513=t,c2514=t," - "c2515=t,c2516=t,c2517=t,c2518=t,c2519=t,c2520=t,c2521=t,c2522=t,c2523=t,c2524=t,c2525=t,c2526=t,c2527=t,c2528=t," - "c2529=t,c2530=t,c2531=t,c2532=t,c2533=t,c2534=t,c2535=t,c2536=t,c2537=t,c2538=t,c2539=t,c2540=t,c2541=t,c2542=t," - "c2543=t,c2544=t,c2545=t,c2546=t,c2547=t,c2548=t,c2549=t,c2550=t,c2551=t,c2552=t,c2553=t,c2554=t,c2555=t,c2556=t," - "c2557=t,c2558=t,c2559=t,c2560=t,c2561=t,c2562=t,c2563=t,c2564=t,c2565=t,c2566=t,c2567=t,c2568=t,c2569=t,c2570=t," - "c2571=t,c2572=t,c2573=t,c2574=t,c2575=t,c2576=t,c2577=t," - "c2578=t,c2579=t,c2580=t,c2581=t,c2582=t,c2583=t,c2584=t,c2585=t,c2586=t,c2587=t,c2588=t,c2589=t,c2590=t,c2591=t," - "c2592=t,c2593=t,c2594=t,c2595=t,c2596=t,c2597=t,c2598=t,c2599=t,c2600=t,c2601=t,c2602=t,c2603=t,c2604=t,c2605=t," - "c2606=t,c2607=t,c2608=t,c2609=t,c2610=t,c2611=t,c2612=t,c2613=t,c2614=t,c2615=t,c2616=t,c2617=t,c2618=t,c2619=t," - "c2620=t,c2621=t,c2622=t,c2623=t,c2624=t,c2625=t,c2626=t,c2627=t,c2628=t,c2629=t,c2630=t,c2631=t,c2632=t,c2633=t," - "c2634=t,c2635=t,c2636=t,c2637=t,c2638=t,c2639=t,c2640=t,c2641=t,c2642=t,c2643=t,c2644=t,c2645=t,c2646=t,c2647=t," - "c2648=t,c2649=t,c2650=t,c2651=t,c2652=t,c2653=t,c2654=t,c2655=t,c2656=t,c2657=t,c2658=t,c2659=t,c2660=t,c2661=t," - "c2662=t,c2663=t,c2664=t,c2665=t,c2666=t,c2667=t,c2668=t,c2669=t,c2670=t,c2671=t,c2672=t,c2673=t,c2674=t,c2675=t," - "c2676=t,c2677=t,c2678=t,c2679=t,c2680=t,c2681=t,c2682=t,c2683=t,c2684=t,c2685=t,c2686=t,c2687=t,c2688=t,c2689=t," - "c2690=t,c2691=t,c2692=t,c2693=t,c2694=t,c2695=t,c2696=t,c2697=t,c2698=t,c2699=t,c2700=t,c2701=t,c2702=t,c2703=t," - "c2704=t,c2705=t,c2706=t,c2707=t,c2708=t,c2709=t,c2710=t,c2711=t,c2712=t,c2713=t,c2714=t,c2715=t,c2716=t,c2717=t," - "c2718=t,c2719=t,c2720=t,c2721=t,c2722=t,c2723=t,c2724=t," - "c2725=t,c2726=t,c2727=t,c2728=t,c2729=t,c2730=t,c2731=t,c2732=t,c2733=t,c2734=t,c2735=t,c2736=t,c2737=t,c2738=t," - "c2739=t,c2740=t,c2741=t,c2742=t,c2743=t,c2744=t,c2745=t,c2746=t,c2747=t,c2748=t,c2749=t,c2750=t,c2751=t,c2752=t," - "c2753=t,c2754=t,c2755=t,c2756=t,c2757=t,c2758=t,c2759=t,c2760=t,c2761=t,c2762=t,c2763=t,c2764=t,c2765=t,c2766=t," - "c2767=t,c2768=t,c2769=t,c2770=t,c2771=t,c2772=t,c2773=t,c2774=t,c2775=t,c2776=t,c2777=t,c2778=t,c2779=t,c2780=t," - "c2781=t,c2782=t,c2783=t,c2784=t,c2785=t,c2786=t,c2787=t,c2788=t,c2789=t,c2790=t,c2791=t,c2792=t,c2793=t,c2794=t," - "c2795=t,c2796=t,c2797=t,c2798=t,c2799=t,c2800=t,c2801=t,c2802=t,c2803=t,c2804=t,c2805=t,c2806=t,c2807=t,c2808=t," - "c2809=t,c2810=t,c2811=t,c2812=t,c2813=t,c2814=t,c2815=t,c2816=t,c2817=t,c2818=t,c2819=t,c2820=t,c2821=t,c2822=t," - "c2823=t,c2824=t,c2825=t,c2826=t,c2827=t,c2828=t,c2829=t,c2830=t,c2831=t,c2832=t,c2833=t,c2834=t,c2835=t,c2836=t," - "c2837=t,c2838=t,c2839=t,c2840=t,c2841=t,c2842=t,c2843=t,c2844=t,c2845=t,c2846=t,c2847=t,c2848=t,c2849=t,c2850=t," - "c2851=t,c2852=t,c2853=t,c2854=t,c2855=t,c2856=t,c2857=t,c2858=t,c2859=t,c2860=t,c2861=t,c2862=t,c2863=t,c2864=t," - "c2865=t,c2866=t,c2867=t,c2868=t,c2869=t,c2870=t,c2871=t," - "c2872=t,c2873=t,c2874=t,c2875=t,c2876=t,c2877=t,c2878=t,c2879=t,c2880=t,c2881=t,c2882=t,c2883=t,c2884=t,c2885=t," - "c2886=t,c2887=t,c2888=t,c2889=t,c2890=t,c2891=t,c2892=t,c2893=t,c2894=t,c2895=t,c2896=t,c2897=t,c2898=t,c2899=t," - "c2900=t,c2901=t,c2902=t,c2903=t,c2904=t,c2905=t,c2906=t,c2907=t,c2908=t,c2909=t,c2910=t,c2911=t,c2912=t,c2913=t," - "c2914=t,c2915=t,c2916=t,c2917=t,c2918=t,c2919=t,c2920=t,c2921=t,c2922=t,c2923=t,c2924=t,c2925=t,c2926=t,c2927=t," - "c2928=t,c2929=t,c2930=t,c2931=t,c2932=t,c2933=t,c2934=t,c2935=t,c2936=t,c2937=t,c2938=t,c2939=t,c2940=t,c2941=t," - "c2942=t,c2943=t,c2944=t,c2945=t,c2946=t,c2947=t,c2948=t,c2949=t,c2950=t,c2951=t,c2952=t,c2953=t,c2954=t,c2955=t," - "c2956=t,c2957=t,c2958=t,c2959=t,c2960=t,c2961=t,c2962=t,c2963=t,c2964=t,c2965=t,c2966=t,c2967=t,c2968=t,c2969=t," - "c2970=t,c2971=t,c2972=t,c2973=t,c2974=t,c2975=t,c2976=t,c2977=t,c2978=t,c2979=t,c2980=t,c2981=t,c2982=t,c2983=t," - "c2984=t,c2985=t,c2986=t,c2987=t,c2988=t,c2989=t,c2990=t,c2991=t,c2992=t,c2993=t,c2994=t,c2995=t,c2996=t,c2997=t," - "c2998=t,c2999=t,c3000=t,c3001=t,c3002=t,c3003=t,c3004=t,c3005=t,c3006=t,c3007=t,c3008=t,c3009=t,c3010=t,c3011=t," - "c3012=t,c3013=t,c3014=t,c3015=t,c3016=t,c3017=t,c3018=t," - "c3019=t,c3020=t,c3021=t,c3022=t,c3023=t,c3024=t,c3025=t,c3026=t,c3027=t,c3028=t,c3029=t,c3030=t,c3031=t,c3032=t," - "c3033=t,c3034=t,c3035=t,c3036=t,c3037=t,c3038=t,c3039=t,c3040=t,c3041=t,c3042=t,c3043=t,c3044=t,c3045=t,c3046=t," - "c3047=t,c3048=t,c3049=t,c3050=t,c3051=t,c3052=t,c3053=t,c3054=t,c3055=t,c3056=t,c3057=t,c3058=t,c3059=t,c3060=t," - "c3061=t,c3062=t,c3063=t,c3064=t,c3065=t,c3066=t,c3067=t,c3068=t,c3069=t,c3070=t,c3071=t,c3072=t,c3073=t,c3074=t," - "c3075=t,c3076=t,c3077=t,c3078=t,c3079=t,c3080=t,c3081=t,c3082=t,c3083=t,c3084=t,c3085=t,c3086=t,c3087=t,c3088=t," - "c3089=t,c3090=t,c3091=t,c3092=t,c3093=t,c3094=t,c3095=t,c3096=t,c3097=t,c3098=t,c3099=t,c3100=t,c3101=t,c3102=t," - "c3103=t,c3104=t,c3105=t,c3106=t,c3107=t,c3108=t,c3109=t,c3110=t,c3111=t,c3112=t,c3113=t,c3114=t,c3115=t,c3116=t," - "c3117=t,c3118=t,c3119=t,c3120=t,c3121=t,c3122=t,c3123=t,c3124=t,c3125=t,c3126=t,c3127=t,c3128=t,c3129=t,c3130=t," - "c3131=t,c3132=t,c3133=t,c3134=t,c3135=t,c3136=t,c3137=t,c3138=t,c3139=t,c3140=t,c3141=t,c3142=t,c3143=t,c3144=t," - "c3145=t,c3146=t,c3147=t,c3148=t,c3149=t,c3150=t,c3151=t,c3152=t,c3153=t,c3154=t,c3155=t,c3156=t,c3157=t,c3158=t," - "c3159=t,c3160=t,c3161=t,c3162=t,c3163=t,c3164=t,c3165=t," - "c3166=t,c3167=t,c3168=t,c3169=t,c3170=t,c3171=t,c3172=t,c3173=t,c3174=t,c3175=t,c3176=t,c3177=t,c3178=t,c3179=t," - "c3180=t,c3181=t,c3182=t,c3183=t,c3184=t,c3185=t,c3186=t,c3187=t,c3188=t,c3189=t,c3190=t,c3191=t,c3192=t,c3193=t," - "c3194=t,c3195=t,c3196=t,c3197=t,c3198=t,c3199=t,c3200=t,c3201=t,c3202=t,c3203=t,c3204=t,c3205=t,c3206=t,c3207=t," - "c3208=t,c3209=t,c3210=t,c3211=t,c3212=t,c3213=t,c3214=t,c3215=t,c3216=t,c3217=t,c3218=t,c3219=t,c3220=t,c3221=t," - "c3222=t,c3223=t,c3224=t,c3225=t,c3226=t,c3227=t,c3228=t,c3229=t,c3230=t,c3231=t,c3232=t,c3233=t,c3234=t,c3235=t," - "c3236=t,c3237=t,c3238=t,c3239=t,c3240=t,c3241=t,c3242=t,c3243=t,c3244=t,c3245=t,c3246=t,c3247=t,c3248=t,c3249=t," - "c3250=t,c3251=t,c3252=t,c3253=t,c3254=t,c3255=t,c3256=t,c3257=t,c3258=t,c3259=t,c3260=t,c3261=t,c3262=t,c3263=t," - "c3264=t,c3265=t,c3266=t,c3267=t,c3268=t,c3269=t,c3270=t,c3271=t,c3272=t,c3273=t,c3274=t,c3275=t,c3276=t,c3277=t," - "c3278=t,c3279=t,c3280=t,c3281=t,c3282=t,c3283=t,c3284=t,c3285=t,c3286=t,c3287=t,c3288=t,c3289=t,c3290=t,c3291=t," - "c3292=t,c3293=t,c3294=t,c3295=t,c3296=t,c3297=t,c3298=t,c3299=t,c3300=t,c3301=t,c3302=t,c3303=t,c3304=t,c3305=t," - "c3306=t,c3307=t,c3308=t,c3309=t,c3310=t,c3311=t,c3312=t," - "c3313=t,c3314=t,c3315=t,c3316=t,c3317=t,c3318=t,c3319=t,c3320=t,c3321=t,c3322=t,c3323=t,c3324=t,c3325=t,c3326=t," - "c3327=t,c3328=t,c3329=t,c3330=t,c3331=t,c3332=t,c3333=t,c3334=t,c3335=t,c3336=t,c3337=t,c3338=t,c3339=t,c3340=t," - "c3341=t,c3342=t,c3343=t,c3344=t,c3345=t,c3346=t,c3347=t,c3348=t,c3349=t,c3350=t,c3351=t,c3352=t,c3353=t,c3354=t," - "c3355=t,c3356=t,c3357=t,c3358=t,c3359=t,c3360=t,c3361=t,c3362=t,c3363=t,c3364=t,c3365=t,c3366=t,c3367=t,c3368=t," - "c3369=t,c3370=t,c3371=t,c3372=t,c3373=t,c3374=t,c3375=t,c3376=t,c3377=t,c3378=t,c3379=t,c3380=t,c3381=t,c3382=t," - "c3383=t,c3384=t,c3385=t,c3386=t,c3387=t,c3388=t,c3389=t,c3390=t,c3391=t,c3392=t,c3393=t,c3394=t,c3395=t,c3396=t," - "c3397=t,c3398=t,c3399=t,c3400=t,c3401=t,c3402=t,c3403=t,c3404=t,c3405=t,c3406=t,c3407=t,c3408=t,c3409=t,c3410=t," - "c3411=t,c3412=t,c3413=t,c3414=t,c3415=t,c3416=t,c3417=t,c3418=t,c3419=t,c3420=t,c3421=t,c3422=t,c3423=t,c3424=t," - "c3425=t,c3426=t,c3427=t,c3428=t,c3429=t,c3430=t,c3431=t,c3432=t,c3433=t,c3434=t,c3435=t,c3436=t,c3437=t,c3438=t," - "c3439=t,c3440=t,c3441=t,c3442=t,c3443=t,c3444=t,c3445=t,c3446=t,c3447=t,c3448=t,c3449=t,c3450=t,c3451=t,c3452=t," - "c3453=t,c3454=t,c3455=t,c3456=t,c3457=t,c3458=t,c3459=t," - "c3460=t,c3461=t,c3462=t,c3463=t,c3464=t,c3465=t,c3466=t,c3467=t,c3468=t,c3469=t,c3470=t,c3471=t,c3472=t,c3473=t," - "c3474=t,c3475=t,c3476=t,c3477=t,c3478=t,c3479=t,c3480=t,c3481=t,c3482=t,c3483=t,c3484=t,c3485=t,c3486=t,c3487=t," - "c3488=t,c3489=t,c3490=t,c3491=t,c3492=t,c3493=t,c3494=t,c3495=t,c3496=t,c3497=t,c3498=t,c3499=t,c3500=t,c3501=t," - "c3502=t,c3503=t,c3504=t,c3505=t,c3506=t,c3507=t,c3508=t,c3509=t,c3510=t,c3511=t,c3512=t,c3513=t," - "c3514=t,c3515=t,c3516=t,c3517=t,c3518=t,c3519=t,c3520=t,c3521=t,c3522=t,c3523=t,c3524=t,c3525=t,c3526=t,c3527=t," - "c3528=t,c3529=t,c3530=t,c3531=t,c3532=t,c3533=t,c3534=t,c3535=t,c3536=t,c3537=t,c3538=t,c3539=t,c3540=t,c3541=t," - "c3542=t,c3543=t,c3544=t,c3545=t,c3546=t,c3547=t,c3548=t,c3549=t,c3550=t,c3551=t,c3552=t,c3553=t,c3554=t,c3555=t," - "c3556=t,c3557=t,c3558=t,c3559=t,c3560=t,c3561=t,c3562=t,c3563=t,c3564=t,c3565=t,c3566=t,c3567=t,c3568=t,c3569=t," - "c3570=t,c3571=t,c3572=t,c3573=t,c3574=t,c3575=t,c3576=t,c3577=t,c3578=t,c3579=t,c3580=t,c3581=t,c3582=t,c3583=t," - "c3584=t,c3585=t,c3586=t,c3587=t,c3588=t,c3589=t,c3590=t,c3591=t,c3592=t,c3593=t,c3594=t,c3595=t,c3596=t,c3597=t," - "c3598=t,c3599=t,c3600=t,c3601=t,c3602=t,c3603=t,c3604=t,c3605=t,c3606=t,c3607=t,c3608=t,c3609=t,c3610=t,c3611=t," - "c3612=t,c3613=t,c3614=t,c3615=t,c3616=t,c3617=t,c3618=t,c3619=t,c3620=t,c3621=t,c3622=t,c3623=t,c3624=t,c3625=t," - "c3626=t,c3627=t,c3628=t,c3629=t,c3630=t,c3631=t,c3632=t,c3633=t,c3634=t,c3635=t,c3636=t,c3637=t,c3638=t,c3639=t," - "c3640=t,c3641=t,c3642=t,c3643=t,c3644=t,c3645=t,c3646=t,c3647=t,c3648=t,c3649=t,c3650=t,c3651=t,c3652=t,c3653=t," - "c3654=t,c3655=t,c3656=t,c3657=t,c3658=t,c3659=t,c3660=t," - "c3661=t,c3662=t,c3663=t,c3664=t,c3665=t,c3666=t,c3667=t,c3668=t,c3669=t,c3670=t,c3671=t,c3672=t,c3673=t,c3674=t," - "c3675=t,c3676=t,c3677=t,c3678=t,c3679=t,c3680=t,c3681=t,c3682=t,c3683=t,c3684=t,c3685=t,c3686=t,c3687=t,c3688=t," - "c3689=t,c3690=t,c3691=t,c3692=t,c3693=t,c3694=t,c3695=t,c3696=t,c3697=t,c3698=t,c3699=t,c3700=t,c3701=t,c3702=t," - "c3703=t,c3704=t,c3705=t,c3706=t,c3707=t,c3708=t,c3709=t,c3710=t,c3711=t,c3712=t,c3713=t,c3714=t,c3715=t,c3716=t," - "c3717=t,c3718=t,c3719=t,c3720=t,c3721=t,c3722=t,c3723=t,c3724=t,c3725=t,c3726=t,c3727=t,c3728=t,c3729=t,c3730=t," - "c3731=t,c3732=t,c3733=t,c3734=t,c3735=t,c3736=t,c3737=t,c3738=t,c3739=t,c3740=t,c3741=t,c3742=t,c3743=t,c3744=t," - "c3745=t,c3746=t,c3747=t,c3748=t,c3749=t,c3750=t,c3751=t,c3752=t,c3753=t,c3754=t,c3755=t,c3756=t,c3757=t,c3758=t," - "c3759=t,c3760=t,c3761=t,c3762=t,c3763=t,c3764=t,c3765=t,c3766=t,c3767=t,c3768=t,c3769=t,c3770=t,c3771=t,c3772=t," - "c3773=t,c3774=t,c3775=t,c3776=t,c3777=t,c3778=t,c3779=t,c3780=t,c3781=t,c3782=t,c3783=t,c3784=t,c3785=t,c3786=t," - "c3787=t,c3788=t,c3789=t,c3790=t,c3791=t,c3792=t,c3793=t,c3794=t,c3795=t,c3796=t,c3797=t,c3798=t,c3799=t,c3800=t," - "c3801=t,c3802=t,c3803=t,c3804=t,c3805=t,c3806=t,c3807=t," - "c3808=t,c3809=t,c3810=t,c3811=t,c3812=t,c3813=t,c3814=t,c3815=t,c3816=t,c3817=t,c3818=t,c3819=t,c3820=t,c3821=t," - "c3822=t,c3823=t,c3824=t,c3825=t,c3826=t,c3827=t,c3828=t,c3829=t,c3830=t,c3831=t,c3832=t,c3833=t,c3834=t,c3835=t," - "c3836=t,c3837=t,c3838=t,c3839=t,c3840=t,c3841=t,c3842=t,c3843=t,c3844=t,c3845=t,c3846=t,c3847=t,c3848=t,c3849=t," - "c3850=t,c3851=t,c3852=t,c3853=t,c3854=t,c3855=t,c3856=t,c3857=t,c3858=t,c3859=t,c3860=t,c3861=t,c3862=t,c3863=t," - "c3864=t,c3865=t,c3866=t,c3867=t,c3868=t,c3869=t,c3870=t,c3871=t,c3872=t,c3873=t,c3874=t,c3875=t,c3876=t,c3877=t," - "c3878=t,c3879=t,c3880=t,c3881=t,c3882=t,c3883=t,c3884=t,c3885=t,c3886=t,c3887=t,c3888=t,c3889=t,c3890=t,c3891=t," - "c3892=t,c3893=t,c3894=t,c3895=t,c3896=t,c3897=t,c3898=t,c3899=t,c3900=t,c3901=t,c3902=t,c3903=t,c3904=t,c3905=t," - "c3906=t,c3907=t,c3908=t,c3909=t,c3910=t,c3911=t,c3912=t,c3913=t,c3914=t,c3915=t,c3916=t,c3917=t,c3918=t,c3919=t," - "c3920=t,c3921=t,c3922=t,c3923=t,c3924=t,c3925=t,c3926=t,c3927=t,c3928=t,c3929=t,c3930=t,c3931=t,c3932=t,c3933=t," - "c3934=t,c3935=t,c3936=t,c3937=t,c3938=t,c3939=t,c3940=t,c3941=t,c3942=t,c3943=t,c3944=t,c3945=t,c3946=t,c3947=t," - "c3948=t,c3949=t,c3950=t,c3951=t,c3952=t,c3953=t,c3954=t," - "c3955=t,c3956=t,c3957=t,c3958=t,c3959=t,c3960=t,c3961=t,c3962=t,c3963=t,c3964=t,c3965=t,c3966=t,c3967=t,c3968=t," - "c3969=t,c3970=t,c3971=t,c3972=t,c3973=t,c3974=t,c3975=t,c3976=t,c3977=t,c3978=t,c3979=t,c3980=t,c3981=t,c3982=t," - "c3983=t,c3984=t,c3985=t,c3986=t,c3987=t,c3988=t,c3989=t,c3990=t,c3991=t,c3992=t,c3993=t,c3994=t,c3995=t,c3996=t," - "c3997=t,c3998=t,c3999=t,c4000=t,c4001=t,c4002=t,c4003=t,c4004=t,c4005=t,c4006=t,c4007=t,c4008=t,c4009=t,c4010=t," - "c4011=t,c4012=t,c4013=t,c4014=t,c4015=t,c4016=t,c4017=t,c4018=t,c4019=t,c4020=t,c4021=t,c4022=t,c4023=t,c4024=t," - "c4025=t,c4026=t,c4027=t,c4028=t,c4029=t,c4030=t,c4031=t,c4032=t,c4033=t,c4034=t,c4035=t,c4036=t,c4037=t,c4038=t," - "c4039=t,c4040=t,c4041=t,c4042=t,c4043=t,c4044=t,c4045=t,c4046=t,c4047=t,c4048=t,c4049=t,c4050=t,c4051=t,c4052=t," - "c4053=t,c4054=t,c4055=t,c4056=t,c4057=t,c4058=t,c4059=t,c4060=t,c4061=t,c4062=t,c4063=t,c4064=t,c4065=t,c4066=t," - "c4067=t,c4068=t,c4069=t,c4070=t,c4071=t,c4072=t,c4073=t,c4074=t,c4075=t,c4076=t,c4077=t,c4078=t,c4079=t,c4080=t," - "c4081=t,c4082=t,c4083=t,c4084=t,c4085=t,c4086=t,c4087=t,c4088=t,c4089=t,c4090=t,c4091=t,c4092=t,c4093=t " - "1626006833640000000"}; - - int ret = TSDB_CODE_SUCCESS; - for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseInfluxLine(info, sql[i], strlen(sql[i])); - if (ret != TSDB_CODE_SUCCESS) break; +TEST(testCase, smlParseNumber_performance_Test) { + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + SSmlKv kv; + + char* str[3] = {"2893f64", "2323u32", "93u8"}; + for (int i = 0; i < 3; ++i) { + int64_t t1 = taosGetTimestampUs(); + for (int j = 0; j < 10000000; ++j) { + kv.value = str[i]; + kv.length = strlen(str[i]); + smlParseNumber(&kv, &msgBuf); + } + printf("smlParseNumber:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t1); + printf("\n"); + int64_t t2 = taosGetTimestampUs(); + for (int j = 0; j < 10000000; ++j) { + kv.value = str[i]; + kv.length = strlen(str[i]); + smlParseNumberOld(&kv, &msgBuf); + } + printf("smlParseNumberOld:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t2); + printf("\n\n"); } - ASSERT_NE(ret, 0); - smlDestroyInfo(info); -} +} \ No newline at end of file diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 2b43229b838fffb14625f01ba98bfc5c196a8458..4373f309ca39e1f3f42908db1cfc3a1b11c41c08 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -19,7 +19,7 @@ #include "tlog.h" #include "tname.h" -#define MALLOC_ALIGN_BYTES 256 +#define MALLOC_ALIGN_BYTES 32 int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { ASSERT(pColumnInfoData != NULL); @@ -38,7 +38,8 @@ int32_t colDataGetFullLength(const SColumnInfoData* pColumnInfoData, int32_t num if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { return pColumnInfoData->varmeta.length + sizeof(int32_t) * numOfRows; } else { - return ((pColumnInfoData->info.type == TSDB_DATA_TYPE_NULL) ? 0 : pColumnInfoData->info.bytes * numOfRows) + BitmapLen(numOfRows); + return ((pColumnInfoData->info.type == TSDB_DATA_TYPE_NULL) ? 0 : pColumnInfoData->info.bytes * numOfRows) + + BitmapLen(numOfRows); } } @@ -279,7 +280,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int pColumnInfoData->varmeta.allocLen = len + oldLen; } - if (pColumnInfoData->pData && pSource->pData) { // TD-20382 + if (pColumnInfoData->pData && pSource->pData) { // TD-20382 memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len); } pColumnInfoData->varmeta.length = len + oldLen; @@ -359,7 +360,7 @@ size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.ro int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex) { if (pDataBlock->info.rows > 0) { -// ASSERT(pDataBlock->info.dataLoad == 1); + // ASSERT(pDataBlock->info.dataLoad == 1); } if (pDataBlock == NULL || pDataBlock->info.rows <= 0 || pDataBlock->info.dataLoad == 0) { @@ -1167,8 +1168,11 @@ void blockDataEmpty(SSDataBlock* pDataBlock) { } // todo temporarily disable it -static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows, bool clearPayload) { + +static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows, + bool clearPayload) { ASSERT(numOfRows > 0); + if (numOfRows <= pBlockInfo->capacity) { return TSDB_CODE_SUCCESS; } @@ -1225,7 +1229,7 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* return TSDB_CODE_SUCCESS; } -void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) { +void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) { pColumn->hasNull = false; if (IS_VAR_DATA_TYPE(pColumn->info.type)) { @@ -1953,7 +1957,8 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) "===stream===%s|block type %d|child id %d|group id:%" PRIu64 "|uid:%" PRId64 "|rows:%d|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.id.groupId, - pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version, pDataBlock->info.calWin.skey, pDataBlock->info.calWin.ekey); + pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version, + pDataBlock->info.calWin.skey, pDataBlock->info.calWin.ekey); if (len >= size - 1) return dumpBuf; for (int32_t j = 0; j < rows; j++) { @@ -2055,6 +2060,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) * @param suid * */ +#if 0 int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataBlock, STSchema* pTSchema, int32_t vgId, tb_uid_t suid) { int32_t bufSize = sizeof(SSubmitReq); @@ -2220,32 +2226,186 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB return TSDB_CODE_SUCCESS; } +#endif + +int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDataBlock, const STSchema* pTSchema, + int64_t uid, int32_t vgId, tb_uid_t suid) { + SSubmitReq2* pReq = *ppReq; + SArray* pVals = NULL; + int32_t numOfBlks = 0; + int32_t sz = 1; + + terrno = TSDB_CODE_SUCCESS; + + if (NULL == pReq) { + if (!(pReq = taosMemoryMalloc(sizeof(SSubmitReq2)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { + goto _end; + } + } + + for (int32_t i = 0; i < sz; ++i) { + int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); + int32_t rows = pDataBlock->info.rows; + + if (colNum <= 1) { // invalid if only with TS col + continue; + } + + // the rsma result should has the same column number with schema. + ASSERT(colNum == pTSchema->numOfCols); + + SSubmitTbData tbData = {0}; + + if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow*)))) { + goto _end; + } + tbData.suid = suid; + tbData.uid = uid; + tbData.sver = pTSchema->version; + + if (!pVals && !(pVals = taosArrayInit(colNum, sizeof(SColVal)))) { + taosArrayDestroy(tbData.aRowP); + goto _end; + } + + for (int32_t j = 0; j < rows; ++j) { // iterate by row + + taosArrayClear(pVals); + + bool isStartKey = false; + int32_t offset = 0; + for (int32_t k = 0; k < colNum; ++k) { // iterate by column + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + const STColumn* pCol = &pTSchema->columns[k]; + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + ASSERT(pColInfoData->info.type == pCol->type); + if (!isStartKey) { + isStartKey = true; + ASSERT(PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId); + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, (SValue){.val = *(TSKEY*)var}); + taosArrayPush(pVals, &cv); + } else if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, (SValue){.val = *(int64_t*)var}); + taosArrayPush(pVals, &cv); + } + break; + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY + ASSERT(pColInfoData->info.type == pCol->type); + if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + void* data = colDataGetVarData(pColInfoData, j); + SValue sv = (SValue){.nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } + break; + } + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_MEDIUMBLOB: + uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); + ASSERT(0); + break; + default: + if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { + if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); // should use pCol->type + taosArrayPush(pVals, &cv); + } else { + SValue sv; + if (pCol->type == pColInfoData->info.type) { + memcpy(&sv.val, var, tDataTypes[pCol->type].bytes); + } else { + /** + * 1. sum/avg would convert to int64_t/uint64_t/double during aggregation + * 2. below conversion may lead to overflow or loss, the app should select the right data type. + */ + char tv[8] = {0}; + if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) { + float v = 0; + GET_TYPED_DATA(v, float, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) { + double v = 0; + GET_TYPED_DATA(v, double, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) { + int64_t v = 0; + GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else { + uint64_t v = 0; + GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } + memcpy(&sv.val, tv, tDataTypes[pCol->type].bytes); + } + SColVal cv = COL_VAL_VALUE(pCol->colId, pColInfoData->info.type, sv); + taosArrayPush(pVals, &cv); + } + } else { + uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); + ASSERT(0); + } + break; + } + } + SRow* pRow = NULL; + if ((terrno = tRowBuild(pVals, pTSchema, &pRow)) < 0) { + tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + goto _end; + } + ASSERT(pRow); + taosArrayPush(tbData.aRowP, &pRow); + } + + taosArrayPush(pReq->aSubmitTbData, &tbData); + } +_end: + taosArrayDestroy(pVals); + if (terrno != 0) { + *ppReq = NULL; + if (pReq) tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + return TSDB_CODE_FAILED; + } + *ppReq = pReq; + return TSDB_CODE_SUCCESS; +} char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { ASSERT(stbFullName[0] != 0); - SArray* tags = taosArrayInit(0, sizeof(void*)); + SArray* tags = taosArrayInit(0, sizeof(SSmlKv)); if (tags == NULL) { return NULL; } - SSmlKv* pTag = taosMemoryCalloc(1, sizeof(SSmlKv)); - if (pTag == NULL) { - taosArrayDestroy(tags); - return NULL; - } - void* cname = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1); if (cname == NULL) { taosArrayDestroy(tags); - taosMemoryFree(pTag); return NULL; } - pTag->key = "group_id"; - pTag->keyLen = strlen(pTag->key); - pTag->type = TSDB_DATA_TYPE_UBIGINT; - pTag->u = groupId; - pTag->length = sizeof(uint64_t); + SSmlKv pTag = {.key = "group_id", + .keyLen = sizeof("group_id") - 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .u = groupId, + .length = sizeof(uint64_t)}; taosArrayPush(tags, &pTag); RandTableName rname = { @@ -2257,7 +2417,6 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { buildChildTableName(&rname); - taosMemoryFree(pTag); taosArrayDestroy(tags); ASSERT(rname.ctbShortName && rname.ctbShortName[0]); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index fc49c033794c946a9881b6ae1efeabcb101e99a7..5d19012273aca3bf48ba37159e016290947fc49f 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -20,6 +20,8 @@ #include "tdatablock.h" #include "tlog.h" +static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData); + // SBuffer ================================ void tBufferDestroy(SBuffer *pBuffer) { tFree(pBuffer->pBuf); @@ -61,9 +63,9 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); #define KV_FLG_MID ((uint8_t)0x20) #define KV_FLG_BIG ((uint8_t)0x30) -#define ROW_BIT_NONE ((uint8_t)0x0) -#define ROW_BIT_NULL ((uint8_t)0x1) -#define ROW_BIT_VALUE ((uint8_t)0x2) +#define BIT_FLG_NONE ((uint8_t)0x0) +#define BIT_FLG_NULL ((uint8_t)0x1) +#define BIT_FLG_VALUE ((uint8_t)0x2) #pragma pack(push, 1) typedef struct { @@ -95,28 +97,29 @@ typedef struct { } \ } while (0) -int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { +int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) { int32_t code = 0; - ASSERT(taosArrayGetSize(aColVal) > 0); + ASSERT(TARRAY_SIZE(aColVal) > 0); ASSERT(((SColVal *)aColVal->pData)[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); ASSERT(((SColVal *)aColVal->pData)[0].type == TSDB_DATA_TYPE_TIMESTAMP); // scan --------------- - uint8_t flag = 0; - int32_t iColVal = 1; - const int32_t nColVal = taosArrayGetSize(aColVal); - SColVal *pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - int32_t iTColumn = 1; - STColumn *pTColumn = pTSchema->columns + iTColumn; - int32_t ntp = 0; - int32_t nkv = 0; - int32_t maxIdx = 0; - int32_t nIdx = 0; + SRow *pRow = NULL; + SColVal *colVals = (SColVal *)TARRAY_DATA(aColVal); + uint8_t flag = 0; + int32_t iColVal = 1; + const int32_t nColVal = TARRAY_SIZE(aColVal); + SColVal *pColVal = (iColVal < nColVal) ? &colVals[iColVal] : NULL; + int32_t iTColumn = 1; + const STColumn *pTColumn = pTSchema->columns + iTColumn; + int32_t ntp = 0; + int32_t nkv = 0; + int32_t maxIdx = 0; + int32_t nIdx = 0; while (pTColumn) { if (pColVal) { if (pColVal->cid == pTColumn->colId) { - ntp += TYPE_BYTES[pTColumn->type]; if (COL_VAL_IS_VALUE(pColVal)) { // VALUE flag |= HAS_VALUE; maxIdx = nkv; @@ -139,17 +142,15 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } else if (pColVal->cid > pTColumn->colId) { // NONE flag |= HAS_NONE; - ntp += TYPE_BYTES[pTColumn->type]; pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } } else { // NONE flag |= HAS_NONE; - ntp += TYPE_BYTES[pTColumn->type]; pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } } @@ -161,17 +162,17 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { ntp = sizeof(SRow); break; case HAS_VALUE: - ntp = sizeof(SRow) + ntp; + ntp = sizeof(SRow) + pTSchema->flen + ntp; break; case (HAS_NULL | HAS_NONE): ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1); break; case (HAS_VALUE | HAS_NONE): case (HAS_VALUE | HAS_NULL): - ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1) + ntp; + ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntp; break; case (HAS_VALUE | HAS_NULL | HAS_NONE): - ntp = sizeof(SRow) + BIT2_SIZE(pTSchema->numOfCols - 1) + ntp; + ntp = sizeof(SRow) + BIT2_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntp; break; default: ASSERT(0); @@ -196,12 +197,14 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } // alloc -------------- - SRow *pRow = NULL; - code = tBufferReserve(pBuffer, nRow, (void **)&pRow); - if (code) return code; + pRow = taosMemoryMalloc(nRow); + if (NULL == pRow) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } // build -------------- - pColVal = (SColVal *)taosArrayGet(aColVal, 0); + pColVal = &colVals[0]; pRow->flag = flag; pRow->rsv = 0; @@ -214,10 +217,10 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } iColVal = 1; - pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (iColVal < nColVal) ? &colVals[iColVal] : NULL; iTColumn = 1; pTColumn = pTSchema->columns + iTColumn; - if (flag & 0xf0) { // KV + if (flag >> 4) { // KV SKVIdx *pIdx = (SKVIdx *)pRow->data; int32_t iIdx = 0; int32_t nv = 0; @@ -266,11 +269,11 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } else if (pColVal->cid > pTColumn->colId) { // NONE pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } } else { // NONE pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; @@ -306,12 +309,20 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { break; } + if (pb) { + if (flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) { + memset(pb, 0, BIT2_SIZE(pTSchema->numOfCols - 1)); + } else { + memset(pb, 0, BIT1_SIZE(pTSchema->numOfCols - 1)); + } + } + // build impl while (pTColumn) { if (pColVal) { if (pColVal->cid == pTColumn->colId) { if (COL_VAL_IS_VALUE(pColVal)) { // VALUE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_VALUE); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_VALUE); if (IS_VAR_DATA_TYPE(pTColumn->type)) { *(int32_t *)(pf + pTColumn->offset) = nv; @@ -324,24 +335,24 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { memcpy(pf + pTColumn->offset, &pColVal->value.val, TYPE_BYTES[pTColumn->type]); } } else if (COL_VAL_IS_NONE(pColVal)) { // NONE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NONE); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); } else { // NULL - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NULL); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NULL); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); } pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } else if (pColVal->cid > pTColumn->colId) { // NONE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NONE); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } } else { // NONE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NONE); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } @@ -349,6 +360,12 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } _exit: + if (code) { + *ppRow = NULL; + tRowDestroy(pRow); + } else { + *ppRow = pRow; + } return code; } @@ -376,7 +393,7 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { return; } - if (pRow->flag & 0xf0) { // KV Row + if (pRow->flag >> 4) { // KV Row SKVIdx *pIdx = (SKVIdx *)pRow->data; uint8_t *pv = NULL; if (pRow->flag & KV_FLG_LIT) { @@ -432,64 +449,190 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); } else { // Tuple Row - uint8_t *pf = NULL; - uint8_t *pv = NULL; - uint8_t bv = ROW_BIT_VALUE; + if (pRow->flag == HAS_VALUE) { + pColVal->cid = pTColumn->colId; + pColVal->type = pTColumn->type; + pColVal->flag = CV_FLAG_VALUE; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + uint8_t *pData = pRow->data + pTSchema->flen + *(int32_t *)(pRow->data + pTColumn->offset); + pData += tGetU32v(pData, &pColVal->value.nData); + if (pColVal->value.nData) { + pColVal->value.pData = pData; + } else { + pColVal->value.pData = NULL; + } + } else { + memcpy(&pColVal->value.val, pRow->data + pTColumn->offset, TYPE_BYTES[pTColumn->type]); + } + } else { + uint8_t *pf; + uint8_t *pv; + uint8_t bv = BIT_FLG_VALUE; - switch (pRow->flag) { - case HAS_VALUE: - pf = pRow->data; - pv = pf + pTSchema->flen; - break; - case (HAS_NULL | HAS_NONE): - bv = GET_BIT1(pRow->data, iCol - 1); - break; - case (HAS_VALUE | HAS_NONE): - bv = GET_BIT1(pRow->data, iCol - 1); - if (bv) bv++; - pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; - break; - case (HAS_VALUE | HAS_NULL): - bv = GET_BIT1(pRow->data, iCol - 1); - bv++; - pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; - break; - case (HAS_VALUE | HAS_NULL | HAS_NONE): - bv = GET_BIT2(pRow->data, iCol - 1); - pf = pRow->data + BIT2_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; - break; - default: - break; - } + switch (pRow->flag) { + case (HAS_NULL | HAS_NONE): + bv = GET_BIT1(pRow->data, iCol - 1); + break; + case (HAS_VALUE | HAS_NONE): + bv = GET_BIT1(pRow->data, iCol - 1); + if (bv) bv++; + pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL): + bv = GET_BIT1(pRow->data, iCol - 1); + bv++; + pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + bv = GET_BIT2(pRow->data, iCol - 1); + pf = pRow->data + BIT2_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + default: + ASSERT(0); + break; + } - if (bv == ROW_BIT_NONE) { - *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); - return; - } else if (bv == ROW_BIT_NULL) { - *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); - return; - } + if (bv == BIT_FLG_NONE) { + *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + return; + } else if (bv == BIT_FLG_NULL) { + *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); + return; + } - pColVal->cid = pTColumn->colId; - pColVal->type = pTColumn->type; - pColVal->flag = CV_FLAG_VALUE; - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset); - pData += tGetU32v(pData, &pColVal->value.nData); - if (pColVal->value.nData) { - pColVal->value.pData = pData; + pColVal->cid = pTColumn->colId; + pColVal->type = pTColumn->type; + pColVal->flag = CV_FLAG_VALUE; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset); + pData += tGetU32v(pData, &pColVal->value.nData); + if (pColVal->value.nData) { + pColVal->value.pData = pData; + } else { + pColVal->value.pData = NULL; + } } else { - pColVal->value.pData = NULL; + memcpy(&pColVal->value.val, pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]); } - } else { - memcpy(&pColVal->value.val, pv + pTColumn->offset, pTColumn->bytes); } } } +void tRowDestroy(SRow *pRow) { + if (pRow) taosMemoryFree(pRow); +} + +static int32_t tRowPCmprFn(const void *p1, const void *p2) { + if ((*(SRow **)p1)->ts < (*(SRow **)p2)->ts) { + return -1; + } else if ((*(SRow **)p1)->ts > (*(SRow **)p2)->ts) { + return 1; + } + + return 0; +} +static void tRowPDestroy(SRow **ppRow) { tRowDestroy(*ppRow); } +static int32_t tRowMergeImpl(SArray *aRowP, STSchema *pTSchema, int32_t iStart, int32_t iEnd, int8_t flag) { + int32_t code = 0; + + int32_t nRow = iEnd - iStart; + SRowIter **aIter = NULL; + SArray *aColVal = NULL; + SRow *pRow = NULL; + + aIter = taosMemoryCalloc(nRow, sizeof(SRowIter *)); + if (aIter == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t i = 0; i < nRow; i++) { + SRow *pRowT = taosArrayGetP(aRowP, iStart + i); + + code = tRowIterOpen(pRowT, pTSchema, &aIter[i]); + if (code) goto _exit; + } + + // merge + aColVal = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)); + if (aColVal == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t iCol = 0; iCol < pTSchema->numOfCols; iCol++) { + SColVal *pColVal = NULL; + for (int32_t iRow = 0; iRow < nRow; iRow++) { + SColVal *pColValT = tRowIterNext(aIter[iRow]); + + // todo: take strategy according to the flag + if (COL_VAL_IS_VALUE(pColValT)) { + pColVal = pColValT; + } else if (COL_VAL_IS_NULL(pColValT)) { + if (pColVal == NULL) { + pColVal = pColValT; + } + } + } + + if (pColVal) taosArrayPush(aColVal, pColVal); + } + + // build + code = tRowBuild(aColVal, pTSchema, &pRow); + if (code) goto _exit; + + taosArrayRemoveBatch(aRowP, iStart, nRow, (FDelete)tRowPDestroy); + taosArrayInsert(aRowP, iStart, &pRow); + +_exit: + if (aIter) { + for (int32_t i = 0; i < nRow; i++) { + tRowIterClose(&aIter[i]); + } + taosMemoryFree(aIter); + } + if (aColVal) taosArrayDestroy(aColVal); + if (code) tRowDestroy(pRow); + return code; +} + +void tRowSort(SArray *aRowP) { + if (TARRAY_SIZE(aRowP) <= 1) return; + taosArraySort(aRowP, tRowPCmprFn); +} + +int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag) { + int32_t code = 0; + + int32_t iStart = 0; + while (iStart < aRowP->size) { + SRow *pRow = (SRow *)taosArrayGetP(aRowP, iStart); + + int32_t iEnd = iStart + 1; + while (iEnd < aRowP->size) { + SRow *pRowT = (SRow *)taosArrayGetP(aRowP, iEnd); + + if (pRow->ts != pRowT->ts) break; + + iEnd++; + } + + if (iEnd - iStart > 1) { + code = tRowMergeImpl(aRowP, pTSchema, iStart, iEnd, flag); + if (code) return code; + } + + // the array is also changing, so the iStart just ++ instead of iEnd + iStart++; + } + + return code; +} + // SRowIter ======================================== struct SRowIter { SRow *pRow; @@ -505,10 +648,9 @@ struct SRowIter { uint8_t *pb; uint8_t *pf; }; - uint8_t *pv; }; - - SColVal cv; + uint8_t *pv; + SColVal cv; }; int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { @@ -528,15 +670,15 @@ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { if (pRow->flag == HAS_NONE || pRow->flag == HAS_NULL) goto _exit; - if (pRow->flag & 0xf0) { + if (pRow->flag >> 4) { pIter->iCol = 0; pIter->pIdx = (SKVIdx *)pRow->data; if (pRow->flag & KV_FLG_LIT) { pIter->pv = pIter->pIdx->idx + pIter->pIdx->nCol; } else if (pRow->flag & KV_FLG_MID) { - pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 1); + pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 1); // * sizeof(uint16_t) } else { - pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 2); + pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 2); // * sizeof(uint32_t) } } else { switch (pRow->flag) { @@ -608,7 +750,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { goto _exit; } - if (pIter->pRow->flag & 0xf0) { // KV + if (pIter->pRow->flag >> 4) { // KV if (pIter->iCol < pIter->pIdx->nCol) { uint8_t *pData; @@ -656,7 +798,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { goto _exit; } } else { // Tuple - uint8_t bv = ROW_BIT_VALUE; + uint8_t bv = BIT_FLG_VALUE; if (pIter->pb) { switch (pIter->pRow->flag) { case (HAS_NULL | HAS_NONE): @@ -677,10 +819,10 @@ SColVal *tRowIterNext(SRowIter *pIter) { break; } - if (bv == ROW_BIT_NONE) { + if (bv == BIT_FLG_NONE) { pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); goto _exit; - } else if (bv == ROW_BIT_NULL) { + } else if (bv == BIT_FLG_NULL) { pIter->cv = COL_VAL_NULL(pTColumn->colId, pTColumn->type); goto _exit; } @@ -698,7 +840,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { pIter->cv.value.pData = NULL; } } else { - memcpy(&pIter->cv.value.val, pIter->pv + pTColumn->offset, pTColumn->bytes); + memcpy(&pIter->cv.value.val, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]); } goto _exit; } @@ -708,7 +850,268 @@ _exit: return &pIter->cv; } -// STSchema ======================================== +static int32_t tRowAppendNoneToColData(SColData *aColData, int32_t nColData) { + int32_t code = 0; + + for (int32_t iColData = 0; iColData < nColData; iColData++) { + SColData *pColData = &aColData[iColData]; + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + } + +_exit: + return code; +} +static int32_t tRowAppendNullToColData(SColData *aColData, int32_t nColData, STSchema *pSchema) { + int32_t code = 0; + + int32_t iColData = 0; + SColData *pColData = &aColData[iColData]; + int32_t iTColumn = 1; + STColumn *pTColumn = &pSchema->columns[iTColumn]; + + while (pColData) { + if (pTColumn) { + if (pTColumn->colId == pColData->cid) { // NULL + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + pTColumn = (++iTColumn < pSchema->numOfCols) ? &pSchema->columns[iTColumn] : NULL; + } else if (pTColumn->colId > pColData->cid) { // NONE + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } else { + pTColumn = (++iTColumn < pSchema->numOfCols) ? &pSchema->columns[iTColumn] : NULL; + } + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } + } + +_exit: + return code; +} +static int32_t tRowAppendTupleToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData) { + int32_t code = 0; + + int32_t iColData = 0; + SColData *pColData = &aColData[iColData]; + int32_t iTColumn = 1; + STColumn *pTColumn = &pTSchema->columns[iTColumn]; + + uint8_t *pb = NULL; + uint8_t *pf; + uint8_t *pv; + + switch (pRow->flag) { + case HAS_VALUE: + pf = pRow->data; + pv = pf + pTSchema->flen; + break; + case (HAS_NULL | HAS_NONE): + pb = pRow->data; + break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + pb = pRow->data; + pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + pb = pRow->data; + pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + default: + ASSERT(0); + break; + } + + while (pColData) { + if (pTColumn) { + if (pTColumn->colId == pColData->cid) { + ASSERT(pTColumn->type == pColData->type); + if (pb) { + uint8_t bv; + switch (pRow->flag) { + case (HAS_NULL | HAS_NONE): + bv = GET_BIT1(pb, iTColumn - 1); + break; + case (HAS_VALUE | HAS_NONE): + bv = GET_BIT1(pb, iTColumn - 1); + if (bv) bv++; + break; + case (HAS_VALUE | HAS_NULL): + bv = GET_BIT1(pb, iTColumn - 1) + 1; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + bv = GET_BIT2(pb, iTColumn - 1); + break; + default: + ASSERT(0); + break; + } + + if (bv == BIT_FLG_NONE) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + goto _continue; + } else if (bv == BIT_FLG_NULL) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + goto _continue; + } + } + + if (IS_VAR_DATA_TYPE(pColData->type)) { + uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset); + uint32_t nData; + pData += tGetU32v(pData, &nData); + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pf + pTColumn->offset, + TYPE_BYTES[pColData->type]); + if (code) goto _exit; + } + + _continue: + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + } else if (pTColumn->colId > pColData->cid) { // NONE + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } else { + pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + } + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } + } + +_exit: + return code; +} +static int32_t tRowAppendKVToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData) { + int32_t code = 0; + + SKVIdx *pKVIdx = (SKVIdx *)pRow->data; + uint8_t *pv = NULL; + int32_t iColData = 0; + SColData *pColData = &aColData[iColData]; + int32_t iTColumn = 1; + STColumn *pTColumn = &pTSchema->columns[iTColumn]; + int32_t iCol = 0; + + if (pRow->flag & KV_FLG_LIT) { + pv = pKVIdx->idx + pKVIdx->nCol; + } else if (pRow->flag & KV_FLG_MID) { + pv = pKVIdx->idx + (pKVIdx->nCol << 1); + } else if (pRow->flag & KV_FLG_BIG) { + pv = pKVIdx->idx + (pKVIdx->nCol << 2); + } else { + ASSERT(0); + } + + while (pColData) { + if (pTColumn) { + if (pTColumn->colId == pColData->cid) { + while (iCol < pKVIdx->nCol) { + uint8_t *pData; + if (pRow->flag & KV_FLG_LIT) { + pData = pv + ((uint8_t *)pKVIdx->idx)[iCol]; + } else if (pRow->flag & KV_FLG_MID) { + pData = pv + ((uint16_t *)pKVIdx->idx)[iCol]; + } else if (pRow->flag & KV_FLG_BIG) { + pData = pv + ((uint32_t *)pKVIdx->idx)[iCol]; + } else { + ASSERT(0); + } + + int16_t cid; + pData += tGetI16v(pData, &cid); + + if (TABS(cid) == pTColumn->colId) { + if (cid < 0) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } else { + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + uint32_t nData; + pData += tGetU32v(pData, &nData); + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, 0); + if (code) goto _exit; + } + } + iCol++; + goto _continue; + } else if (TABS(cid) > pTColumn->colId) { // NONE + break; + } else { + iCol++; + } + } + + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + + _continue: + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + } else if (pTColumn->colId > pColData->cid) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } else { + pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + } + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } + } + +_exit: + return code; +} +int32_t tRowAppendToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData) { + ASSERT(pRow->sver == pTSchema->version); + ASSERT(nColData > 0); + + int32_t code = 0; + + if (pRow->flag == HAS_NONE) { + code = tRowAppendNoneToColData(aColData, nColData); + goto _exit; + } else if (pRow->flag == HAS_NULL) { + code = tRowAppendNullToColData(aColData, nColData, pTSchema); + goto _exit; + } + + if (pRow->flag >> 4) { // KV row + code = tRowAppendKVToColData(pRow, pTSchema, aColData, nColData); + if (code) goto _exit; + } else { + code = tRowAppendTupleToColData(pRow, pTSchema, aColData, nColData); + if (code) goto _exit; + } + +_exit: + return code; +} // STag ======================================== static int tTagValCmprFn(const void *p1, const void *p2) { @@ -1086,120 +1489,47 @@ void tTagSetCid(const STag *pTag, int16_t iTag, int16_t cid) { tPutI16v(p + offset, cid); } -#if 1 // =================================================================================================================== -int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) { - if (pBuilder == NULL) return -1; +// STSchema ======================================== +STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version) { + STSchema *pTSchema = taosMemoryCalloc(1, sizeof(STSchema) + sizeof(STColumn) * numOfCols); + if (pTSchema == NULL) return NULL; - pBuilder->tCols = 256; - pBuilder->columns = (STColumn *)taosMemoryMalloc(sizeof(STColumn) * pBuilder->tCols); - if (pBuilder->columns == NULL) return -1; + pTSchema->numOfCols = numOfCols; + pTSchema->version = version; - tdResetTSchemaBuilder(pBuilder, version); - return 0; -} + // timestamp column + ASSERT(aSchema[0].type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(aSchema[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID); + pTSchema->columns[0].colId = aSchema[0].colId; + pTSchema->columns[0].type = aSchema[0].type; + pTSchema->columns[0].flags = aSchema[0].flags; + pTSchema->columns[0].bytes = TYPE_BYTES[aSchema[0].type]; + pTSchema->columns[0].offset = -1; -void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder) { - if (pBuilder) { - taosMemoryFreeClear(pBuilder->columns); - } -} + // other columns + for (int32_t iCol = 1; iCol < numOfCols; iCol++) { + SSchema *pSchema = &aSchema[iCol]; + STColumn *pTColumn = &pTSchema->columns[iCol]; -void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) { - pBuilder->nCols = 0; - pBuilder->tlen = 0; - pBuilder->flen = 0; - pBuilder->version = version; -} + pTColumn->colId = pSchema->colId; + pTColumn->type = pSchema->type; + pTColumn->flags = pSchema->flags; + pTColumn->offset = pTSchema->flen; -int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes) { - if (!isValidDataType(type)) return -1; + if (IS_VAR_DATA_TYPE(pSchema->type)) { + pTColumn->bytes = pSchema->bytes; + pTSchema->tlen += (TYPE_BYTES[pSchema->type] + pSchema->bytes); // todo: remove + } else { + pTColumn->bytes = TYPE_BYTES[pSchema->type]; + pTSchema->tlen += TYPE_BYTES[pSchema->type]; // todo: remove + } - if (pBuilder->nCols >= pBuilder->tCols) { - pBuilder->tCols *= 2; - STColumn *columns = (STColumn *)taosMemoryRealloc(pBuilder->columns, sizeof(STColumn) * pBuilder->tCols); - if (columns == NULL) return -1; - pBuilder->columns = columns; + pTSchema->flen += TYPE_BYTES[pTColumn->type]; } - STColumn *pCol = &(pBuilder->columns[pBuilder->nCols]); - pCol->type = type; - pCol->colId = colId; - pCol->flags = flags; - if (pBuilder->nCols == 0) { - pCol->offset = -1; - } else { - pCol->offset = pBuilder->flen; - pBuilder->flen += TYPE_BYTES[type]; - } - - if (IS_VAR_DATA_TYPE(type)) { - pCol->bytes = bytes; - pBuilder->tlen += (TYPE_BYTES[type] + bytes); - } else { - pCol->bytes = TYPE_BYTES[type]; - pBuilder->tlen += TYPE_BYTES[type]; - } - - pBuilder->nCols++; - - ASSERT(pCol->offset < pBuilder->flen); - - return 0; -} - -STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { - if (pBuilder->nCols <= 0) return NULL; - - int tlen = sizeof(STSchema) + sizeof(STColumn) * pBuilder->nCols; - - STSchema *pSchema = (STSchema *)taosMemoryMalloc(tlen); - if (pSchema == NULL) return NULL; - - pSchema->version = pBuilder->version; - pSchema->numOfCols = pBuilder->nCols; - pSchema->tlen = pBuilder->tlen; - pSchema->flen = pBuilder->flen; - -#ifdef TD_SUPPORT_BITMAP - pSchema->tlen += (int)TD_BITMAP_BYTES(pSchema->numOfCols); -#endif - - memcpy(&pSchema->columns[0], pBuilder->columns, sizeof(STColumn) * pBuilder->nCols); - - return pSchema; -} - -#endif - -STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version) { - STSchema *pTSchema = taosMemoryCalloc(1, sizeof(STSchema) + sizeof(STColumn) * numOfCols); - if (pTSchema == NULL) return NULL; - - pTSchema->numOfCols = numOfCols; - pTSchema->version = version; - - // timestamp column - ASSERT(aSchema[0].type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(aSchema[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID); - pTSchema->columns[0].colId = aSchema[0].colId; - pTSchema->columns[0].type = aSchema[0].type; - pTSchema->columns[0].flags = aSchema[0].flags; - pTSchema->columns[0].bytes = aSchema[0].bytes; - pTSchema->columns[0].offset = -1; - - // other columns - for (int32_t iCol = 1; iCol < numOfCols; iCol++) { - SSchema *pSchema = &aSchema[iCol]; - STColumn *pTColumn = &pTSchema->columns[iCol]; - - pTColumn->colId = pSchema->colId; - pTColumn->type = pSchema->type; - pTColumn->flags = pSchema->flags; - pTColumn->bytes = pSchema->bytes; - pTColumn->offset = pTSchema->flen; - - pTSchema->flen += TYPE_BYTES[pTColumn->type]; - } +#if 1 // todo : remove this + pTSchema->tlen += (int32_t)TD_BITMAP_BYTES(numOfCols); +#endif return pTSchema; } @@ -1213,7 +1543,7 @@ void tColDataDestroy(void *ph) { SColData *pColData = (SColData *)ph; tFree(pColData->pBitMap); - tFree((uint8_t *)pColData->aOffset); + tFree(pColData->aOffset); tFree(pColData->pData); } @@ -1230,25 +1560,37 @@ void tColDataClear(SColData *pColData) { pColData->nData = 0; } -static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVal) { +void tColDataDeepClear(SColData *pColData) { + pColData->pBitMap = NULL; + pColData->aOffset = NULL; + pColData->pData = NULL; + + tColDataClear(pColData); +} + +static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; if (IS_VAR_DATA_TYPE(pColData->type)) { - code = tRealloc((uint8_t **)(&pColData->aOffset), sizeof(int32_t) * (pColData->nVal + 1)); + code = tRealloc((uint8_t **)(&pColData->aOffset), (pColData->nVal + 1) << 2); if (code) goto _exit; pColData->aOffset[pColData->nVal] = pColData->nData; - if (pColVal->value.nData) { - code = tRealloc(&pColData->pData, pColData->nData + pColVal->value.nData); + if (nData) { + code = tRealloc(&pColData->pData, pColData->nData + nData); if (code) goto _exit; - memcpy(pColData->pData + pColData->nData, pColVal->value.pData, pColVal->value.nData); - pColData->nData += pColVal->value.nData; + memcpy(pColData->pData + pColData->nData, pData, nData); + pColData->nData += nData; } } else { ASSERT(pColData->nData == tDataTypes[pColData->type].bytes * pColData->nVal); code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes); if (code) goto _exit; - memcpy(pColData->pData + pColData->nData, &pColVal->value.val, tDataTypes[pColData->type].bytes); + if (pData) { + memcpy(pColData->pData + pColData->nData, pData, TYPE_BYTES[pColData->type]); + } else { + memset(pColData->pData + pColData->nData, 0, TYPE_BYTES[pColData->type]); + } pColData->nData += tDataTypes[pColData->type].bytes; } pColData->nVal++; @@ -1256,21 +1598,21 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVa _exit: return code; } -static FORCE_INLINE int32_t tColDataAppendValue00(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue00(SColData *pColData, uint8_t *pData, uint32_t nData) { pColData->flag = HAS_VALUE; - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue01(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue01(SColData *pColData, uint8_t *pData, uint32_t nData) { pColData->flag = HAS_NONE; pColData->nVal++; return 0; } -static FORCE_INLINE int32_t tColDataAppendValue02(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue02(SColData *pColData, uint8_t *pData, uint32_t nData) { pColData->flag = HAS_NULL; pColData->nVal++; return 0; } -static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; int32_t nBit = BIT1_SIZE(pColData->nVal + 1); @@ -1278,7 +1620,7 @@ static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 0, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); pColData->flag |= HAS_VALUE; @@ -1296,13 +1638,13 @@ static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *p } } - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue11(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue11(SColData *pColData, uint8_t *pData, uint32_t nData) { pColData->nVal++; return 0; } -static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; int32_t nBit = BIT1_SIZE(pColData->nVal + 1); @@ -1310,14 +1652,14 @@ static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 0, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); pColData->flag |= HAS_NULL; pColData->nVal++; return code; } -static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; int32_t nBit = BIT1_SIZE(pColData->nVal + 1); @@ -1325,7 +1667,7 @@ static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 0, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); pColData->flag |= HAS_VALUE; @@ -1343,9 +1685,9 @@ static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, SColVal *p } } - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; int32_t nBit = BIT1_SIZE(pColData->nVal + 1); @@ -1353,18 +1695,18 @@ static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 255, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); pColData->flag |= HAS_NONE; pColData->nVal++; return code; } -static FORCE_INLINE int32_t tColDataAppendValue22(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue22(SColData *pColData, uint8_t *pData, uint32_t nData) { pColData->nVal++; return 0; } -static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; pColData->flag |= HAS_VALUE; @@ -1374,9 +1716,9 @@ static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, SColVal *p if (code) return code; for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal)); + SET_BIT2_EX(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal)); } - SET_BIT2(pBitMap, pColData->nVal, 2); + SET_BIT2_EX(pBitMap, pColData->nVal, 2); tFree(pColData->pBitMap); pColData->pBitMap = pBitMap; @@ -1395,32 +1737,32 @@ static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, SColVal *p } } - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue31(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue31(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); pColData->nVal++; return code; } -static FORCE_INLINE int32_t tColDataAppendValue32(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue32(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); pColData->nVal++; return code; } #define tColDataAppendValue40 tColDataPutValue -static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; pColData->flag |= HAS_NONE; @@ -1430,11 +1772,11 @@ static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 255, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; pColData->flag |= HAS_NULL; @@ -1444,31 +1786,31 @@ static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 255, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; pColData->flag |= HAS_NULL; @@ -1478,25 +1820,25 @@ static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, SColVal *p if (code) return code; for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0); + SET_BIT2_EX(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0); } - SET_BIT2(pBitMap, pColData->nVal, 1); + SET_BIT2_EX(pBitMap, pColData->nVal, 1); tFree(pColData->pBitMap); pColData->pBitMap = pBitMap; - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; pColData->flag |= HAS_NONE; @@ -1506,52 +1848,52 @@ static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, SColVal *p if (code) return code; for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1); + SET_BIT2_EX(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1); } - SET_BIT2(pBitMap, pColData->nVal, 0); + SET_BIT2_EX(pBitMap, pColData->nVal, 0); tFree(pColData->pBitMap); pColData->pBitMap = pBitMap; - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT2(pColData->pBitMap, pColData->nVal, 2); + SET_BIT2_EX(pColData->pBitMap, pColData->nVal, 2); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT2(pColData->pBitMap, pColData->nVal, 0); + SET_BIT2_EX(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT2(pColData->pBitMap, pColData->nVal, 1); + SET_BIT2_EX(pColData->pBitMap, pColData->nVal, 1); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, SColVal *pColVal) = { +static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData) = { {tColDataAppendValue00, tColDataAppendValue01, tColDataAppendValue02}, // 0 {tColDataAppendValue10, tColDataAppendValue11, tColDataAppendValue12}, // HAS_NONE {tColDataAppendValue20, tColDataAppendValue21, tColDataAppendValue22}, // HAS_NULL @@ -1563,7 +1905,9 @@ static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, SColVal *pCo }; int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) { ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type); - return tColDataAppendValueImpl[pColData->flag][pColVal->flag](pColData, pColVal); + return tColDataAppendValueImpl[pColData->flag][pColVal->flag]( + pColData, IS_VAR_DATA_TYPE(pColData->type) ? pColVal->value.pData : (uint8_t *)&pColVal->value.val, + pColVal->value.nData); } static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NONE @@ -1572,7 +1916,8 @@ static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SCo static FORCE_INLINE void tColDataGetValue2(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NULL *pColVal = COL_VAL_NULL(pColData->cid, pColData->type); } -static FORCE_INLINE void tColDataGetValue3(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NULL|HAS_NONE +static FORCE_INLINE void tColDataGetValue3(SColData *pColData, int32_t iVal, + SColVal *pColVal) { // HAS_NULL|HAS_NONE switch (GET_BIT1(pColData->pBitMap, iVal)) { case 0: *pColVal = COL_VAL_NONE(pColData->cid, pColData->type); @@ -1687,46 +2032,776 @@ uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal) { return v; } -int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { +int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMalloc, void *arg) { int32_t code = 0; - int32_t size; - - ASSERT(pColDataSrc->nVal > 0); - ASSERT(pColDataDest->cid == pColDataSrc->cid); - ASSERT(pColDataDest->type == pColDataSrc->type); - pColDataDest->smaOn = pColDataSrc->smaOn; - pColDataDest->nVal = pColDataSrc->nVal; - pColDataDest->flag = pColDataSrc->flag; + *pColData = *pColDataFrom; // bitmap - if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) { - size = BIT2_SIZE(pColDataSrc->nVal); - code = tRealloc(&pColDataDest->pBitMap, size); - if (code) goto _exit; - memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size); + switch (pColData->flag) { + case (HAS_NULL | HAS_NONE): + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + pColData->pBitMap = xMalloc(arg, BIT1_SIZE(pColData->nVal)); + if (pColData->pBitMap == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT1_SIZE(pColData->nVal)); + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + pColData->pBitMap = xMalloc(arg, BIT2_SIZE(pColData->nVal)); + if (pColData->pBitMap == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT2_SIZE(pColData->nVal)); + break; + default: + pColData->pBitMap = NULL; + break; } // offset - if (IS_VAR_DATA_TYPE(pColDataDest->type)) { - size = sizeof(int32_t) * pColDataSrc->nVal; + if (IS_VAR_DATA_TYPE(pColData->type) && (pColData->flag & HAS_VALUE)) { + pColData->aOffset = xMalloc(arg, pColData->nVal << 2); + if (pColData->aOffset == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(pColData->aOffset, pColDataFrom->aOffset, pColData->nVal << 2); + } else { + pColData->aOffset = NULL; + } - code = tRealloc((uint8_t **)&pColDataDest->aOffset, size); - if (code) goto _exit; + // value + if (pColData->nData) { + pColData->pData = xMalloc(arg, pColData->nData); + if (pColData->pData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } - memcpy(pColDataDest->aOffset, pColDataSrc->aOffset, size); + memcpy(pColData->pData, pColDataFrom->pData, pColData->nData); + } else { + pColData->pData = NULL; } - // value - pColDataDest->nData = pColDataSrc->nData; - code = tRealloc(&pColDataDest->pData, pColDataSrc->nData); - if (code) goto _exit; - memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData); +_exit: + return code; +} + +int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t bytes, int32_t nRows, char *lengthOrbitmap, + char *data) { + int32_t code = 0; + + if (IS_VAR_DATA_TYPE(type)) { // var-length data type + for (int32_t i = 0; i < nRows; ++i) { + int32_t offset = *((int32_t *)lengthOrbitmap + i); + if (offset == -1) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, (uint8_t *)varDataVal(data + offset), + varDataLen(data + offset)); + } + } + } else { // fixed-length data type + bool allValue = true; + bool allNull = true; + for (int32_t i = 0; i < nRows; ++i) { + if (!colDataIsNull_f(lengthOrbitmap, i)) { + allNull = false; + } else { + allValue = false; + } + } + + if (allValue) { + // optimize (todo) + for (int32_t i = 0; i < nRows; ++i) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, (uint8_t *)data + bytes * i, bytes); + } + } else if (allNull) { + // optimize (todo) + for (int32_t i = 0; i < nRows; ++i) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } + } else { + for (int32_t i = 0; i < nRows; ++i) { + if (colDataIsNull_f(lengthOrbitmap, i)) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, (uint8_t *)data + bytes * i, bytes); + } + } + } + } _exit: return code; } +int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) { + int32_t code = 0; + + ASSERT(pColData->type == pBind->buffer_type); + + if (IS_VAR_DATA_TYPE(pBind->buffer_type)) { // var-length data type + for (int32_t i = 0; i < pBind->num; ++i) { + if (pBind->is_null && pBind->is_null[i]) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE]( + pColData, (uint8_t *)pBind->buffer + pBind->buffer_length * i, pBind->length[i]); + } + } + } else { // fixed-length data type + bool allValue; + bool allNull; + if (pBind->is_null) { + bool same = (memcmp(pBind->is_null, pBind->is_null + 1, pBind->num - 1) == 0); + allNull = (same && pBind->is_null[0] != 0); + allValue = (same && pBind->is_null[0] == 0); + } else { + allNull = false; + allValue = true; + } + + if (allValue) { + // optimize (todo) + for (int32_t i = 0; i < pBind->num; ++i) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE]( + pColData, (uint8_t *)pBind->buffer + TYPE_BYTES[pColData->type] * i, pBind->buffer_length); + } + } else if (allNull) { + // optimize (todo) + for (int32_t i = 0; i < pBind->num; ++i) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } + } else { + for (int32_t i = 0; i < pBind->num; ++i) { + if (pBind->is_null[i]) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE]( + pColData, (uint8_t *)pBind->buffer + TYPE_BYTES[pColData->type] * i, pBind->buffer_length); + } + } + } + } + +_exit: + return code; +} + +static int32_t tColDataSwapValue(SColData *pColData, int32_t i, int32_t j) { + int32_t code = 0; + + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nData1 = pColData->aOffset[i + 1] - pColData->aOffset[i]; + int32_t nData2 = (j < pColData->nVal - 1) ? pColData->aOffset[j + 1] - pColData->aOffset[j] + : pColData->nData - pColData->aOffset[j]; + uint8_t *pData = taosMemoryMalloc(TMAX(nData1, nData2)); + if (pData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + if (nData1 > nData2) { + memcpy(pData, pColData->pData + pColData->aOffset[i], nData1); + memcpy(pColData->pData + pColData->aOffset[i], pColData->pData + pColData->aOffset[j], nData2); + // memmove(pColData->pData + pColData->aOffset[i] + nData2, pColData->pData + pColData->aOffset[i] + nData1, + // pColData->aOffset[j] - pColData->aOffset[i + 1]); + memmove(pColData->pData + pColData->aOffset[i] + nData2, pColData->pData + pColData->aOffset[i + 1], + pColData->aOffset[j] - pColData->aOffset[i + 1]); + memcpy(pColData->pData + pColData->aOffset[j] + nData2 - nData1, pData, nData1); + } else { + memcpy(pData, pColData->pData + pColData->aOffset[j], nData2); + memcpy(pColData->pData + pColData->aOffset[j] + nData2 - nData1, pColData->pData + pColData->aOffset[i], nData1); + // memmove(pColData->pData + pColData->aOffset[j] + nData2 - nData1, pColData->pData + pColData->aOffset[i] + + // nData1, + // pColData->aOffset[j] - pColData->aOffset[i + 1]); + memmove(pColData->pData + pColData->aOffset[i] + nData2, pColData->pData + pColData->aOffset[i + 1], + pColData->aOffset[j] - pColData->aOffset[i + 1]); + memcpy(pColData->pData + pColData->aOffset[i], pData, nData2); + } + for (int32_t k = i + 1; k <= j; ++k) { + pColData->aOffset[k] = pColData->aOffset[k] + nData2 - nData1; + } + + taosMemoryFree(pData); + } else { + uint64_t val; + memcpy(&val, &pColData->pData[TYPE_BYTES[pColData->type] * i], TYPE_BYTES[pColData->type]); + memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * i], &pColData->pData[TYPE_BYTES[pColData->type] * j], + TYPE_BYTES[pColData->type]); + memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * j], &val, TYPE_BYTES[pColData->type]); + } + +_exit: + return code; +} + +static void tColDataSwap(SColData *pColData, int32_t i, int32_t j) { + ASSERT(i < j); + ASSERT(j < pColData->nVal); + + switch (pColData->flag) { + case HAS_NONE: + case HAS_NULL: + break; + case (HAS_NULL | HAS_NONE): { + uint8_t bv = GET_BIT1(pColData->pBitMap, i); + SET_BIT1(pColData->pBitMap, i, GET_BIT1(pColData->pBitMap, j)); + SET_BIT1(pColData->pBitMap, j, bv); + } break; + case HAS_VALUE: { + tColDataSwapValue(pColData, i, j); + } break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): { + uint8_t bv = GET_BIT1(pColData->pBitMap, i); + SET_BIT1(pColData->pBitMap, i, GET_BIT1(pColData->pBitMap, j)); + SET_BIT1(pColData->pBitMap, j, bv); + tColDataSwapValue(pColData, i, j); + } break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): { + uint8_t bv = GET_BIT2(pColData->pBitMap, i); + SET_BIT2(pColData->pBitMap, i, GET_BIT2(pColData->pBitMap, j)); + SET_BIT2(pColData->pBitMap, j, bv); + tColDataSwapValue(pColData, i, j); + } break; + default: + ASSERT(0); + break; + } +} + +static int32_t tColDataCopyRowCell(SColData *pFromColData, int32_t iFromRow, SColData *pToColData, int32_t iToRow) { + int32_t code = TSDB_CODE_SUCCESS; + + if (IS_VAR_DATA_TYPE(pToColData->type)) { + int32_t nData = (iFromRow < pFromColData->nVal - 1) + ? pFromColData->aOffset[iFromRow + 1] - pFromColData->aOffset[iFromRow] + : pFromColData->nData - pFromColData->aOffset[iFromRow]; + if (iToRow == 0) { + pToColData->aOffset[iToRow] = 0; + } + + if (iToRow < pToColData->nVal - 1) { + pToColData->aOffset[iToRow + 1] = pToColData->aOffset[iToRow] + nData; + } + + memcpy(pToColData->pData + pToColData->aOffset[iToRow], pFromColData->pData + pFromColData->aOffset[iFromRow], + nData); + } else { + memcpy(&pToColData->pData[TYPE_BYTES[pToColData->type] * iToRow], + &pFromColData->pData[TYPE_BYTES[pToColData->type] * iFromRow], TYPE_BYTES[pToColData->type]); + } + return code; +} + +static int32_t tColDataCopyRowSingleCol(SColData *pFromColData, int32_t iFromRow, SColData *pToColData, + int32_t iToRow) { + int32_t code = TSDB_CODE_SUCCESS; + + switch (pFromColData->flag) { + case HAS_NONE: + case HAS_NULL: + break; + case (HAS_NULL | HAS_NONE): { + SET_BIT1(pToColData->pBitMap, iToRow, GET_BIT1(pFromColData->pBitMap, iFromRow)); + } break; + case HAS_VALUE: { + tColDataCopyRowCell(pFromColData, iFromRow, pToColData, iToRow); + } break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): { + SET_BIT1(pToColData->pBitMap, iToRow, GET_BIT1(pFromColData->pBitMap, iFromRow)); + tColDataCopyRowCell(pFromColData, iFromRow, pToColData, iToRow); + } break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): { + SET_BIT2(pToColData->pBitMap, iToRow, GET_BIT2(pFromColData->pBitMap, iFromRow)); + tColDataCopyRowCell(pFromColData, iFromRow, pToColData, iToRow); + } break; + default: + return -1; + } + + return code; +} + +static int32_t tColDataCopyRow(SColData *aFromColData, int32_t iFromRow, SColData *aToColData, int32_t iToRow, + int32_t nColData) { + int32_t code = TSDB_CODE_SUCCESS; + + for (int32_t i = 0; i < nColData; i++) { + code = tColDataCopyRowSingleCol(&aFromColData[i], iFromRow, &aToColData[i], iToRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + return code; +} + +static int32_t tColDataCopyRowAppend(SColData *aFromColData, int32_t iFromRow, SColData *aToColData, int32_t nColData) { + int32_t code = TSDB_CODE_SUCCESS; + + for (int32_t i = 0; i < nColData; i++) { + SColVal cv = {0}; + tColDataGetValue(&aFromColData[i], iFromRow, &cv); + code = tColDataAppendValue(&aToColData[i], &cv); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + return code; +} + +static int32_t tColDataMergeSortMerge(SColData *aColData, int32_t start, int32_t mid, int32_t end, int32_t nColData) { + SColData *aDstColData = NULL; + TSKEY *aKey = (TSKEY *)aColData[0].pData; + + int32_t i = start, j = mid + 1, k = 0; + + if (end > start) { + aDstColData = taosMemoryCalloc(1, sizeof(SColData) * nColData); + for (int c = 0; c < nColData; ++c) { + tColDataInit(&aDstColData[c], aColData[c].cid, aColData[c].type, aColData[c].smaOn); + } + if (aDstColData == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + /* + for (int32_t i = 0; i < nColData; i++) { + tColDataCopy(&aColData[i], &aDstColData[i], tColDataDefaultMalloc, NULL); + } + */ + } + + while (i <= mid && j <= end) { + if (aKey[i] <= aKey[j]) { + // tColDataCopyRow(aColData, i++, aDstColData, k++); + tColDataCopyRowAppend(aColData, i++, aDstColData, nColData); + } else { + // tColDataCopyRow(aColData, j++, aDstColData, k++); + tColDataCopyRowAppend(aColData, j++, aDstColData, nColData); + } + } + + while (i <= mid) { + // tColDataCopyRow(aColData, i++, aDstColData, k++); + tColDataCopyRowAppend(aColData, i++, aDstColData, nColData); + } + + while (j <= end) { + // tColDataCopyRow(aColData, j++, aDstColData, k++); + tColDataCopyRowAppend(aColData, j++, aDstColData, nColData); + } + + for (i = start, k = 0; i <= end; ++i, ++k) { + tColDataCopyRow(aDstColData, k, aColData, i, nColData); + } + + if (aDstColData) { + for (int32_t i = 0; i < nColData; i++) { + tColDataDestroy(&aDstColData[i]); + } + taosMemoryFree(aDstColData); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t tColDataMergeSort(SColData *aColData, int32_t start, int32_t end, int32_t nColData) { + int32_t ret = TSDB_CODE_SUCCESS; + int32_t mid; + + if (start >= end) { + return TSDB_CODE_SUCCESS; + } + + mid = (start + end) / 2; + + ret = tColDataMergeSort(aColData, start, mid, nColData); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + ret = tColDataMergeSort(aColData, mid + 1, end, nColData); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + return tColDataMergeSortMerge(aColData, start, mid, end, nColData); +} + +static int32_t tColDataSort(SColData *aColData, int32_t nColData) { + int32_t nVal = aColData[0].nVal; + + if (nVal < 2) return TSDB_CODE_SUCCESS; + + return tColDataMergeSort(aColData, 0, nVal - 1, nColData); +} +static void tColDataMergeImpl(SColData *pColData, int32_t iStart, int32_t iEnd /* not included */) { + switch (pColData->flag) { + case HAS_NONE: + case HAS_NULL: { + pColData->nVal -= (iEnd - iStart - 1); + } break; + case (HAS_NULL | HAS_NONE): { + if (GET_BIT1(pColData->pBitMap, iStart) == 0) { + for (int32_t i = iStart + 1; i < iEnd; ++i) { + if (GET_BIT1(pColData->pBitMap, i) == 1) { + SET_BIT1(pColData->pBitMap, iStart, 1); + break; + } + } + } + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + SET_BIT1(pColData->pBitMap, j, GET_BIT1(pColData->pBitMap, i)); + } + + pColData->nVal -= (iEnd - iStart - 1); + + uint8_t flag = 0; + for (int32_t i = 0; i < pColData->nVal; ++i) { + uint8_t bv = GET_BIT1(pColData->pBitMap, i); + if (bv == BIT_FLG_NONE) { + flag |= HAS_NONE; + } else if (bv == BIT_FLG_NULL) { + flag |= HAS_NULL; + } else { + ASSERT(0); + } + + if (flag == pColData->flag) break; + } + pColData->flag = flag; + } break; + case HAS_VALUE: { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nDiff = pColData->aOffset[iEnd - 1] - pColData->aOffset[iStart]; + + memmove(pColData->pData + pColData->aOffset[iStart], pColData->pData + pColData->aOffset[iEnd - 1], + pColData->nData - pColData->aOffset[iEnd - 1]); + pColData->nData -= nDiff; + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + pColData->aOffset[j] = pColData->aOffset[i] - nDiff; + } + } else { + memmove(pColData->pData + TYPE_BYTES[pColData->type] * iStart, + pColData->pData + TYPE_BYTES[pColData->type] * (iEnd - 1), + TYPE_BYTES[pColData->type] * (pColData->nVal - iEnd + 1)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + pColData->nVal -= (iEnd - iStart - 1); + } break; + case (HAS_VALUE | HAS_NONE): { + uint8_t bv; + int32_t iv; + for (int32_t i = iEnd - 1; i >= iStart; --i) { + bv = GET_BIT1(pColData->pBitMap, i); + if (bv) { + iv = i; + break; + } + } + + if (bv) { // has a value + if (IS_VAR_DATA_TYPE(pColData->type)) { + if (iv != iStart) { + memmove(&pColData->pData[pColData->aOffset[iStart]], &pColData->pData[pColData->aOffset[iv]], + iv < (pColData->nVal - 1) ? pColData->aOffset[iv + 1] - pColData->aOffset[iv] + : pColData->nData - pColData->aOffset[iv]); + } + // TODO + ASSERT(0); + } else { + if (iv != iStart) { + memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * iStart], + &pColData->pData[TYPE_BYTES[pColData->type] * iv], TYPE_BYTES[pColData->type]); + } + memmove(&pColData->pData[TYPE_BYTES[pColData->type] * (iStart + 1)], + &pColData->pData[TYPE_BYTES[pColData->type] * iEnd], + TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + SET_BIT1(pColData->pBitMap, iStart, 1); + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + SET_BIT1(pColData->pBitMap, j, GET_BIT1(pColData->pBitMap, i)); + } + + uint8_t flag = HAS_VALUE; + for (int32_t i = 0; i < pColData->nVal - (iEnd - iStart - 1); ++i) { + if (GET_BIT1(pColData->pBitMap, i) == 0) { + flag |= HAS_NONE; + } + + if (flag == pColData->flag) break; + } + pColData->flag = flag; + } else { // all NONE + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nDiff = pColData->aOffset[iEnd - 1] - pColData->aOffset[iStart]; + + memmove(&pColData->pData[pColData->aOffset[iStart]], &pColData->pData[pColData->aOffset[iEnd - 1]], + pColData->nData - pColData->aOffset[iEnd - 1]); + pColData->nData -= nDiff; + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + pColData->aOffset[j] = pColData->aOffset[i] - nDiff; + } + } else { + memmove(pColData->pData + TYPE_BYTES[pColData->type] * (iStart + 1), + pColData->pData + TYPE_BYTES[pColData->type] * iEnd, + TYPE_BYTES[pColData->type] * (pColData->nVal - iEnd + 1)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + SET_BIT1(pColData->pBitMap, j, GET_BIT1(pColData->pBitMap, i)); + } + } + pColData->nVal -= (iEnd - iStart - 1); + } break; + case (HAS_VALUE | HAS_NULL): { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nDiff = pColData->aOffset[iEnd - 1] - pColData->aOffset[iStart]; + + memmove(pColData->pData + pColData->aOffset[iStart], pColData->pData + pColData->aOffset[iEnd - 1], + pColData->nData - pColData->aOffset[iEnd - 1]); + pColData->nData -= nDiff; + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + pColData->aOffset[j] = pColData->aOffset[i] - nDiff; + } + } else { + memmove(pColData->pData + TYPE_BYTES[pColData->type] * iStart, + pColData->pData + TYPE_BYTES[pColData->type] * (iEnd - 1), + TYPE_BYTES[pColData->type] * (pColData->nVal - iEnd + 1)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + for (int32_t i = iEnd - 1, j = iStart; i < pColData->nVal; ++i, ++j) { + SET_BIT1(pColData->pBitMap, j, GET_BIT1(pColData->pBitMap, i)); + } + + pColData->nVal -= (iEnd - iStart - 1); + + uint8_t flag = 0; + for (int32_t i = 0; i < pColData->nVal; ++i) { + if (GET_BIT1(pColData->pBitMap, i)) { + flag |= HAS_VALUE; + } else { + flag |= HAS_NULL; + } + + if (flag == pColData->flag) break; + } + pColData->flag = flag; + } break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): { + uint8_t bv; + int32_t iv; + for (int32_t i = iEnd - 1; i >= iStart; --i) { + bv = GET_BIT2(pColData->pBitMap, i); + if (bv) { + iv = i; + break; + } + } + + if (bv) { + // TODO + ASSERT(0); + } else { // ALL NONE + if (IS_VAR_DATA_TYPE(pColData->type)) { + // TODO + ASSERT(0); + } else { + memmove(pColData->pData + TYPE_BYTES[pColData->type] * (iStart + 1), + pColData->pData + TYPE_BYTES[pColData->type] * iEnd, + TYPE_BYTES[pColData->type] * (pColData->nVal - iEnd)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + SET_BIT2(pColData->pBitMap, j, GET_BIT2(pColData->pBitMap, i)); + } + } + pColData->nVal -= (iEnd - iStart - 1); + } break; + default: + ASSERT(0); + break; + } +} +static void tColDataMerge(SColData *aColData, int32_t nColData) { + int32_t iStart = 0; + for (;;) { + if (iStart >= aColData[0].nVal - 1) break; + + int32_t iEnd = iStart + 1; + while (iEnd < aColData[0].nVal) { + if (((TSKEY *)aColData[0].pData)[iEnd] != ((TSKEY *)aColData[0].pData)[iStart]) break; + + iEnd++; + } + + if (iEnd - iStart > 1) { + for (int32_t i = 0; i < nColData; i++) { + tColDataMergeImpl(&aColData[i], iStart, iEnd); + } + } + + iStart++; + } +} +void tColDataSortMerge(SArray *colDataArr) { + int32_t nColData = TARRAY_SIZE(colDataArr); + SColData *aColData = (SColData *)TARRAY_DATA(colDataArr); + + if (aColData[0].nVal <= 1) goto _exit; + + ASSERT(aColData[0].type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(aColData[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); + ASSERT(aColData[0].flag == HAS_VALUE); + + int8_t doSort = 0; + int8_t doMerge = 0; + // scan ------- + TSKEY *aKey = (TSKEY *)aColData[0].pData; + for (int32_t iVal = 1; iVal < aColData[0].nVal; ++iVal) { + if (aKey[iVal] > aKey[iVal - 1]) { + continue; + } else if (aKey[iVal] < aKey[iVal - 1]) { + doSort = 1; + break; + } else { + doMerge = 1; + } + } + + // sort ------- + if (doSort) { + tColDataSort(aColData, nColData); + } + + if (doMerge != 1) { + for (int32_t iVal = 1; iVal < aColData[0].nVal; ++iVal) { + if (aKey[iVal] == aKey[iVal - 1]) { + doMerge = 1; + break; + } + } + } + + // merge ------- + if (doMerge) { + tColDataMerge(aColData, nColData); + } + +_exit: + return; +} + +int32_t tPutColData(uint8_t *pBuf, SColData *pColData) { + int32_t n = 0; + + n += tPutI16v(pBuf ? pBuf + n : NULL, pColData->cid); + n += tPutI8(pBuf ? pBuf + n : NULL, pColData->type); + n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nVal); + n += tPutI8(pBuf ? pBuf + n : NULL, pColData->flag); + + // bitmap + switch (pColData->flag) { + case (HAS_NULL | HAS_NONE): + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT1_SIZE(pColData->nVal)); + n += BIT1_SIZE(pColData->nVal); + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT2_SIZE(pColData->nVal)); + n += BIT2_SIZE(pColData->nVal); + break; + default: + break; + } + + // value + if (pColData->flag & HAS_VALUE) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + if (pBuf) memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2); + n += (pColData->nVal << 2); + + n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nData); + if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); + n += pColData->nData; + } else { + if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); + n += pColData->nData; + } + } + + return n; +} + +int32_t tGetColData(uint8_t *pBuf, SColData *pColData) { + int32_t n = 0; + + n += tGetI16v(pBuf + n, &pColData->cid); + n += tGetI8(pBuf + n, &pColData->type); + n += tGetI32v(pBuf + n, &pColData->nVal); + n += tGetI8(pBuf + n, &pColData->flag); + + // bitmap + switch (pColData->flag) { + case (HAS_NULL | HAS_NONE): + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + pColData->pBitMap = pBuf + n; + n += BIT1_SIZE(pColData->nVal); + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + pColData->pBitMap = pBuf + n; + n += BIT2_SIZE(pColData->nVal); + break; + default: + break; + } + + // value + if (pColData->flag & HAS_VALUE) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + pColData->aOffset = (int32_t *)(pBuf + n); + n += (pColData->nVal << 2); + + n += tGetI32v(pBuf + n, &pColData->nData); + pColData->pData = pBuf + n; + n += pColData->nData; + } else { + pColData->pData = pBuf + n; + pColData->nData = TYPE_BYTES[pColData->type] * pColData->nVal; + n += pColData->nData; + } + } + + return n; +} + #define CALC_SUM_MAX_MIN(SUM, MAX, MIN, VAL) \ do { \ (SUM) += (VAL); \ diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 9caf0cc33e3e97243b0fd00b7ccfb7f2c3e81110..98b9b566ecc0c0f8d8ea2fdb8866efc5e1c22769 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -47,7 +47,7 @@ int32_t tsNumOfMnodeQueryThreads = 4; int32_t tsNumOfMnodeFetchThreads = 1; int32_t tsNumOfMnodeReadThreads = 1; int32_t tsNumOfVnodeQueryThreads = 4; -int32_t tsNumOfVnodeStreamThreads = 2; +float tsRatioOfVnodeStreamThreads = 1.0; int32_t tsNumOfVnodeFetchThreads = 4; int32_t tsNumOfVnodeRsmaThreads = 2; int32_t tsNumOfQnodeQueryThreads = 4; @@ -392,9 +392,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 4); if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 4, 1024, 0) != 0) return -1; - tsNumOfVnodeStreamThreads = tsNumOfCores / 4; - tsNumOfVnodeStreamThreads = TMAX(tsNumOfVnodeStreamThreads, 4); - if (cfgAddInt32(pCfg, "numOfVnodeStreamThreads", tsNumOfVnodeStreamThreads, 4, 1024, 0) != 0) return -1; + if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 100, 0) != 0) return -1; tsNumOfVnodeFetchThreads = tsNumOfCores / 4; tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4); @@ -513,11 +511,9 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { pItem->stype = stype; } - pItem = cfgGetItem(tsCfg, "numOfVnodeStreamThreads"); + pItem = cfgGetItem(tsCfg, "ratioOfVnodeStreamThreads"); if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { - tsNumOfVnodeStreamThreads = numOfCores / 4; - tsNumOfVnodeStreamThreads = TMAX(tsNumOfVnodeStreamThreads, 4); - pItem->i32 = tsNumOfVnodeStreamThreads; + pItem->fval = tsRatioOfVnodeStreamThreads; pItem->stype = stype; } @@ -710,7 +706,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32; tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32; tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32; - tsNumOfVnodeStreamThreads = cfgGetItem(pCfg, "numOfVnodeStreamThreads")->i32; + tsRatioOfVnodeStreamThreads = cfgGetItem(pCfg, "ratioOfVnodeStreamThreads")->fval; tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32; tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32; tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32; @@ -1316,12 +1312,17 @@ void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal, if (pItem != NULL && (rewrite || pItem->i32 == 0)) { pItem->i32 = flagVal; } - *pFlagPtr = flagVal; + if (pFlagPtr != NULL) { + *pFlagPtr = flagVal; + } } void taosSetAllDebugFlag(int32_t flag, bool rewrite) { if (flag <= 0) return; + taosSetDebugFlag(NULL, "debugFlag", flag, rewrite); + taosSetDebugFlag(NULL, "simDebugFlag", flag, rewrite); + taosSetDebugFlag(NULL, "tmrDebugFlag", flag, rewrite); taosSetDebugFlag(&uDebugFlag, "uDebugFlag", flag, rewrite); taosSetDebugFlag(&rpcDebugFlag, "rpcDebugFlag", flag, rewrite); taosSetDebugFlag(&jniDebugFlag, "jniDebugFlag", flag, rewrite); @@ -1340,6 +1341,5 @@ void taosSetAllDebugFlag(int32_t flag, bool rewrite) { taosSetDebugFlag(&idxDebugFlag, "idxDebugFlag", flag, rewrite); taosSetDebugFlag(&tdbDebugFlag, "tdbDebugFlag", flag, rewrite); taosSetDebugFlag(&metaDebugFlag, "metaDebugFlag", flag, rewrite); - taosSetDebugFlag(&metaDebugFlag, "tmrDebugFlag", flag, rewrite); uInfo("all debug flag are set to %d", flag); } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 95625e8d9360946fbcf1432f804ec7841e006dfc..83f447fd0e99586c59fbfcbd52ddb9da3c019805 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -5424,6 +5424,7 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS if (tEncodeI32(&encoder, pField->bytes) < 0) return -1; if (tEncodeCStr(&encoder, pField->name) < 0) return -1; } + if (tEncodeI8(&encoder, pReq->createStb) < 0) return -1; tEndEncode(&encoder); @@ -5484,6 +5485,7 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea } } } + if (tDecodeI8(&decoder, &pReq->createStb) < 0) return -1; tEndDecode(&decoder); @@ -5627,30 +5629,6 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) { return 0; } -STSchema *tdGetSTSChemaFromSSChema(SSchema *pSchema, int32_t nCols, int32_t sver) { - STSchemaBuilder schemaBuilder = {0}; - if (tdInitTSchemaBuilder(&schemaBuilder, sver) < 0) { - return NULL; - } - - for (int i = 0; i < nCols; i++) { - SSchema *schema = pSchema + i; - if (tdAddColToSchema(&schemaBuilder, schema->type, schema->flags, schema->colId, schema->bytes) < 0) { - tdDestroyTSchemaBuilder(&schemaBuilder); - return NULL; - } - } - - STSchema *pNSchema = tdGetSchemaFromBuilder(&schemaBuilder); - if (pNSchema == NULL) { - tdDestroyTSchemaBuilder(&schemaBuilder); - return NULL; - } - - tdDestroyTSchemaBuilder(&schemaBuilder); - return pNSchema; -} - int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) { if (tStartEncode(pCoder) < 0) return -1; @@ -5728,6 +5706,25 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) { return 0; } +void tDestroySVCreateTbReq(SVCreateTbReq *pReq, int32_t flags) { + if (pReq == NULL) return; + + if (flags & TSDB_MSG_FLG_ENCODE) { + // TODO + } else if (flags & TSDB_MSG_FLG_DECODE) { + if (pReq->comment) { + pReq->comment = NULL; + taosMemoryFree(pReq->comment); + } + + if (pReq->type == TSDB_CHILD_TABLE) { + if (pReq->ctb.tagName) taosArrayDestroy(pReq->ctb.tagName); + } else if (pReq->type == TSDB_NORMAL_TABLE) { + if (pReq->ntb.schemaRow.pSchema) taosMemoryFree(pReq->ntb.schemaRow.pSchema); + } + } +} + int tEncodeSVCreateTbBatchReq(SEncoder *pCoder, const SVCreateTbBatchReq *pReq) { int32_t nReq = taosArrayGetSize(pReq->pArray); @@ -6719,3 +6716,328 @@ int32_t tDecodeSBatchDeleteReq(SDecoder *pDecoder, SBatchDeleteReq *pReq) { } return 0; } + +static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubmitTbData) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32v(pCoder, pSubmitTbData->flags) < 0) return -1; + + // auto create table + if (pSubmitTbData->flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { + ASSERT(pSubmitTbData->pCreateTbReq); + if (tEncodeSVCreateTbReq(pCoder, pSubmitTbData->pCreateTbReq) < 0) return -1; + } + + // submit data + if (tEncodeI64(pCoder, pSubmitTbData->suid) < 0) return -1; + if (tEncodeI64(pCoder, pSubmitTbData->uid) < 0) return -1; + if (tEncodeI32v(pCoder, pSubmitTbData->sver) < 0) return -1; + + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + uint64_t nColData = TARRAY_SIZE(pSubmitTbData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pSubmitTbData->aCol); + + if (tEncodeU64v(pCoder, nColData) < 0) return -1; + + for (uint64_t i = 0; i < nColData; i++) { + pCoder->pos += tPutColData(pCoder->data ? pCoder->data + pCoder->pos : NULL, &aColData[i]); + } + } else { + if (tEncodeU64v(pCoder, TARRAY_SIZE(pSubmitTbData->aRowP)) < 0) return -1; + + SRow **rows = (SRow **)TARRAY_DATA(pSubmitTbData->aRowP); + for (int32_t iRow = 0; iRow < TARRAY_SIZE(pSubmitTbData->aRowP); ++iRow) { + if (pCoder->data) memcpy(pCoder->data + pCoder->pos, rows[iRow], rows[iRow]->len); + pCoder->pos += rows[iRow]->len; + } + } + + tEndEncode(pCoder); + return 0; +} + +static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbData) { + int32_t code = 0; + + if (tStartDecode(pCoder) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (tDecodeI32v(pCoder, &pSubmitTbData->flags) < 0) return -1; + + if (pSubmitTbData->flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { + pSubmitTbData->pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (pSubmitTbData->pCreateTbReq == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + if (tDecodeSVCreateTbReq(pCoder, pSubmitTbData->pCreateTbReq) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + } + + // submit data + if (tDecodeI64(pCoder, &pSubmitTbData->suid) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + if (tDecodeI64(pCoder, &pSubmitTbData->uid) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + if (tDecodeI32v(pCoder, &pSubmitTbData->sver) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + uint64_t nColData; + + if (tDecodeU64v(pCoder, &nColData) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + pSubmitTbData->aCol = taosArrayInit(nColData, sizeof(SColData)); + if (pSubmitTbData->aCol == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t i = 0; i < nColData; ++i) { + pCoder->pos += tGetColData(pCoder->data + pCoder->pos, taosArrayReserve(pSubmitTbData->aCol, 1)); + } + } else { + uint64_t nRow; + if (tDecodeU64v(pCoder, &nRow) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + pSubmitTbData->aRowP = taosArrayInit(nRow, sizeof(SRow *)); + if (pSubmitTbData->aRowP == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t iRow = 0; iRow < nRow; ++iRow) { + SRow **ppRow = taosArrayReserve(pSubmitTbData->aRowP, 1); + + *ppRow = (SRow *)(pCoder->data + pCoder->pos); + pCoder->pos += (*ppRow)->len; + } + } + + tEndDecode(pCoder); + +_exit: + if (code) { + // TODO: clear + } + return 0; +} + +int32_t tEncodeSSubmitReq2(SEncoder *pCoder, const SSubmitReq2 *pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeU64v(pCoder, taosArrayGetSize(pReq->aSubmitTbData)) < 0) return -1; + for (uint64_t i = 0; i < taosArrayGetSize(pReq->aSubmitTbData); i++) { + if (tEncodeSSubmitTbData(pCoder, taosArrayGet(pReq->aSubmitTbData, i)) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSSubmitReq2(SDecoder *pCoder, SSubmitReq2 *pReq) { + int32_t code = 0; + + memset(pReq, 0, sizeof(*pReq)); + + // decode + if (tStartDecode(pCoder) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + uint64_t nSubmitTbData; + if (tDecodeU64v(pCoder, &nSubmitTbData) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + pReq->aSubmitTbData = taosArrayInit(nSubmitTbData, sizeof(SSubmitTbData)); + if (pReq->aSubmitTbData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (uint64_t i = 0; i < nSubmitTbData; i++) { + if (tDecodeSSubmitTbData(pCoder, taosArrayReserve(pReq->aSubmitTbData, 1)) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + } + + tEndDecode(pCoder); + +_exit: + if (code) { + if (pReq->aSubmitTbData) { + // todo + taosArrayDestroy(pReq->aSubmitTbData); + pReq->aSubmitTbData = NULL; + } + } + return code; +} + +void tDestroySSubmitTbData(SSubmitTbData *pTbData, int32_t flag) { + if (NULL == pTbData) { + return; + } + + if (flag == TSDB_MSG_FLG_ENCODE) { + if (pTbData->pCreateTbReq) { + tdDestroySVCreateTbReq(pTbData->pCreateTbReq); + taosMemoryFree(pTbData->pCreateTbReq); + } + + if (pTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + int32_t nColData = TARRAY_SIZE(pTbData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pTbData->aCol); + + for (int32_t i = 0; i < nColData; ++i) { + tColDataDestroy(&aColData[i]); + } + taosArrayDestroy(pTbData->aCol); + } else { + int32_t nRow = TARRAY_SIZE(pTbData->aRowP); + SRow **rows = (SRow **)TARRAY_DATA(pTbData->aRowP); + + for (int32_t i = 0; i < nRow; ++i) { + tRowDestroy(rows[i]); + } + taosArrayDestroy(pTbData->aRowP); + } + } else if (flag == TSDB_MSG_FLG_DECODE) { + if (pTbData->pCreateTbReq) { + tDestroySVCreateTbReq(pTbData->pCreateTbReq, TSDB_MSG_FLG_DECODE); + taosMemoryFree(pTbData->pCreateTbReq); + } + + if (pTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + taosArrayDestroy(pTbData->aCol); + } else { + taosArrayDestroy(pTbData->aRowP); + } + } +} + +void tDestroySSubmitReq2(SSubmitReq2 *pReq, int32_t flag) { + if (pReq->aSubmitTbData == NULL) return; + + int32_t nSubmitTbData = TARRAY_SIZE(pReq->aSubmitTbData); + SSubmitTbData *aSubmitTbData = (SSubmitTbData *)TARRAY_DATA(pReq->aSubmitTbData); + + for (int32_t i = 0; i < nSubmitTbData; i++) { + tDestroySSubmitTbData(&aSubmitTbData[i], flag); + } + taosArrayDestroy(pReq->aSubmitTbData); +} + +int32_t tEncodeSSubmitRsp2(SEncoder *pCoder, const SSubmitRsp2 *pRsp) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32v(pCoder, pRsp->affectedRows) < 0) return -1; + + if (tEncodeU64v(pCoder, taosArrayGetSize(pRsp->aCreateTbRsp)) < 0) return -1; + for (int32_t i = 0; i < taosArrayGetSize(pRsp->aCreateTbRsp); ++i) { + if (tEncodeSVCreateTbRsp(pCoder, taosArrayGet(pRsp->aCreateTbRsp, i)) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSSubmitRsp2(SDecoder *pCoder, SSubmitRsp2 *pRsp) { + int32_t code = 0; + + memset(pRsp, 0, sizeof(SSubmitRsp2)); + + // decode + if (tStartDecode(pCoder) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (tDecodeI32v(pCoder, &pRsp->affectedRows) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + uint64_t nCreateTbRsp; + if (tDecodeU64v(pCoder, &nCreateTbRsp) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (nCreateTbRsp) { + pRsp->aCreateTbRsp = taosArrayInit(nCreateTbRsp, sizeof(SVCreateTbRsp)); + if (pRsp->aCreateTbRsp == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t i = 0; i < nCreateTbRsp; ++i) { + SVCreateTbRsp *pCreateTbRsp = taosArrayReserve(pRsp->aCreateTbRsp, 1); + if (tDecodeSVCreateTbRsp(pCoder, pCreateTbRsp) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + } + } + + tEndDecode(pCoder); + +_exit: + if (code) { + if (pRsp->aCreateTbRsp) { + taosArrayDestroyEx(pRsp->aCreateTbRsp, NULL /* todo */); + } + } + return code; +} + +void tDestroySSubmitRsp2(SSubmitRsp2 *pRsp, int32_t flag) { + if (NULL == pRsp) { + return; + } + + if (flag & TSDB_MSG_FLG_ENCODE) { + if (pRsp->aCreateTbRsp) { + int32_t nCreateTbRsp = TARRAY_SIZE(pRsp->aCreateTbRsp); + SVCreateTbRsp *aCreateTbRsp = TARRAY_DATA(pRsp->aCreateTbRsp); + for (int32_t i = 0; i < nCreateTbRsp; ++i) { + if (aCreateTbRsp[i].pMeta) { + taosMemoryFree(aCreateTbRsp[i].pMeta); + } + } + taosArrayDestroy(pRsp->aCreateTbRsp); + } + } else if (flag & TSDB_MSG_FLG_DECODE) { + if (pRsp->aCreateTbRsp) { + int32_t nCreateTbRsp = TARRAY_SIZE(pRsp->aCreateTbRsp); + SVCreateTbRsp *aCreateTbRsp = TARRAY_DATA(pRsp->aCreateTbRsp); + for (int32_t i = 0; i < nCreateTbRsp; ++i) { + if (aCreateTbRsp[i].pMeta) { + taosMemoryFree(aCreateTbRsp[i].pMeta); + } + } + taosArrayDestroy(pRsp->aCreateTbRsp); + } + } +} diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 0d47ef1e7f70c791a32feaa6b4075bd9c625d69f..644b253cc21c243152130a488dc2ebf8ad717d01 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -298,8 +298,8 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { } static int compareKv(const void* p1, const void* p2) { - SSmlKv* kv1 = *(SSmlKv**)p1; - SSmlKv* kv2 = *(SSmlKv**)p2; + SSmlKv* kv1 = (SSmlKv*)p1; + SSmlKv* kv2 = (SSmlKv*)p2; int32_t kvLen1 = kv1->keyLen; int32_t kvLen2 = kv2->keyLen; int32_t res = strncasecmp(kv1->key, kv2->key, TMIN(kvLen1, kvLen2)); @@ -320,7 +320,7 @@ void buildChildTableName(RandTableName* rName) { taosArraySort(rName->tags, compareKv); for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) { taosStringBuilderAppendChar(&sb, ','); - SSmlKv* tagKv = taosArrayGetP(rName->tags, j); + SSmlKv* tagKv = taosArrayGet(rName->tags, j); taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); taosStringBuilderAppendChar(&sb, '='); if (IS_VAR_DATA_TYPE(tagKv->type)) { diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index b05ae602f8ee5299ed108756f73f648867d1f81a..4f8652d02cc0602583edff7f59354d9d093275fc 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -117,7 +117,7 @@ STSchema *genSTSchema(int16_t nCols) { } STSchema *pResult = NULL; - pResult = tdGetSTSChemaFromSSChema(pSchema, nCols, 1); + pResult = tBuildTSchema(pSchema, nCols, 1); taosMemoryFree(pSchema); return pResult; diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index a8103351b429d2676c2cbaa37bddb2aa17741241..a432201413c93d3df9a1898289bb326b51affd38 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -268,6 +268,11 @@ int mainWindows(int argc, char **argv) { if (dmInit() != 0) { dError("failed to init dnode since %s", terrstr()); + + taosCleanupCfg(); + taosCloseLog(); + taosCleanupArgs(); + taosConvDestroy(); return -1; } diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index b5c554e0caf83724e7e2bb8b157793765527a844..6e724f4d4322c3aee7a143b1cffaa4cae155d3e7 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -26,20 +26,20 @@ extern "C" { #endif typedef struct SVnodeMgmt { - SDnodeData *pData; - SMsgCb msgCb; - const char *path; - const char *name; - SQWorkerPool queryPool; - SQWorkerPool streamPool; - SWWorkerPool fetchPool; - SSingleWorker mgmtWorker; - SHashObj *hash; - TdThreadRwlock lock; - SVnodesStat state; - STfs *pTfs; - TdThread thread; - bool stop; + SDnodeData *pData; + SMsgCb msgCb; + const char *path; + const char *name; + SQWorkerPool queryPool; + SAutoQWorkerPool streamPool; + SWWorkerPool fetchPool; + SSingleWorker mgmtWorker; + SHashObj *hash; + TdThreadRwlock lock; + SVnodesStat state; + STfs *pTfs; + TdThread thread; + bool stop; } SVnodeMgmt; typedef struct { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 7e3915f3d1dcdfc713746aad6af1bd609231d001..202dc50ac6de75c550e651d2ad3e599b0900eedf 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -140,6 +140,12 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) { const STraceId *trace = &pMsg->info.traceId; + if (pMsg->contLen < sizeof(SMsgHead)) { + dGError("invalid rpc msg with no msg head at pCont. pMsg:%p, type:%s, contLen:%d", pMsg, TMSG_INFO(pMsg->msgType), + pMsg->contLen); + return -1; + } + SMsgHead *pHead = pMsg->pCont; int32_t code = 0; @@ -233,6 +239,14 @@ int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { + if (pRpc->contLen < sizeof(SMsgHead)) { + dError("invalid rpc msg with no msg head at pCont. pRpc:%p, type:%s, len:%d", pRpc, TMSG_INFO(pRpc->msgType), + pRpc->contLen); + rpcFreeCont(pRpc->pCont); + pRpc->pCont = NULL; + return -1; + } + SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen); if (pMsg == NULL) { rpcFreeCont(pRpc->pCont); @@ -304,7 +318,7 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { (void)tMultiWorkerInit(&pVnode->pApplyW, &acfg); pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue); - pVnode->pStreamQ = tQWorkerAllocQueue(&pMgmt->streamPool, pVnode, (FItem)vmProcessStreamQueue); + pVnode->pStreamQ = tAutoQWorkerAllocQueue(&pMgmt->streamPool, pVnode, (FItem)vmProcessStreamQueue); pVnode->pFetchQ = tWWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItems)vmProcessFetchQueue); if (pVnode->pWriteW.queue == NULL || pVnode->pSyncW.queue == NULL || pVnode->pSyncCtrlW.queue == NULL || @@ -330,7 +344,7 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ); - tQWorkerFreeQueue(&pMgmt->streamPool, pVnode->pStreamQ); + tAutoQWorkerFreeQueue(&pMgmt->streamPool, pVnode->pStreamQ); tWWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); pVnode->pQueryQ = NULL; pVnode->pStreamQ = NULL; @@ -345,11 +359,10 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) { pQPool->max = tsNumOfVnodeQueryThreads; if (tQWorkerInit(pQPool) != 0) return -1; - SQWorkerPool *pStreamPool = &pMgmt->streamPool; + SAutoQWorkerPool *pStreamPool = &pMgmt->streamPool; pStreamPool->name = "vnode-stream"; - pStreamPool->min = tsNumOfVnodeStreamThreads; - pStreamPool->max = tsNumOfVnodeStreamThreads; - if (tQWorkerInit(pStreamPool) != 0) return -1; + pStreamPool->ratio = tsRatioOfVnodeStreamThreads; + if (tAutoQWorkerInit(pStreamPool) != 0) return -1; SWWorkerPool *pFPool = &pMgmt->fetchPool; pFPool->name = "vnode-fetch"; @@ -371,7 +384,7 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) { void vmStopWorker(SVnodeMgmt *pMgmt) { tQWorkerCleanup(&pMgmt->queryPool); - tQWorkerCleanup(&pMgmt->streamPool); + tAutoQWorkerCleanup(&pMgmt->streamPool); tWWorkerCleanup(&pMgmt->fetchPool); dDebug("vnode workers are closed"); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index f11378f84ce216af5c6e8a5f8e73bc4dc46290c0..0ff41d429edf5fb1afd37d643f38462dd67128ce 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -111,12 +111,12 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { dGError("msg:%p, type:%s pCont is NULL", pRpc, TMSG_INFO(pRpc->msgType)); terrno = TSDB_CODE_INVALID_MSG_LEN; goto _OVER; - } /* else if ((pRpc->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pRpc->code == TSDB_CODE_RPC_BROKEN_LINK) && - (!IsReq(pRpc)) && (pRpc->pCont == NULL)) { - dGError("msg:%p, type:%s pCont is NULL, err: %s", pRpc, TMSG_INFO(pRpc->msgType), tstrerror(pRpc->code)); - terrno = pRpc->code; - goto _OVER; - }*/ + } else if ((pRpc->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pRpc->code == TSDB_CODE_RPC_BROKEN_LINK) && + (!IsReq(pRpc)) && (pRpc->pCont == NULL)) { + dGError("msg:%p, type:%s pCont is NULL, err: %s", pRpc, TMSG_INFO(pRpc->msgType), tstrerror(pRpc->code)); + terrno = pRpc->code; + goto _OVER; + } if (pHandle->defaultNtype == NODE_END) { dGError("msg:%p, type:%s not processed since no handle", pRpc, TMSG_INFO(pRpc->msgType)); @@ -248,8 +248,9 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { rpcRe static bool rpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_MNODE_NOT_FOUND || - code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_VND_STOPPED || - code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) { + code == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED || code == TSDB_CODE_SYN_NOT_LEADER || + code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_VND_STOPPED || code == TSDB_CODE_APP_IS_STARTING || + code == TSDB_CODE_APP_IS_STOPPING) { if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) { return false; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 4e93a1d96ee0eb3b70742eee69d892b8cb1dc451..2f824b48b4b22eb6c758d05ee30787099278a9f2 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -193,6 +193,7 @@ typedef struct { int64_t lastAccessTime; int32_t accessTimes; int32_t numOfVnodes; + int32_t numOfOtherNodes; int32_t numOfSupportVnodes; float numOfCores; int64_t memTotal; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index cf2386348ee8486984c0fbe2be1b8ea6b25ea536..b2ca1e7ad8f19c016a155e462c693fda50e47027 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -558,6 +558,10 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { goto SUBSCRIBE_OVER; } + if (mndCheckTopicPrivilege(pMnode, pMsg->info.conn.user, MND_OPER_SUBSCRIBE, pTopic) != 0) { + goto SUBSCRIBE_OVER; + } + mndReleaseTopic(pMnode, pTopic); } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 58ae85a62865d56030724bc6f555a62256ab4575..d7b16c2c8ea7cd4e351b3333551f450af77929b8 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -397,8 +397,6 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { bool reboot = (pDnode->rebootTime != statusReq.rebootTime); bool needCheck = !online || dnodeChanged || reboot; - pDnode->accessTimes++; - pDnode->lastAccessTime = curMs; const STraceId *trace = &pReq->info.traceId; mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id, pDnode->accessTimes, needCheck, online, reboot, dnodeChanged, statusReq.statusSeq); @@ -534,6 +532,8 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { pReq->info.rsp = pHead; } + pDnode->accessTimes++; + pDnode->lastAccessTime = curMs; code = 0; _OVER: diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 57f9c085ad3c2597a21029f3b1848eedf7c8fd77..61374aa0bf53a4394bcaa1da160fac628f052027 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -637,7 +637,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; // create stb for stream - if (mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user) < 0) { + if (createStreamReq.createStb == STREAM_CREATE_STABLE_TRUE && mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user) < 0) { mError("trans:%d, failed to create stb for stream %s since %s", pTrans->id, createStreamReq.name, terrstr()); mndTransDrop(pTrans); goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 071edf992fe2ca38c6c74a698d91d9e88bfbf918..7f5ebd4a9042a69def1952190b6456e03e7d44e8 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -696,14 +696,9 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { sdbRelease(pSdb, pConsumer); } -#if 0 - if (pTopic->refConsumerCnt != 0) { - mndReleaseTopic(pMnode, pTopic); - terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED; - mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) { return -1; } -#endif STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq, "drop-topic"); if (pTrans == NULL) { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 67d3f7e11983f9550d59c042d1a0b84243884d4b..718fc5c73f27e96fb261effbdc54c73310a3e924 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -957,7 +957,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { for (int32_t i = 0; i < size; ++i) { SRpcHandleInfo *pInfo = taosArrayGet(pTrans->pRpcArray, i); if (pInfo->handle != NULL) { - if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED) { code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL; } if (code == TSDB_CODE_SYN_TIMEOUT) { diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 31ab1f3259cd45b0223c5962fbca5f27386da57a..2550c68cfb408843de163b5ff6d78b1fcf726fdb 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -425,6 +425,7 @@ void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgOb static bool mndResetDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { SDnodeObj *pDnode = pObj; pDnode->numOfVnodes = 0; + pDnode->numOfOtherNodes = 0; return true; } @@ -447,7 +448,7 @@ static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2 pDnode->numOfVnodes, pDnode->numOfSupportVnodes, isMnode, online, pDnode->memAvail, pDnode->memUsed); if (isMnode) { - pDnode->numOfVnodes++; + pDnode->numOfOtherNodes++; } if (online && pDnode->numOfSupportVnodes > 0) { @@ -468,14 +469,25 @@ SArray *mndBuildDnodesArray(SMnode *pMnode, int32_t exceptDnodeId) { sdbTraverse(pSdb, SDB_DNODE, mndResetDnodesArrayFp, NULL, NULL, NULL); sdbTraverse(pSdb, SDB_DNODE, mndBuildDnodesArrayFp, pArray, &exceptDnodeId, NULL); + + mDebug("build %d dnodes array", (int32_t)taosArrayGetSize(pArray)); + for (int32_t i = 0; i < (int32_t)taosArrayGetSize(pArray); ++i) { + SDnodeObj *pDnode = taosArrayGet(pArray, i); + mDebug("dnode:%d, vnodes:%d others:%d", pDnode->id, pDnode->numOfVnodes, pDnode->numOfOtherNodes); + } return pArray; } static int32_t mndCompareDnodeId(int32_t *dnode1Id, int32_t *dnode2Id) { return *dnode1Id >= *dnode2Id ? 1 : 0; } +static float mndGetDnodeScore(SDnodeObj *pDnode, int32_t additionDnodes, float ratio) { + float totalDnodes = pDnode->numOfVnodes + (float)pDnode->numOfOtherNodes * ratio + additionDnodes; + return totalDnodes / pDnode->numOfSupportVnodes; +} + static int32_t mndCompareDnodeVnodes(SDnodeObj *pDnode1, SDnodeObj *pDnode2) { - float d1Score = (float)pDnode1->numOfVnodes / pDnode1->numOfSupportVnodes; - float d2Score = (float)pDnode2->numOfVnodes / pDnode2->numOfSupportVnodes; + float d1Score = mndGetDnodeScore(pDnode1, 0, 0.9); + float d2Score = mndGetDnodeScore(pDnode2, 0, 0.9); return d1Score >= d2Score ? 1 : 0; } @@ -494,7 +506,12 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup int32_t allocedVnodes = 0; void *pIter = NULL; + mDebug("start to sort %d dnodes", (int32_t)taosArrayGetSize(pArray)); taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); + for (int32_t i = 0; i < (int32_t)taosArrayGetSize(pArray); ++i) { + SDnodeObj *pDnode = taosArrayGet(pArray, i); + mDebug("dnode:%d, score:%f", pDnode->id, mndGetDnodeScore(pDnode, 0, 0.9)); + } int32_t size = taosArrayGetSize(pArray); if (size < pVgroup->replica) { @@ -875,7 +892,7 @@ static int32_t mndAddVnodeToVgroup(SMnode *pMnode, STrans *pTrans, SVgObj *pVgro taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { SDnodeObj *pDnode = taosArrayGet(pArray, i); - mInfo("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); + mInfo("dnode:%d, equivalent vnodes:%d others:%d", pDnode->id, pDnode->numOfVnodes, pDnode->numOfOtherNodes); } SVnodeGid *pVgid = &pVgroup->vnodeGid[pVgroup->replica]; @@ -935,7 +952,7 @@ static int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, STrans *pTrans, SVgObj * taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { SDnodeObj *pDnode = taosArrayGet(pArray, i); - mInfo("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); + mInfo("dnode:%d, equivalent vnodes:%d others:%d", pDnode->id, pDnode->numOfVnodes, pDnode->numOfOtherNodes); } int32_t code = -1; @@ -1970,16 +1987,16 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) { taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { SDnodeObj *pDnode = taosArrayGet(pArray, i); - mInfo("dnode:%d, equivalent vnodes:%d support:%d, score:%f", pDnode->id, pDnode->numOfVnodes, - pDnode->numOfSupportVnodes, (float)pDnode->numOfVnodes / pDnode->numOfSupportVnodes); + mInfo("dnode:%d, equivalent vnodes:%d others:%d support:%d, score:%f", pDnode->id, pDnode->numOfVnodes, + pDnode->numOfSupportVnodes, pDnode->numOfOtherNodes, mndGetDnodeScore(pDnode, 0, 1)); } SDnodeObj *pSrc = taosArrayGet(pArray, taosArrayGetSize(pArray) - 1); SDnodeObj *pDst = taosArrayGet(pArray, 0); - float srcScore = (float)(pSrc->numOfVnodes - 1) / pSrc->numOfSupportVnodes; - float dstScore = (float)(pDst->numOfVnodes + 1) / pDst->numOfSupportVnodes; - mInfo("trans:%d, after balance, src dnode:%d score:%f, dst dnode:%d score:%f", pTrans->id, pSrc->id, srcScore, + float srcScore = mndGetDnodeScore(pSrc, -1, 1); + float dstScore = mndGetDnodeScore(pDst, 1, 1); + mInfo("trans:%d, after balance, src dnode:%d score:%f, dst dnode:%d score:%f", pTrans->id, pSrc->id, dstScore, pDst->id, dstScore); if (srcScore > dstScore - 0.000001) { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index cbee70ad0318d190c76d89197a0096aadb4594d9..a7564e352c997085636fdfecc947fbe20032d755 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -222,11 +222,19 @@ typedef struct SSnapContext { } SSnapContext; typedef struct STqReader { - int64_t ver; - const SSubmitReq *pMsg; - SSubmitBlk *pBlock; - SSubmitMsgIter msgIter; - SSubmitBlkIter blkIter; + // const SSubmitReq *pMsg; + // SSubmitBlk *pBlock; + // SSubmitMsgIter msgIter; + // SSubmitBlkIter blkIter; + + int64_t ver; + SPackedData msg2; + + int8_t setMsg; + SSubmitReq2 submit; + int32_t nextBlk; + + int64_t lastBlkUid; SWalReader *pWalReader; @@ -251,11 +259,14 @@ int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqSeekVer(STqReader *pReader, int64_t ver); int32_t tqNextBlock(STqReader *pReader, SFetchRet *ret); -int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t ver); -bool tqNextDataBlock(STqReader *pReader); -bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); -int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader); -int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas); +int32_t tqReaderSetSubmitReq2(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); +// int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t ver); +bool tqNextDataBlock2(STqReader *pReader); +bool tqNextDataBlockFilterOut2(STqReader *pReader, SHashObj *filterOutUids); +int32_t tqRetrieveDataBlock2(SSDataBlock *pBlock, STqReader *pReader); +int32_t tqRetrieveTaosxBlock2(STqReader *pReader, SArray *blocks, SArray *schemas); +// int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader); +// int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas); int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 828341ddd8666bbb55c2cedce1b34c50f6b3362a..9a89c732bd4bb0fa595360fc72ac71a7f2fd156e 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -154,7 +154,8 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum); // tqExec -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp); +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp); +// int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp); int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision); int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp); int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry); @@ -182,7 +183,8 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore); // tqSink // void tqSinkToTableMerge(SStreamTask* pTask, void* vnode, int64_t ver, void* data); -void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* data); +// void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* data); +void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* data); // tqOffset char* tqOffsetBuildFName(const char* path, int32_t fVer); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 5a63af41afd97c74fc86e5371e0cf477df5361ee..49f8d9be3cb368a02cc15560461d2f15d937d3fb 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -70,6 +70,9 @@ typedef struct SDiskData SDiskData; typedef struct SDiskDataBuilder SDiskDataBuilder; typedef struct SBlkInfo SBlkInfo; +#define TSDBROW_ROW_FMT ((int8_t)0x0) +#define TSDBROW_COL_FMT ((int8_t)0x1) + #define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_MAX_SUBBLOCKS 8 #define TSDB_FHDR_SIZE 512 @@ -102,26 +105,29 @@ static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) { // tsdbUtil.c ============================================================================================== // TSDBROW -#define TSDBROW_TS(ROW) (((ROW)->type == 0) ? (ROW)->pTSRow->ts : (ROW)->pBlockData->aTSKEY[(ROW)->iRow]) -#define TSDBROW_VERSION(ROW) (((ROW)->type == 0) ? (ROW)->version : (ROW)->pBlockData->aVersion[(ROW)->iRow]) -#define TSDBROW_SVERSION(ROW) TD_ROW_SVER((ROW)->pTSRow) -#define TSDBROW_KEY(ROW) ((TSDBKEY){.version = TSDBROW_VERSION(ROW), .ts = TSDBROW_TS(ROW)}) -#define tsdbRowFromTSRow(VERSION, TSROW) ((TSDBROW){.type = 0, .version = (VERSION), .pTSRow = (TSROW)}) -#define tsdbRowFromBlockData(BLOCKDATA, IROW) ((TSDBROW){.type = 1, .pBlockData = (BLOCKDATA), .iRow = (IROW)}) -void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); -// int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); +#define TSDBROW_TS(ROW) (((ROW)->type == TSDBROW_ROW_FMT) ? (ROW)->pTSRow->ts : (ROW)->pBlockData->aTSKEY[(ROW)->iRow]) +#define TSDBROW_VERSION(ROW) \ + (((ROW)->type == TSDBROW_ROW_FMT) ? (ROW)->version : (ROW)->pBlockData->aVersion[(ROW)->iRow]) +#define TSDBROW_SVERSION(ROW) ((ROW)->type == TSDBROW_ROW_FMT ? (ROW)->pTSRow->sver : -1) +#define TSDBROW_KEY(ROW) ((TSDBKEY){.version = TSDBROW_VERSION(ROW), .ts = TSDBROW_TS(ROW)}) +#define tsdbRowFromTSRow(VERSION, TSROW) ((TSDBROW){.type = TSDBROW_ROW_FMT, .version = (VERSION), .pTSRow = (TSROW)}) +#define tsdbRowFromBlockData(BLOCKDATA, IROW) \ + ((TSDBROW){.type = TSDBROW_COL_FMT, .pBlockData = (BLOCKDATA), .iRow = (IROW)}) + +void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); int32_t tsdbRowCmprFn(const void *p1, const void *p2); // STSDBRowIter -void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); +int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); +void tsdbRowClose(STSDBRowIter *pIter); SColVal *tsdbRowIterNext(STSDBRowIter *pIter); // SRowMerger -int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); -int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); +int32_t tsdbRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); +int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); -int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); -void tRowMergerClear(SRowMerger *pMerger); -int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow); -int32_t tRowMergerGetRow(SRowMerger *pMerger, STSRow **ppRow); +int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); +void tsdbRowMergerClear(SRowMerger *pMerger); +int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow); +int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow); // TABLEID int32_t tTABLEIDCmprFn(const void *p1, const void *p2); // TSDBKEY @@ -146,24 +152,22 @@ int32_t tGetBlockIdx(uint8_t *p, void *ph); int32_t tCmprBlockIdx(void const *lhs, void const *rhs); int32_t tCmprBlockL(void const *lhs, void const *rhs); // SBlockData -#define tBlockDataFirstRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, 0) -#define tBlockDataLastRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1) -#define tBlockDataFirstKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataFirstRow(PBLOCKDATA)) -#define tBlockDataLastKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA)) - -int32_t tBlockDataCreate(SBlockData *pBlockData); -void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear); -int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid); -void tBlockDataReset(SBlockData *pBlockData); -int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); -void tBlockDataClear(SBlockData *pBlockData); -SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx); -void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); -int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); -int32_t tBlockDataAddColData(SBlockData *pBlockData, SColData **ppColData); -int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[], - int32_t aBufN[]); -int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]); +#define tBlockDataFirstRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, 0) +#define tBlockDataLastRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1) +#define tBlockDataFirstKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataFirstRow(PBLOCKDATA)) +#define tBlockDataLastKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA)) +#define tBlockDataGetColDataByIdx(PBLOCKDATA, IDX) (&(PBLOCKDATA)->aColData[IDX]) + +int32_t tBlockDataCreate(SBlockData *pBlockData); +void tBlockDataDestroy(SBlockData *pBlockData); +int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid); +void tBlockDataReset(SBlockData *pBlockData); +int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); +void tBlockDataClear(SBlockData *pBlockData); +void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); +int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[], + int32_t aBufN[]); +int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]); // SDiskDataHdr int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr); int32_t tGetDiskDataHdr(uint8_t *p, void *ph); @@ -335,10 +339,13 @@ struct SVersionRange { typedef struct SMemSkipListNode SMemSkipListNode; struct SMemSkipListNode { int8_t level; + int8_t flag; // TSDBROW_ROW_FMT for row format, TSDBROW_COL_FMT for col format + int32_t iRow; int64_t version; - STSRow *pTSRow; + void *pData; SMemSkipListNode *forwards[0]; }; + typedef struct SMemSkipList { int64_t size; uint32_t seed; @@ -376,11 +383,11 @@ struct SMemTable { }; struct TSDBROW { - int8_t type; // 0 for row from tsRow, 1 for row from block data + int8_t type; // TSDBROW_ROW_FMT for row from tsRow, TSDBROW_COL_FMT for row from block data union { struct { int64_t version; - STSRow *pTSRow; + SRow *pTSRow; }; struct { SBlockData *pBlockData; @@ -466,14 +473,14 @@ struct SSttBlk { // (SBlockData){.suid = suid, .uid = 0}: block data for N child tables int .last file // (SBlockData){.suid = 0, .uid = uid}: block data for 1 normal table int .last/.data file struct SBlockData { - int64_t suid; // 0 means normal table block data, otherwise child table block data - int64_t uid; // 0 means block data in .last file, otherwise in .data file - int32_t nRow; // number of rows - int64_t *aUid; // uids of each row, only exist in block data in .last file (uid == 0) - int64_t *aVersion; // versions of each row - TSKEY *aTSKEY; // timestamp of each row - int32_t nColData; - SArray *aColData; // SArray + int64_t suid; // 0 means normal table block data, otherwise child table block data + int64_t uid; // 0 means block data in .last file, otherwise in .data file + int32_t nRow; // number of rows + int64_t *aUid; // uids of each row, only exist in block data in .last file (uid == 0) + int64_t *aVersion; // versions of each row + TSKEY *aTSKEY; // timestamp of each row + int32_t nColData; + SColData *aColData; }; struct TABLEID { @@ -565,10 +572,14 @@ struct SDFileSet { }; struct STSDBRowIter { - TSDBROW *pRow; - STSchema *pTSchema; - SColVal colVal; - int32_t i; + TSDBROW *pRow; + union { + SRowIter *pIter; + struct { + int32_t iColData; + SColVal cv; + }; + }; }; struct SRowMerger { STSchema *pTSchema; @@ -739,8 +750,8 @@ typedef struct { int32_t tsdbOpenCache(STsdb *pTsdb); void tsdbCloseCache(STsdb *pTsdb); -int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb); -int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, STSRow *row, bool dup); +int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, TSDBROW *row, STsdb *pTsdb); +int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, TSDBROW *row, bool dup); int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, LRUHandle **h); int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, LRUHandle **h); int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h); @@ -752,6 +763,8 @@ int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); size_t tsdbCacheGetCapacity(SVnode *pVnode); +// int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema); + // ========== inline functions ========== static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { TSDBKEY *pKey1 = (TSDBKEY *)p1; @@ -793,8 +806,13 @@ static FORCE_INLINE TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) { } pIter->pRow = &pIter->row; - pIter->pRow->version = pIter->pNode->version; - pIter->pRow->pTSRow = pIter->pNode->pTSRow; + if (pIter->pNode->flag == TSDBROW_ROW_FMT) { + pIter->row = tsdbRowFromTSRow(pIter->pNode->version, pIter->pNode->pData); + } else if (pIter->pNode->flag == TSDBROW_COL_FMT) { + pIter->row = tsdbRowFromBlockData(pIter->pNode->pData, pIter->pNode->iRow); + } else { + ASSERT(0); + } return pIter->pRow; } diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index b2e3245fca5c9f6f5aa774684c5b9ea7863f4b0a..d160cd1facdcc0e165a888403690fcddffb3a316 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -90,6 +90,7 @@ typedef struct SCommitInfo SCommitInfo; #define VND_INFO_FNAME "vnode.json" // vnd.h + void* vnodeBufPoolMalloc(SVBufPool* pPool, int size); void* vnodeBufPoolMallocAligned(SVBufPool* pPool, int size); void vnodeBufPoolFree(SVBufPool* pPool, void* p); @@ -162,10 +163,9 @@ int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo); int32_t tsdbFinishCommit(STsdb* pTsdb); int32_t tsdbRollbackCommit(STsdb* pTsdb); int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now); -int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg); -int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); -int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, - SSubmitBlkRsp* pRsp); +int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg); +int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp); +int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows); int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey); int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg); @@ -190,7 +190,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msg int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t version, char* msg, int32_t msgLen); -int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* data, int64_t ver); +int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit); int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver); int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec); @@ -203,9 +203,9 @@ int32_t tqProcessTaskRecoverFinishReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRecoverFinishRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqCheckLogInWal(STQ* pTq, int64_t version); -SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pSchema, - SSchemaWrapper* pTagSchemaWrapper, bool createTb, int64_t suid, const char* stbFullName, - SBatchDeleteReq* pDeleteReq); +int32_t tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pSchema, + SSchemaWrapper* pTagSchemaWrapper, bool createTb, int64_t suid, const char* stbFullName, + SBatchDeleteReq* pDeleteReq, void** ppData, int32_t* pLen); // sma int32_t smaInit(); @@ -223,7 +223,7 @@ int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); int32_t tdProcessRSmaCreate(SSma* pSma, SVCreateStbReq* pReq); -int32_t tdProcessRSmaSubmit(SSma* pSma, void* pMsg, int32_t inputType); +int32_t tdProcessRSmaSubmit(SSma* pSma, int64_t version, void* pReq, void* pMsg, int32_t len, int32_t inputType); int32_t tdProcessRSmaDrop(SSma* pSma, SVDropStbReq* pReq); int32_t tdFetchTbUidList(SSma* pSma, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid); int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore, bool isAdd); diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 513ee5a1c23d140463384971e971e59bad6d6a75..893bf1b49664b5e0ac9ca8c24a1882917c795ac6 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -57,7 +57,6 @@ struct SMetaCache { TdThreadMutex lock; SHashObj* pTableEntry; SLRUCache* pUidResCache; - uint64_t keyBuf[3]; } sTagFilterResCache; }; @@ -211,7 +210,7 @@ _exit: int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo) { int32_t code = 0; - // ASSERT(metaIsWLocked(pMeta)); + // meta is wlocked for calling this func. // search SMetaCache* pCache = pMeta->pCache; @@ -222,7 +221,10 @@ int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo) { } if (*ppEntry) { // update - ASSERT(pInfo->suid == (*ppEntry)->info.suid); + if (pInfo->suid != (*ppEntry)->info.suid) { + metaError("meta/cache: suid should be same as the one in cache."); + return TSDB_CODE_FAILED; + } if (pInfo->version > (*ppEntry)->info.version) { (*ppEntry)->info.version = pInfo->version; (*ppEntry)->info.skmVer = pInfo->skmVer; @@ -341,7 +343,7 @@ _exit: int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo) { int32_t code = 0; - // ASSERT(metaIsWLocked(pMeta)); + // meta is wlocked for calling this func. // search SMetaCache* pCache = pMeta->pCache; @@ -429,20 +431,20 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK bool* acquireRes) { // generate the composed key for LRU cache SLRUCache* pCache = pMeta->pCache->sTagFilterResCache.pUidResCache; - uint64_t* pBuf = pMeta->pCache->sTagFilterResCache.keyBuf; SHashObj* pTableMap = pMeta->pCache->sTagFilterResCache.pTableEntry; TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock; + uint64_t buf[3] = {0}; uint32_t times = 0; *acquireRes = 0; - pBuf[0] = suid; - memcpy(&pBuf[1], pKey, keyLen); + buf[0] = suid; + memcpy(&buf[1], pKey, keyLen); taosThreadMutexLock(pLock); int32_t len = keyLen + sizeof(uint64_t); - LRUHandle* pHandle = taosLRUCacheLookup(pCache, pBuf, len); + LRUHandle* pHandle = taosLRUCacheLookup(pCache, buf, len); if (pHandle == NULL) { taosThreadMutexUnlock(pLock); return TSDB_CODE_SUCCESS; @@ -450,7 +452,11 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK // do some book mark work after acquiring the filter result from cache STagFilterResEntry** pEntry = taosHashGet(pTableMap, &suid, sizeof(uint64_t)); - ASSERT(pEntry != NULL); + if (NULL == pEntry) { + metaError("meta/cache: pEntry should not be NULL."); + return TSDB_CODE_FAILED; + } + *acquireRes = 1; const char* p = taosLRUCacheValue(pCache, pHandle); @@ -476,10 +482,10 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK SListNode* pNode = NULL; while ((pNode = tdListNext(&iter)) != NULL) { - memcpy(&pBuf[1], pNode->data, keyLen); + memcpy(&buf[1], pNode->data, keyLen); // check whether it is existed in LRU cache, and remove it from linked list if not. - LRUHandle* pRes = taosLRUCacheLookup(pCache, pBuf, len); + LRUHandle* pRes = taosLRUCacheLookup(pCache, buf, len); if (pRes == NULL) { // remove the item in the linked list taosArrayPush(pInvalidRes, &pNode); } else { @@ -495,7 +501,7 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK taosMemoryFree(*p1); } - atomic_store_32(&(*pEntry)->qTimes, 0); // reset the query times + atomic_store_32(&(*pEntry)->qTimes, 0); // reset the query times taosArrayDestroy(pInvalidRes); taosThreadMutexUnlock(pLock); @@ -547,14 +553,17 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int tdListAppend(&(*pEntry)->list, pKey); } - uint64_t* pBuf = pMeta->pCache->sTagFilterResCache.keyBuf; - pBuf[0] = suid; + uint64_t buf[3] = {0}; + buf[0] = suid; - memcpy(&pBuf[1], pKey, keyLen); - ASSERT(sizeof(uint64_t) + keyLen == 24); + memcpy(&buf[1], pKey, keyLen); + if (sizeof(uint64_t) + keyLen != 24) { + metaError("meta/cache: incorrect keyLen:%" PRId32 " length.", keyLen); + return TSDB_CODE_FAILED; + } // add to cache. - taosLRUCacheInsert(pCache, pBuf, sizeof(uint64_t) + keyLen, pPayload, payloadLen, freePayload, NULL, + taosLRUCacheInsert(pCache, buf, sizeof(uint64_t) + keyLen, pPayload, payloadLen, freePayload, NULL, TAOS_LRU_PRIORITY_LOW); taosThreadMutexUnlock(pLock); diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index 72f7365a1e3304c8fa5c387eea24a968ebb87c14..e50931ac0618252eea58afc0d1a734d252a05d1e 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -51,7 +51,9 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { } else if (pME->type == TSDB_TSMA_TABLE) { if (tEncodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1; } else { - ASSERT(0); + metaError("meta/entry: invalide table type: %" PRId8 " encode failed.", pME->type); + + return -1; } tEndEncode(pCoder); @@ -99,7 +101,9 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { } if (tDecodeTSma(pCoder, pME->smaEntry.tsma, true) < 0) return -1; } else { - ASSERT(0); + metaError("meta/entry: invalide table type: %" PRId8 " decode failed.", pME->type); + + return -1; } tEndDecode(pCoder); diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 1b5f74255948da71938eb274fe3cfd05700d8aa9..a78239eb565aaa9946a5561fde73a871e3b19fed 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -358,7 +358,10 @@ static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL return -1; } - ASSERT(pTagIdxKey1->type == pTagIdxKey2->type); + if (pTagIdxKey1->type != pTagIdxKey2->type) { + metaError("meta/open: incorrect tag idx type."); + return TSDB_CODE_FAILED; + } // check NULL, NULL is always the smallest if (pTagIdxKey1->isNull && !pTagIdxKey2->isNull) { diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 6c468b89f66e438d2f923a034f18f1f02282d349..0697f68f898f86b860af5b435942ce4f91bf1a51 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -611,23 +611,14 @@ tb_uid_t metaStbCursorNext(SMStbCursor *pStbCur) { } STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) { - // SMetaReader mr = {0}; STSchema *pTSchema = NULL; SSchemaWrapper *pSW = NULL; - STSchemaBuilder sb = {0}; - SSchema *pSchema; + SSchema *pSchema = NULL; pSW = metaGetTableSchema(pMeta, uid, sver, lock); if (!pSW) return NULL; - tdInitTSchemaBuilder(&sb, pSW->version); - for (int i = 0; i < pSW->nCols; i++) { - pSchema = pSW->pSchema + i; - tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes); - } - pTSchema = tdGetSchemaFromBuilder(&sb); - - tdDestroyTSchemaBuilder(&sb); + pTSchema = tBuildTSchema(pSW->pSchema, pSW->nCols, pSW->version); taosMemoryFree(pSW->pSchema); taosMemoryFree(pSW); @@ -661,7 +652,11 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv goto _exit; } - ASSERT(c); + if (c == 0) { + metaError("meta/query: incorrect c: %" PRId32 ".", c); + code = TSDB_CODE_FAILED; + goto _exit; + } if (c < 0) { tdbTbcMoveToPrev(pSkmDbC); @@ -685,7 +680,11 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv } } - ASSERT(sver > 0); + if (sver <= 0) { + metaError("meta/query: incorrect sver: %" PRId32 ".", sver); + code = TSDB_CODE_FAILED; + goto _exit; + } skmDbKey.uid = suid ? suid : uid; skmDbKey.sver = sver; @@ -708,21 +707,11 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv tdbFree(pData); // convert - STSchemaBuilder sb = {0}; - - tdInitTSchemaBuilder(&sb, pSchemaWrapper->version); - for (int i = 0; i < pSchemaWrapper->nCols; i++) { - SSchema *pSchema = pSchemaWrapper->pSchema + i; - tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes); - } - - STSchema *pTSchema = tdGetSchemaFromBuilder(&sb); + STSchema *pTSchema = tBuildTSchema(pSchemaWrapper->pSchema, pSchemaWrapper->nCols, pSchemaWrapper->version); if (pTSchema == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; } - tdDestroyTSchemaBuilder(&sb); - *ppTSchema = pTSchema; taosMemoryFree(pSchemaWrapper->pSchema); diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 054e785980370f85afd2782896cbb4095349b692..4dcb7f4ec0842d8d45bc15e29e4755f88d05129a 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -100,7 +100,10 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { break; } - ASSERT(pData && nData); + if (!pData || !nData) { + metaError("meta/snap: invalide nData: %" PRId32 " meta snap read failed.", nData); + goto _exit; + } *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + nData); if (*ppData == NULL) { @@ -113,8 +116,8 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { pHdr->size = nData; memcpy(pHdr->data, pData, nData); - metaInfo("vgId:%d, vnode snapshot meta read data, version:%" PRId64 " uid:%" PRId64 " nData:%d", - TD_VID(pReader->pMeta->pVnode), key.version, key.uid, nData); + metaDebug("vgId:%d, vnode snapshot meta read data, version:%" PRId64 " uid:%" PRId64 " blockLen:%d", + TD_VID(pReader->pMeta->pVnode), key.version, key.uid, nData); _exit: return code; @@ -356,7 +359,11 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t for (int i = 0; i < taosArrayGetSize(ctx->idList); i++) { int64_t* uid = taosArrayGet(ctx->idList, i); SIdInfo* idData = (SIdInfo*)taosHashGet(ctx->idVersion, uid, sizeof(int64_t)); - ASSERT(idData); + if (!idData) { + metaError("meta/snap: null idData"); + return TSDB_CODE_FAILED; + } + idData->index = i; metaDebug("tmqsnap init idVersion uid:%" PRIi64 " version:%" PRIi64 " index:%d", *uid, idData->version, idData->index); @@ -473,7 +480,10 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in int64_t* uidTmp = taosArrayGet(ctx->idList, ctx->index); ctx->index++; SIdInfo* idInfo = (SIdInfo*)taosHashGet(ctx->idVersion, uidTmp, sizeof(tb_uid_t)); - ASSERT(idInfo); + if (!idInfo) { + metaError("meta/snap: null idInfo"); + return TSDB_CODE_FAILED; + } *uid = *uidTmp; ret = MoveToPosition(ctx, idInfo->version, *uidTmp); @@ -507,7 +517,11 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in (ctx->subType == TOPIC_SUB_TYPE__TABLE && me.type == TSDB_CHILD_TABLE && me.ctbEntry.suid == ctx->suid)) { STableInfoForChildTable* data = (STableInfoForChildTable*)taosHashGet(ctx->suidInfo, &me.ctbEntry.suid, sizeof(tb_uid_t)); - ASSERT(data); + if (!data) { + metaError("meta/snap: null data"); + return TSDB_CODE_FAILED; + } + SVCreateTbReq req = {0}; req.type = TSDB_CHILD_TABLE; @@ -528,7 +542,8 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in } else { SArray* pTagVals = NULL; if (tTagToValArray((const STag*)p, &pTagVals) != 0) { - ASSERT(0); + metaError("meta/snap: tag to val array failed."); + return TSDB_CODE_FAILED; } int16_t nCols = taosArrayGetSize(pTagVals); for (int j = 0; j < nCols; ++j) { @@ -572,7 +587,8 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in ret = buildNormalChildTableInfo(&req, pBuf, contLen); *type = TDMT_VND_CREATE_TABLE; } else { - ASSERT(0); + metaError("meta/snap: invalid topic sub type: %" PRId8 " get meta from snap failed.", ctx->subType); + ret = -1; } tDecoderClear(&dc); @@ -593,7 +609,10 @@ SMetaTableInfo getUidfromSnapShot(SSnapContext* ctx) { int64_t* uidTmp = taosArrayGet(ctx->idList, ctx->index); ctx->index++; SIdInfo* idInfo = (SIdInfo*)taosHashGet(ctx->idVersion, uidTmp, sizeof(tb_uid_t)); - ASSERT(idInfo); + if (!idInfo) { + metaError("meta/snap: null idInfo"); + return result; + } int32_t ret = MoveToPosition(ctx, idInfo->version, *uidTmp); if (ret != 0) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 60a27bec102d78983747aa89fa1dd11c9f0c10f3..0aaf6417a82fbcce3e834aaca7f5e0f9100667c7 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -46,7 +46,7 @@ static void metaGetEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) { pInfo->suid = 0; pInfo->skmVer = pEntry->ntbEntry.schemaRow.version; } else { - ASSERT(0); + metaError("meta/table: invalide table type: %" PRId8 " get entry info failed.", pEntry->type); } } @@ -342,10 +342,18 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c); - ASSERT(ret == 0 && c == 0); + if (!(ret == 0 && c == 0)) { + metaError("meta/table: invalide ret: %" PRId32 " or c: %" PRId32 "alter stb failed.", ret, c); + return -1; + } ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); - ASSERT(ret == 0); + if (ret < 0) { + tdbTbcClose(pTbDbc); + + terrno = TSDB_CODE_TDB_STB_NOT_EXIST; + return -1; + } oStbEntry.pBuf = taosMemoryMalloc(nData); memcpy(oStbEntry.pBuf, pData, nData); @@ -558,7 +566,8 @@ static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME) { ctime = pME->ntbEntry.ctime; ttlDays = pME->ntbEntry.ttlDays; } else { - ASSERT(0); + metaError("meta/table: invalide table type: %" PRId8 " build ttl idx key failed.", pME->type); + return; } if (ttlDays <= 0) return; @@ -773,7 +782,10 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); - ASSERT(c == 0); + if (c != 0) { + metaError("meta/table: invalide c: %" PRId32 " alt tb column failed.", c); + return -1; + } tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); oversion = ((SUidIdxVal *)pData)[0].version; @@ -783,7 +795,11 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); - ASSERT(c == 0); + if (c != 0) { + metaError("meta/table: invalide c: %" PRId32 " alt tb column failed.", c); + return -1; + } + tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); // get table entry @@ -792,7 +808,11 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl memcpy(entry.pBuf, pData, nData); tDecoderInit(&dc, entry.pBuf, nData); ret = metaDecodeEntry(&dc, &entry); - ASSERT(ret == 0); + if (ret != 0) { + tDecoderClear(&dc); + metaError("meta/table: invalide ret: %" PRId32 " alt tb column failed.", ret); + return -1; + } if (entry.type != TSDB_NORMAL_TABLE) { terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; @@ -812,7 +832,11 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl if (iCol >= pSchema->nCols) break; pColumn = &pSchema->pSchema[iCol]; - ASSERT(pAlterTbReq->colName); + if (NULL == pAlterTbReq->colName) { + metaError("meta/table: null pAlterTbReq->colName"); + return -1; + } + if (strcmp(pColumn->name, pAlterTbReq->colName) == 0) break; iCol++; } @@ -964,7 +988,10 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); - ASSERT(c == 0); + if (c != 0) { + metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); + return -1; + } tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); oversion = ((SUidIdxVal *)pData)[0].version; @@ -977,7 +1004,11 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA /* get ctbEntry */ tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); - ASSERT(c == 0); + if (c != 0) { + metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); + return -1; + } + tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); ctbEntry.pBuf = taosMemoryMalloc(nData); @@ -1075,7 +1106,11 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA metaUpdateTagIdx(pMeta, &ctbEntry); } - ASSERT(ctbEntry.ctbEntry.pTags); + if (NULL == ctbEntry.ctbEntry.pTags) { + metaError("meta/table: null tags, update tag val failed."); + goto _err; + } + SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, ((STag *)(ctbEntry.ctbEntry.pTags))->len, pMeta->txn); @@ -1130,7 +1165,10 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); - ASSERT(c == 0); + if (c != 0) { + metaError("meta/table: invalide c: %" PRId32 " update tb options failed.", c); + return -1; + } tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); oversion = ((SUidIdxVal *)pData)[0].version; @@ -1140,7 +1178,11 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); - ASSERT(c == 0); + if (c != 0) { + metaError("meta/table: invalide c: %" PRId32 " update tb options failed.", c); + return -1; + } + tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); // get table entry @@ -1149,7 +1191,11 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p memcpy(entry.pBuf, pData, nData); tDecoderInit(&dc, entry.pBuf, nData); ret = metaDecodeEntry(&dc, &entry); - ASSERT(ret == 0); + if (ret != 0) { + tDecoderClear(&dc); + metaError("meta/table: invalide ret: %" PRId32 " alt tb options failed.", ret); + return -1; + } entry.version = version; metaWLock(pMeta); @@ -1408,7 +1454,8 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { } else if (pME->type == TSDB_NORMAL_TABLE) { pSW = &pME->ntbEntry.schemaRow; } else { - ASSERT(0); + metaError("meta/table: invalide table type: %" PRId8 " save skm db failed.", pME->type); + return TSDB_CODE_FAILED; } skmDbKey.uid = pME->uid; diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 65f2116a2d61b44ea0d8fd132c513c6d60de61dc..81f2285d03d236c7008df17371a081df01075871 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -563,8 +563,8 @@ static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { return TSDB_CODE_FAILED; } - SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; - // TODO: spin lock for race condition + SSubmitReq2 *pSubmitReq = (SSubmitReq2 *)pReq; + // spin lock for race condition during insert data if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { return TSDB_CODE_FAILED; } @@ -572,29 +572,19 @@ static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { return TSDB_CODE_SUCCESS; } -static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SSubmitBlkIter blkIter = {0}; - STSRow *row = NULL; +static int32_t tdFetchSubmitReqSuids(SSubmitReq2 *pMsg, STbUidStore *pStore) { + SArray *pSubmitTbData = pMsg ? pMsg->aSubmitTbData : NULL; + int32_t size = taosArrayGetSize(pSubmitTbData); terrno = TSDB_CODE_SUCCESS; - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { - return -1; - } - while (true) { - if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) { + for (int32_t i = 0; i < size; ++i) { + SSubmitTbData *pData = TARRAY_GET_ELEM(pSubmitTbData, i); + if ((terrno = tdUidStorePut(pStore, pData->suid, NULL)) < 0) { return -1; } - - if (!pBlock) break; - tdUidStorePut(pStore, msgIter.suid, NULL); } - if (terrno != TSDB_CODE_SUCCESS) { - return -1; - } return 0; } @@ -673,11 +663,11 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%d", output->info.id.uid, output->info.id.groupId, output->info.rows); - STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); - SSubmitReq *pReq = NULL; + STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); + SSubmitReq2 *pReq = NULL; // TODO: the schema update should be handled later(TD-17965) - if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, SMA_VID(pSma), suid) < 0) { + if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, output->info.id.groupId, SMA_VID(pSma), suid) < 0) { smaError("vgId:%d, build submit req for rsma table suid:%" PRIu64 ", uid:%" PRIu64 ", level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, terrstr()); @@ -685,19 +675,21 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma } if (pReq && tdProcessSubmitReq(sinkTsdb, output->info.version, pReq) < 0) { - taosMemoryFreeClear(pReq); + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); smaError("vgId:%d, process submit req for rsma suid:%" PRIu64 ", uid:%" PRIu64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, terrstr()); goto _err; } - smaDebug("vgId:%d, process submit req for rsma suid:%" PRIu64 ",uid:%" PRIu64 ", level %" PRIi8 " ver %" PRIi64 - " len %" PRIu32, - SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, output->info.version, - htonl(pReq->header.contLen)); + smaDebug("vgId:%d, process submit req for rsma suid:%" PRIu64 ",uid:%" PRIu64 ", level %" PRIi8 " ver %" PRIi64, + SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, output->info.version); - taosMemoryFreeClear(pReq); + if (pReq) { + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); + } } } @@ -715,22 +707,29 @@ _err: * @brief Copy msg to rsmaQueueBuffer for batch process * * @param pSma + * @param version * @param pMsg + * @param len * @param inputType * @param pInfo * @param suid * @return int32_t */ -static int32_t tdExecuteRSmaImplAsync(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfo *pInfo, - tb_uid_t suid) { - const SSubmitReq *pReq = (const SSubmitReq *)pMsg; +static int32_t tdExecuteRSmaImplAsync(SSma *pSma, int64_t version, const void *pMsg, int32_t len, int32_t inputType, + SRSmaInfo *pInfo, tb_uid_t suid) { + int32_t size = sizeof(int32_t) + sizeof(int64_t) + len; + void *qItem = taosAllocateQitem(size, DEF_QITEM, 0); - void *qItem = taosAllocateQitem(pReq->header.contLen, DEF_QITEM, 0); if (!qItem) { return TSDB_CODE_FAILED; } - memcpy(qItem, pMsg, pReq->header.contLen); + void *pItem = qItem; + + *(int32_t *)pItem = len; + pItem = POINTER_SHIFT(pItem, sizeof(int32_t)); + *(int64_t *)pItem = version; + memcpy(POINTER_SHIFT(pItem, sizeof(int64_t)), pMsg, len); taosWriteQitem(pInfo->queue, qItem); @@ -997,12 +996,14 @@ static FORCE_INLINE void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { * @brief async mode * * @param pSma + * @param version * @param pMsg * @param inputType * @param suid * @return int32_t */ -static int32_t tdExecuteRSmaAsync(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) { +static int32_t tdExecuteRSmaAsync(SSma *pSma, int64_t version, const void *pMsg, int32_t len, int32_t inputType, + tb_uid_t suid) { SRSmaInfo *pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, suid); if (!pRSmaInfo) { smaDebug("vgId:%d, execute rsma, no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); @@ -1010,7 +1011,7 @@ static int32_t tdExecuteRSmaAsync(SSma *pSma, const void *pMsg, int32_t inputTyp } if (inputType == STREAM_INPUT__DATA_SUBMIT) { - if (tdExecuteRSmaImplAsync(pSma, pMsg, inputType, pRSmaInfo, suid) < 0) { + if (tdExecuteRSmaImplAsync(pSma, version, pMsg, len, inputType, pRSmaInfo, suid) < 0) { tdReleaseRSmaInfo(pSma, pRSmaInfo); return TSDB_CODE_FAILED; } @@ -1036,7 +1037,7 @@ static int32_t tdExecuteRSmaAsync(SSma *pSma, const void *pMsg, int32_t inputTyp return TSDB_CODE_SUCCESS; } -int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { +int32_t tdProcessRSmaSubmit(SSma *pSma, int64_t version, void *pReq, void *pMsg, int32_t len, int32_t inputType) { SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); if (!pEnv) { // only applicable when rsma env exists @@ -1046,19 +1047,22 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { STbUidStore uidStore = {0}; if (inputType == STREAM_INPUT__DATA_SUBMIT) { - if (tdFetchSubmitReqSuids(pMsg, &uidStore) < 0) { + if (tdFetchSubmitReqSuids(pReq, &uidStore) < 0) { + smaError("vgId:%d, failed to process rsma submit fetch suid since: %s", SMA_VID(pSma), terrstr()); goto _err; } if (uidStore.suid != 0) { - if (tdExecuteRSmaAsync(pSma, pMsg, inputType, uidStore.suid) < 0) { + if (tdExecuteRSmaAsync(pSma, version, pMsg, len, inputType, uidStore.suid) < 0) { + smaError("vgId:%d, failed to process rsma submit exec 1 since: %s", SMA_VID(pSma), terrstr()); goto _err; } void *pIter = NULL; while ((pIter = taosHashIterate(uidStore.uidHash, pIter))) { tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - if (tdExecuteRSmaAsync(pSma, pMsg, inputType, *pTbSuid) < 0) { + if (tdExecuteRSmaAsync(pSma, version, pMsg, len, inputType, *pTbSuid) < 0) { + smaError("vgId:%d, failed to process rsma submit exec 2 since: %s", SMA_VID(pSma), terrstr()); goto _err; } } @@ -1068,7 +1072,6 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { return TSDB_CODE_SUCCESS; _err: tdUidStoreDestory(&uidStore); - smaError("vgId:%d, failed to process rsma submit since: %s", SMA_VID(pSma), terrstr()); return TSDB_CODE_FAILED; } @@ -1428,7 +1431,8 @@ _end: static void tdFreeRSmaSubmitItems(SArray *pItems) { for (int32_t i = 0; i < taosArrayGetSize(pItems); ++i) { - taosFreeQitem(*(void **)taosArrayGet(pItems, i)); + SPackedData *packData = taosArrayGet(pItems, i); + taosFreeQitem(POINTER_SHIFT(packData->msgStr, -sizeof(int32_t) - sizeof(int64_t))); } taosArrayClear(pItems); } @@ -1497,8 +1501,12 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA void *msg = NULL; taosGetQitem(qall, (void **)&msg); if (msg) { - if (!taosArrayPush(pSubmitArr, &msg)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + SPackedData packData = {.msgLen = *(int32_t *)msg, + .ver = *(int64_t *)POINTER_SHIFT(msg, sizeof(int32_t)), + .msgStr = POINTER_SHIFT(msg, sizeof(int32_t) + sizeof(int64_t))}; + + if (!taosArrayPush(pSubmitArr, &packData)) { + tdFreeRSmaSubmitItems(pSubmitArr); goto _err; } } else { diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 9ce4a2077171b25ec3d98d970db99fee09aa1240..e633d93de69465673a08e530bdc647c9f372a87a 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -217,19 +217,18 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char } SBatchDeleteReq deleteReq = {0}; - SSubmitReq *pSubmitReq = - tqBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true, - pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq); - // TODO deleteReq - taosArrayDestroy(deleteReq.deleteReqs); - + void *pSubmitReq = NULL; + int32_t contLen = 0; - if (!pSubmitReq) { - smaError("vgId:%d, failed to gen submit blk while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), + if (tqBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true, + pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq, &pSubmitReq, &contLen) < 0) { + smaError("vgId:%d, failed to gen submit msg while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid, tstrerror(terrno)); goto _err; } - + + // TODO deleteReq + taosArrayDestroy(deleteReq.deleteReqs); #if 0 if (!strncasecmp("td.tsma.rst.tb", pTsmaStat->pTSma->dstTbName, 14)) { terrno = TSDB_CODE_APP_ERROR; @@ -242,7 +241,7 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char SRpcMsg submitReqMsg = { .msgType = TDMT_VND_SUBMIT, .pCont = pSubmitReq, - .contLen = ntohl(pSubmitReq->length), + .contLen = ntohl(contLen), }; if (tmsgPutToQueue(&pSma->pVnode->msgCb, WRITE_QUEUE, &submitReqMsg) < 0) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ee00e2961624cc62a7d19ce3f509e02e263c3080..d0017e4ba227d197d72a88a487bac26e4868c3f5 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -677,9 +677,12 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { req.epoch, TD_VID(pTq->pVnode), fetchVer, pHead->msgType); if (pHead->msgType == TDMT_VND_SUBMIT) { - SSubmitReq* pCont = (SSubmitReq*)&pHead->body; - - if (tqTaosxScanLog(pTq, pHandle, pCont, &taosxRsp) < 0) { + SPackedData submit = { + .msgStr = POINTER_SHIFT(pHead->body, sizeof(SMsgHead)), + .msgLen = pHead->bodyLen - sizeof(SMsgHead), + .ver = pHead->version, + }; + if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp) < 0) { } if (taosxRsp.blockNum > 0 /* threshold */) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); @@ -731,6 +734,7 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t version, char* msg, int32_t msgL STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey)); if (pHandle) { + // walCloseRef(pHandle->pWalReader->pWal, pHandle->pRef->refId); if (pHandle->pRef) { walCloseRef(pTq->pVnode->pWal, pHandle->pRef->refId); } @@ -954,14 +958,14 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { pTask->smaSink.smaSink = smaHandleRes; } else if (pTask->outputType == TASK_OUTPUT__TABLE) { pTask->tbSink.vnode = pTq->pVnode; - pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline; + pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline2; /*A(pTask->tbSink.pSchemaWrapper);*/ /*A(pTask->tbSink.pSchemaWrapper->pSchema);*/ pTask->tbSink.pTSchema = - tdGetSTSChemaFromSSChema(pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols, 1); - /*A(pTask->tbSink.pTSchema);*/ + tBuildTSchema(pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols, 1); + ASSERT(pTask->tbSink.pTSchema); } streamSetupTrigger(pTask); @@ -1010,6 +1014,7 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { tqError("unable to encode rsp %d", __LINE__); return -1; } + void* buf = rpcMallocCont(sizeof(SMsgHead) + len); ((SMsgHead*)buf)->vgId = htonl(req.upstreamNodeId); @@ -1334,12 +1339,12 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { return 0; } -int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* pReq, int64_t ver) { - void* pIter = NULL; - bool failed = false; - SStreamDataSubmit* pSubmit = NULL; +int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { + void* pIter = NULL; + bool failed = false; + SStreamDataSubmit2* pSubmit = NULL; - pSubmit = streamDataSubmitNew(pReq); + pSubmit = streamDataSubmitNew(submit); if (pSubmit == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to create data submit for stream since out of memory"); @@ -1356,7 +1361,7 @@ int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* pReq, int64_t ver) { continue; } - tqDebug("data submit enqueue stream task: %d, ver: %" PRId64, pTask->taskId, ver); + tqDebug("data submit enqueue stream task: %d, ver: %" PRId64, pTask->taskId, submit.ver); if (!failed) { if (streamTaskInput(pTask, (SStreamQueueItem*)pSubmit) < 0) { diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 9a56df7cb0f535852c35fab8b6a5d56b5c7a865c..40a82cc8e82edbab0a776f0f5f69d6013b5e2de3 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -113,14 +113,8 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs return -1; } - if (pRsp->withTbName) { - if (pRsp->rspOffset.type == TMQ_OFFSET__LOG) { - int64_t uid = pExec->pExecReader->msgIter.uid; - tqAddTbNameToRsp(pTq, uid, pRsp, 1); - } else { - pRsp->withTbName = false; - } - } + ASSERT(pRsp->withTbName == false); + ASSERT(pRsp->withSchema == false); return 0; } @@ -157,9 +151,8 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta if (pDataBlock != NULL) { if (pRsp->withTbName) { - int64_t uid = 0; if (pOffset->type == TMQ_OFFSET__LOG) { - uid = pExec->pExecReader->msgIter.uid; + int64_t uid = pExec->pExecReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp, 1) < 0) { continue; } @@ -229,7 +222,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta return 0; } -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp) { +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp) { STqExecHandle* pExec = &pHandle->execHandle; /*A(pExec->subType != TOPIC_SUB_TYPE__COLUMN);*/ @@ -238,8 +231,9 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { STqReader* pReader = pExec->pExecReader; - tqReaderSetDataMsg(pReader, pReq, 0); - while (tqNextDataBlock(pReader)) { + /*tqReaderSetDataMsg(pReader, pReq, 0);*/ + tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver); + while (tqNextDataBlock2(pReader)) { /*SSDataBlock block = {0};*/ /*if (tqRetrieveDataBlock(&block, pReader) < 0) {*/ /*if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;*/ @@ -247,11 +241,12 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp taosArrayClear(pBlocks); taosArrayClear(pSchemas); - if (tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas) < 0) { + if (tqRetrieveTaosxBlock2(pReader, pBlocks, pSchemas) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; } if (pRsp->withTbName) { - int64_t uid = pExec->pExecReader->msgIter.uid; + /*int64_t uid = pExec->pExecReader->msgIter.uid;*/ + int64_t uid = pExec->pExecReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlocks)) < 0) { taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper); @@ -261,7 +256,9 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp } } if (pHandle->fetchMeta) { +#if 0 SSubmitBlk* pBlk = pReader->pBlock; + int64_t uid = pExec->pExecReader->lastBlkUid; int32_t schemaLen = htonl(pBlk->schemaLen); if (schemaLen > 0) { if (pRsp->createTableNum == 0) { @@ -274,6 +271,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp taosArrayPush(pRsp->createTableReq, &createReq); pRsp->createTableNum++; } +#endif } for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) { SSDataBlock* pBlock = taosArrayGet(pBlocks, i); @@ -287,19 +285,20 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp } } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { STqReader* pReader = pExec->pExecReader; - tqReaderSetDataMsg(pReader, pReq, 0); - while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) { + /*tqReaderSetDataMsg(pReader, pReq, 0);*/ + tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver); + while (tqNextDataBlockFilterOut2(pReader, pExec->execDb.pFilterOutTbUid)) { /*SSDataBlock block = {0};*/ /*if (tqRetrieveDataBlock(&block, pReader) < 0) {*/ /*if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;*/ /*}*/ taosArrayClear(pBlocks); taosArrayClear(pSchemas); - if (tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas) < 0) { + if (tqRetrieveTaosxBlock2(pReader, pBlocks, pSchemas) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; } if (pRsp->withTbName) { - int64_t uid = pExec->pExecReader->msgIter.uid; + int64_t uid = pExec->pExecReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlocks)) < 0) { taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper); @@ -309,6 +308,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp } } if (pHandle->fetchMeta) { +#if 0 SSubmitBlk* pBlk = pReader->pBlock; int32_t schemaLen = htonl(pBlk->schemaLen); if (schemaLen > 0) { @@ -322,6 +322,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp taosArrayPush(pRsp->createTableReq, &createReq); pRsp->createTableNum++; } +#endif } /*tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock),*/ /*pTq->pVnode->config.tsdbCfg.precision);*/ diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 63c3c25218c0532e58c1b52e0ac2ace30eef3d11..559a3b76fec3a532d2e40fe62a203e515ce38252 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -207,7 +207,11 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ #endif int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { - tqDebug("vgId:%d, tq push msg ver %" PRId64 ", type: %s", pTq->pVnode->config.vgId, ver, TMSG_INFO(msgType)); + void* pReq = POINTER_SHIFT(msg, sizeof(SMsgHead)); + int32_t len = msgLen - sizeof(SMsgHead); + + tqDebug("vgId:%d, tq push msg ver %" PRId64 ", type: %s, p head %p, p body %p, len %d", pTq->pVnode->config.vgId, ver, + TMSG_INFO(msgType), msg, pReq, len); if (msgType == TDMT_VND_SUBMIT) { // lock push mgr to avoid potential msg lost @@ -216,7 +220,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) if (taosHashGetSize(pTq->pPushMgr) != 0) { SArray* cachedKeys = taosArrayInit(0, sizeof(void*)); SArray* cachedKeyLens = taosArrayInit(0, sizeof(size_t)); - void* data = taosMemoryMalloc(msgLen); + void* data = taosMemoryMalloc(len); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to copy data for stream since out of memory"); @@ -224,9 +228,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) taosArrayDestroy(cachedKeyLens); return -1; } - memcpy(data, msg, msgLen); - SSubmitReq* pReq = (SSubmitReq*)data; - pReq->version = ver; + memcpy(data, pReq, len); void* pIter = NULL; while (1) { @@ -250,7 +252,12 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) SMqDataRsp* pRsp = &pPushEntry->dataRsp; // prepare scan mem data - qStreamScanMemData(task, pReq); + SPackedData submit = { + .msgStr = data, + .msgLen = len, + .ver = ver, + }; + qStreamSetScanMemData(task, submit); // exec while (1) { @@ -304,17 +311,22 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) if (vnodeIsRoleLeader(pTq->pVnode)) { if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) return 0; if (msgType == TDMT_VND_SUBMIT) { - void* data = taosMemoryMalloc(msgLen); + void* data = taosMemoryMalloc(len); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to copy data for stream since out of memory"); return -1; } - memcpy(data, msg, msgLen); - SSubmitReq* pReq = (SSubmitReq*)data; - pReq->version = ver; + memcpy(data, pReq, len); + SPackedData submit = { + .msgStr = data, + .msgLen = len, + .ver = ver, + }; + + tqDebug("tq copy write msg %p %d %" PRId64 " from %p", data, len, ver, pReq); - tqProcessSubmitReq(pTq, data, ver); + tqProcessSubmitReq(pTq, submit); } if (msgType == TDMT_VND_DELETE) { tqProcessDelReq(pTq, POINTER_SHIFT(msg, sizeof(SMsgHead)), msgLen - sizeof(SMsgHead), ver); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index e5392bbd0263e724617fbba47ac28924bc9bc8e4..eb9c0c3eeb252b53caf2b04a0674e879242cca42 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -247,7 +247,7 @@ END: } STqReader* tqOpenReader(SVnode* pVnode) { - STqReader* pReader = taosMemoryMalloc(sizeof(STqReader)); + STqReader* pReader = taosMemoryCalloc(1, sizeof(STqReader)); if (pReader == NULL) { return NULL; } @@ -259,7 +259,7 @@ STqReader* tqOpenReader(SVnode* pVnode) { } pReader->pVnodeMeta = pVnode->pMeta; - pReader->pMsg = NULL; + /*pReader->pMsg = NULL;*/ pReader->ver = -1; pReader->pColIdList = NULL; pReader->cachedSchemaVer = 0; @@ -298,7 +298,7 @@ int32_t tqSeekVer(STqReader* pReader, int64_t ver) { } int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { - bool fromProcessedMsg = pReader->pMsg != NULL; + bool fromProcessedMsg = pReader->msg2.msgStr != NULL; while (1) { if (!fromProcessedMsg) { @@ -311,7 +311,9 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { tqDebug("return offset %" PRId64 ", no more valid", ret->offset.version); return -1; } - void* body = pReader->pWalReader->pHead->head.body; + void* body = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SMsgHead)); + int32_t bodyLen = pReader->pWalReader->pHead->head.bodyLen - sizeof(SMsgHead); + int64_t ver = pReader->pWalReader->pHead->head.version; #if 0 if (pReader->pWalReader->pHead->head.msgType != TDMT_VND_SUBMIT) { // TODO do filter @@ -320,16 +322,17 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { return 0; } else { #endif - tqReaderSetDataMsg(pReader, body, pReader->pWalReader->pHead->head.version); + tqReaderSetSubmitReq2(pReader, body, bodyLen, ver); + /*tqReaderSetDataMsg(pReader, body, pReader->pWalReader->pHead->head.version);*/ #if 0 } #endif } - while (tqNextDataBlock(pReader)) { + while (tqNextDataBlock2(pReader)) { // TODO mem free memset(&ret->data, 0, sizeof(SSDataBlock)); - int32_t code = tqRetrieveDataBlock(&ret->data, pReader); + int32_t code = tqRetrieveDataBlock2(&ret->data, pReader); if (code != 0 || ret->data.info.rows == 0) { continue; } @@ -348,6 +351,7 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { } } +#if 0 int32_t tqReaderSetDataMsg(STqReader* pReader, const SSubmitReq* pMsg, int64_t ver) { pReader->pMsg = pMsg; @@ -362,7 +366,33 @@ int32_t tqReaderSetDataMsg(STqReader* pReader, const SSubmitReq* pMsg, int64_t v memset(&pReader->blkIter, 0, sizeof(SSubmitBlkIter)); return 0; } +#endif + +int32_t tqReaderSetSubmitReq2(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) { + ASSERT(pReader->msg2.msgStr == NULL); + ASSERT(msgStr); + ASSERT(msgLen); + ASSERT(ver >= 0); + pReader->msg2.msgStr = msgStr; + pReader->msg2.msgLen = msgLen; + pReader->msg2.ver = ver; + pReader->ver = ver; + + tqDebug("tq reader set msg %p %d", msgStr, msgLen); + if (pReader->setMsg == 0) { + SDecoder decoder; + tDecoderInit(&decoder, pReader->msg2.msgStr, pReader->msg2.msgLen); + if (tDecodeSSubmitReq2(&decoder, &pReader->submit) < 0) { + ASSERT(0); + } + tDecoderClear(&decoder); + pReader->setMsg = 1; + } + return 0; +} + +#if 0 bool tqNextDataBlock(STqReader* pReader) { if (pReader->pMsg == NULL) return false; while (1) { @@ -386,6 +416,59 @@ bool tqNextDataBlock(STqReader* pReader) { } return false; } +#endif + +bool tqNextDataBlock2(STqReader* pReader) { + if (pReader->msg2.msgStr == NULL) return false; + ASSERT(pReader->setMsg == 1); + + tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg2.msgStr, pReader->msg2.msgLen, + pReader->msg2.ver, pReader->nextBlk); + + int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); + while (pReader->nextBlk < blockSz) { + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); + ASSERT(pSubmitTbData->uid); + + if (pReader->tbIdHash == NULL) return true; + + void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); + if (ret != NULL) { + return true; + } + pReader->nextBlk++; + } + + tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); + pReader->setMsg = 0; + pReader->nextBlk = 0; + pReader->msg2.msgStr = NULL; + + return false; +} + +bool tqNextDataBlockFilterOut2(STqReader* pReader, SHashObj* filterOutUids) { + if (pReader->msg2.msgStr == NULL) return false; + ASSERT(pReader->setMsg == 1); + + int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); + while (pReader->nextBlk < blockSz) { + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); + if (pReader->tbIdHash == NULL) return true; + + void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); + if (ret == NULL) { + return true; + } + } + + tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); + pReader->setMsg = 0; + pReader->nextBlk = 0; + pReader->msg2.msgStr = NULL; + + return false; +} int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrapper* pSrc, char* mask) { int32_t code; @@ -416,6 +499,7 @@ int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrap return 0; } +#if 0 bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) { while (1) { if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { @@ -431,6 +515,253 @@ bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) { return false; } +int32_t tqScanSubmitSplit(SArray* pBlocks, SArray* schemas, STqReader* pReader) { + // + int32_t sversion = htonl(pReader->pBlock->sversion); + if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || + pReader->cachedSchemaSuid != pReader->msgIter.suid) { + taosMemoryFree(pReader->pSchema); + pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); + if (pReader->pSchema == NULL) { + tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 + "), version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, pReader->msgIter.uid, pReader->msgIter.suid, sversion); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + tDeleteSSchemaWrapper(pReader->pSchemaWrapper); + pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); + if (pReader->pSchemaWrapper == NULL) { + tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, pReader->msgIter.uid, pReader->cachedSchemaVer); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + STSchema* pTschema = pReader->pSchema; + SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; + + int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList); + } + return 0; +} +#endif + +int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader) { + int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); + ASSERT(pReader->nextBlk < blockSz); + + tqDebug("tq reader retrieve data block %p, %d", pReader->msg2.msgStr, pReader->nextBlk); + + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); + pReader->nextBlk++; + + int32_t sversion = pSubmitTbData->sver; + int64_t suid = pSubmitTbData->suid; + int64_t uid = pSubmitTbData->uid; + pReader->lastBlkUid = uid; + + pBlock->info.id.uid = uid; + pBlock->info.version = pReader->msg2.ver; + + if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || pReader->cachedSchemaSuid != suid) { + taosMemoryFree(pReader->pSchema); + pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, uid, sversion, 1); + if (pReader->pSchema == NULL) { + tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 + "), version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, uid, suid, sversion); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + tDeleteSSchemaWrapper(pReader->pSchemaWrapper); + pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1); + if (pReader->pSchemaWrapper == NULL) { + tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, uid, pReader->cachedSchemaVer); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + STSchema* pTschema = pReader->pSchema; + SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; + + int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList); + + if (colNumNeed == 0) { + int32_t colMeta = 0; + while (colMeta < pSchemaWrapper->nCols) { + SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { + goto FAIL; + } + colMeta++; + } + } else { + if (colNumNeed > pSchemaWrapper->nCols) { + colNumNeed = pSchemaWrapper->nCols; + } + + int32_t colMeta = 0; + int32_t colNeed = 0; + while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { + SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; + col_id_t colIdSchema = pColSchema->colId; + col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pReader->pColIdList, colNeed); + if (colIdSchema < colIdNeed) { + colMeta++; + } else if (colIdSchema > colIdNeed) { + colNeed++; + } else { + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { + goto FAIL; + } + colMeta++; + colNeed++; + } + } + } + + int32_t numOfRows = 0; + + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + SArray* pCols = pSubmitTbData->aCol; + SColData* pCol = taosArrayGet(pCols, 0); + numOfRows = pCol->nVal; + } else { + SArray* pRows = pSubmitTbData->aRowP; + numOfRows = taosArrayGetSize(pRows); + } + + if (blockDataEnsureCapacity(pBlock, numOfRows) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto FAIL; + } + pBlock->info.rows = numOfRows; + + int32_t colActual = blockDataGetNumOfCols(pBlock); + + // convert and scan one block + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + SArray* pCols = pSubmitTbData->aCol; + int32_t numOfCols = taosArrayGetSize(pCols); + int32_t targetIdx = 0; + int32_t sourceIdx = 0; + while (targetIdx < colActual) { + ASSERT(sourceIdx < numOfCols); + + SColData* pCol = taosArrayGet(pCols, sourceIdx); + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx); + SColVal colVal; + + ASSERT(pCol->nVal == numOfRows); + + if (pCol->cid < pColData->info.colId) { + sourceIdx++; + } else if (pCol->cid == pColData->info.colId) { + for (int32_t i = 0; i < pCol->nVal; i++) { + tColDataGetValue(pCol, sourceIdx, &colVal); +#if 0 + void* val = NULL; + if (IS_STR_DATA_TYPE(colVal.type)) { + val = colVal.value.pData; + } else { + val = &colVal.value.val; + } + if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } +#endif + if (IS_STR_DATA_TYPE(colVal.type)) { + if (colVal.value.pData != NULL) { + char val[65535 + 2]; + memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); + varDataSetLen(val, colVal.value.nData); + if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } else { + colDataAppendNULL(pColData, i); + } + } else { + if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } + } + sourceIdx++; + targetIdx++; + } else { + ASSERT(0); + } + } + } else { + SArray* pRows = pSubmitTbData->aRowP; + + for (int32_t i = 0; i < numOfRows; i++) { + SRow* pRow = taosArrayGetP(pRows, i); + int32_t targetIdx = 0; + int32_t sourceIdx = 0; + + for (int32_t j = 0; j < colActual; j++) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, j); + while (1) { + ASSERT(sourceIdx < pTschema->numOfCols); + + SColVal colVal; + tRowGet(pRow, pTschema, sourceIdx, &colVal); + if (colVal.cid < pColData->info.colId) { + sourceIdx++; + continue; + } else if (colVal.cid == pColData->info.colId) { + if (IS_STR_DATA_TYPE(colVal.type)) { + if (colVal.value.pData != NULL) { + char val[65535 + 2]; + memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); + varDataSetLen(val, colVal.value.nData); + if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } else { + colDataAppendNULL(pColData, i); + } + /*val = colVal.value.pData;*/ + } else { + if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } + + sourceIdx++; + targetIdx++; + break; + } else { + ASSERT(0); + } + } + } + } + } + } + + return 0; + +FAIL: + blockDataFreeRes(pBlock); + return -1; +} + +#if 0 int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { // TODO: cache multiple schema int32_t sversion = htonl(pReader->pBlock->sversion); @@ -673,6 +1004,18 @@ FAIL: taosMemoryFree(assigned); return -1; } +#endif + +int32_t tqRetrieveTaosxBlock2(STqReader* pReader, SArray* blocks, SArray* schemas) { + SSDataBlock block = {0}; + if (tqRetrieveDataBlock2(&block, pReader) == 0) { + taosArrayPush(blocks, &block); + SSchemaWrapper* pSW = tCloneSSchemaWrapper(pReader->pSchemaWrapper); + taosArrayPush(schemas, &pSW); + return 0; + } + return -1; +} void tqReaderSetColIdList(STqReader* pReader, SArray* pColIdList) { pReader->pColIdList = pColIdList; } diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 078b50db680d05e1127f54d5bd94fe71459abd4c..e8f2555f177470b6aef444ee28ccb2a8b6e0b53d 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -71,6 +71,7 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBl return 0; } +#if 0 SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pTSchema, SSchemaWrapper* pTagSchemaWrapper, bool createTb, int64_t suid, const char* stbFullName, SBatchDeleteReq* pDeleteReq) { @@ -251,8 +252,6 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem int32_t rows = pDataBlock->info.rows; - tqDebug("tq sink, convert block1 %d, rows: %d", i, rows); - int32_t dataLen = 0; int32_t schemaLen = 0; void* blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); @@ -298,6 +297,201 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem return ret; } +#endif + +int32_t tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pTSchema, + SSchemaWrapper* pTagSchemaWrapper, bool createTb, int64_t suid, const char* stbFullName, + SBatchDeleteReq* pDeleteReq, void** ppData, int32_t* pLen) { + void* pBuf = NULL; + int32_t len = 0; + SSubmitReq2* pReq = NULL; + SArray* tagArray = NULL; + SArray* createTbArray = NULL; + SArray* pVals = NULL; + + int32_t sz = taosArrayGetSize(pBlocks); + + if (!(tagArray = taosArrayInit(1, sizeof(STagVal)))) { + goto _end; + } + + if (!(createTbArray = taosArrayInit(sz, POINTER_BYTES))) { + goto _end; + } + + if (!(pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { + goto _end; + } + + // create table req + if (createTb) { + for (int32_t i = 0; i < sz; ++i) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + SVCreateTbReq* pCreateTbReq = NULL; + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + taosArrayPush(createTbArray, &pCreateTbReq); + continue; + } + + if (!(pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq)))) { + goto _end; + }; + + // don't move to the end of loop as to destroy in the end of func when error occur + taosArrayPush(createTbArray, &pCreateTbReq); + + // set const + pCreateTbReq->flags = 0; + pCreateTbReq->type = TSDB_CHILD_TABLE; + pCreateTbReq->ctb.suid = suid; + + // set super table name + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + pCreateTbReq->ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName); + + // set tag content + taosArrayClear(tagArray); + STagVal tagVal = { + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.id.groupId, + }; + taosArrayPush(tagArray, &tagVal); + pCreateTbReq->ctb.tagNum = taosArrayGetSize(tagArray); + + STag* pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + if (pTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + pCreateTbReq->ctb.pTag = (uint8_t*)pTag; + + // set tag name + SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = {0}; + strcpy(tagNameStr, "group_id"); + taosArrayPush(tagName, tagNameStr); + pCreateTbReq->ctb.tagName = tagName; + + // set table name + if (pDataBlock->info.parTbName[0]) { + pCreateTbReq->name = strdup(pDataBlock->info.parTbName); + } else { + pCreateTbReq->name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId); + } + } + } + + // SSubmitTbData req + for (int32_t i = 0; i < sz; ++i) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + pDeleteReq->suid = suid; + pDeleteReq->deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); + tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, pDeleteReq); + continue; + } + + int32_t rows = pDataBlock->info.rows; + + SSubmitTbData* pTbData = (SSubmitTbData*)taosMemoryCalloc(1, sizeof(SSubmitTbData)); + if (!pTbData) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + if (!(pTbData->aRowP = taosArrayInit(rows, sizeof(SRow*)))) { + taosMemoryFree(pTbData); + goto _end; + } + pTbData->suid = suid; + pTbData->uid = 0; // uid is assigned by vnode + pTbData->sver = pTSchema->version; + + if (createTb) { + pTbData->pCreateTbReq = taosArrayGetP(createTbArray, i); + if (pTbData->pCreateTbReq) pTbData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; + } + + if (!pVals && !(pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)))) { + taosArrayDestroy(pTbData->aRowP); + taosMemoryFree(pTbData); + goto _end; + } + + for (int32_t j = 0; j < rows; j++) { + taosArrayClear(pVals); + for (int32_t k = 0; k < pTSchema->numOfCols; k++) { + const STColumn* pCol = &pTSchema->columns[k]; + SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, k); + if (colDataIsNull_s(pColData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + void* data = colDataGetData(pColData, j); + if (IS_STR_DATA_TYPE(pCol->type)) { + SValue sv = (SValue){.nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } else { + SValue sv; + memcpy(&sv.val, data, tDataTypes[pCol->type].bytes); + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } + } + } + SRow* pRow = NULL; + if ((terrno = tRowBuild(pVals, (STSchema*)pTSchema, &pRow)) < 0) { + tDestroySSubmitTbData(pTbData, TSDB_MSG_FLG_ENCODE); + goto _end; + } + ASSERT(pRow); + taosArrayPush(pTbData->aRowP, &pRow); + } + + taosArrayPush(pReq->aSubmitTbData, pTbData); + } + + // encode + tEncodeSize(tEncodeSSubmitReq2, pReq, len, terrno); + if (TSDB_CODE_SUCCESS == terrno) { + SEncoder encoder; + len += sizeof(SMsgHead); + pBuf = rpcMallocCont(len); + if (NULL == pBuf) { + goto _end; + } + ((SMsgHead*)pBuf)->vgId = htonl(TD_VID(pVnode)); + ((SMsgHead*)pBuf)->contLen = htonl(len); + tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SMsgHead)), len - sizeof(SMsgHead)); + if (tEncodeSSubmitReq2(&encoder, pReq) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("failed to encode submit req since %s", terrstr()); + } + tEncoderClear(&encoder); + } +_end: + taosArrayDestroy(tagArray); + taosArrayDestroy(pVals); + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + + if (terrno != 0) { + rpcFreeCont(pBuf); + taosArrayDestroy(pDeleteReq->deleteReqs); + return TSDB_CODE_FAILED; + } + if (ppData) *ppData = pBuf; + if (pLen) *pLen = len; + return TSDB_CODE_SUCCESS; +} void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { const SArray* pBlocks = (const SArray*)data; @@ -492,7 +686,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d blkHead->uid = 0; blkHead->schemaLen = 0; - tqDebug("tq sink, convert block2 %d, rows: %d", i, rows); + tqDebug("tq sink pipe1, convert block2 %d, rows: %d", i, rows); int32_t dataLen = 0; void* blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); @@ -521,7 +715,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d } else { void* colData = colDataGetData(pColData, j); if (k == 0) { - tqDebug("tq sink, row %d ts %" PRId64, j, *(int64_t*)colData); + tqDebug("tq sink pipe1, row %d ts %" PRId64, j, *(int64_t*)colData); } tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, pColumn->offset, k); } @@ -550,6 +744,247 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d taosArrayDestroy(tagArray); } +void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { + const SArray* pBlocks = (const SArray*)data; + SVnode* pVnode = (SVnode*)vnode; + int64_t suid = pTask->tbSink.stbUid; + char* stbFullName = pTask->tbSink.stbFullName; + STSchema* pTSchema = pTask->tbSink.pTSchema; + /*SSchemaWrapper* pSchemaWrapper = pTask->tbSink.pSchemaWrapper;*/ + + int32_t blockSz = taosArrayGetSize(pBlocks); + + tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz); + + void* pBuf = NULL; + SArray* tagArray = NULL; + SArray* pVals = NULL; + + if (!(tagArray = taosArrayInit(1, sizeof(STagVal)))) { + goto _end; + } + + for (int32_t i = 0; i < blockSz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + SBatchDeleteReq deleteReq = {0}; + deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); + deleteReq.suid = suid; + tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, &deleteReq); + if (taosArrayGetSize(deleteReq.deleteReqs) == 0) { + taosArrayDestroy(deleteReq.deleteReqs); + continue; + } + + int32_t len; + int32_t code; + tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); + if (code < 0) { + // + ASSERT(0); + } + SEncoder encoder; + void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); + void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); + tEncoderInit(&encoder, abuf, len); + tEncodeSBatchDeleteReq(&encoder, &deleteReq); + tEncoderClear(&encoder); + taosArrayDestroy(deleteReq.deleteReqs); + + ((SMsgHead*)serializedDeleteReq)->vgId = pVnode->config.vgId; + + SRpcMsg msg = { + .msgType = TDMT_VND_BATCH_DEL, + .pCont = serializedDeleteReq, + .contLen = len + sizeof(SMsgHead), + }; + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + tqDebug("failed to put delete req into write-queue since %s", terrstr()); + } + } else { + SSubmitTbData tbData = {0}; + int32_t rows = pDataBlock->info.rows; + tqDebug("tq sink pipe2, convert block1 %d, rows: %d", i, rows); + + if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow*)))) { + goto _end; + } + + tbData.suid = suid; + tbData.uid = 0; // uid is assigned by vnode + tbData.sver = pTSchema->version; + + char* ctbName = NULL; + if (pDataBlock->info.parTbName[0]) { + ctbName = strdup(pDataBlock->info.parTbName); + } else { + ctbName = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId); + } + + SMetaReader mr = {0}; + metaReaderInit(&mr, pVnode->pMeta, 0); + if (metaGetTableEntryByName(&mr, ctbName) < 0) { + metaReaderClear(&mr); + tqDebug("vgId:%d, stream write into %s, table auto created", TD_VID(pVnode), ctbName); + + SVCreateTbReq* pCreateTbReq = NULL; + + if (!(pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq)))) { + goto _end; + }; + + // set const + pCreateTbReq->flags = 0; + pCreateTbReq->type = TSDB_CHILD_TABLE; + pCreateTbReq->ctb.suid = suid; + + // set super table name + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + pCreateTbReq->ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName); + + // set tag content + taosArrayClear(tagArray); + STagVal tagVal = { + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.id.groupId, + }; + taosArrayPush(tagArray, &tagVal); + pCreateTbReq->ctb.tagNum = taosArrayGetSize(tagArray); + + STag* pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + if (pTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + pCreateTbReq->ctb.pTag = (uint8_t*)pTag; + + // set tag name + SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = {0}; + strcpy(tagNameStr, "group_id"); + taosArrayPush(tagName, tagNameStr); + pCreateTbReq->ctb.tagName = tagName; + + // set table name + pCreateTbReq->name = ctbName; + ctbName = NULL; + + tbData.pCreateTbReq = pCreateTbReq; + tbData.flags = SUBMIT_REQ_AUTO_CREATE_TABLE; + } else { + if (mr.me.type != TSDB_CHILD_TABLE) { + tqError("vgId:%d, failed to write into %s, since table type incorrect, type %d", TD_VID(pVnode), ctbName, + mr.me.type); + metaReaderClear(&mr); + taosMemoryFree(ctbName); + continue; + } + + if (mr.me.ctbEntry.suid != suid) { + tqError("vgId:%d, failed to write into %s, since suid mismatch, expect suid: %" PRId64 + ", actual suid %" PRId64 "", + TD_VID(pVnode), ctbName, suid, mr.me.ctbEntry.suid); + metaReaderClear(&mr); + taosMemoryFree(ctbName); + } + + tbData.uid = mr.me.uid; + metaReaderClear(&mr); + taosMemoryFreeClear(ctbName); + } + + // rows + if (!pVals && !(pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)))) { + taosArrayDestroy(tbData.aRowP); + goto _end; + } + + for (int32_t j = 0; j < rows; j++) { + taosArrayClear(pVals); + for (int32_t k = 0; k < pTSchema->numOfCols; k++) { + const STColumn* pCol = &pTSchema->columns[k]; + SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, k); + if (k == 0) { + void* colData = colDataGetData(pColData, j); + tqDebug("tq sink pipe2, row %d, col %d ts %" PRId64, j, k, *(int64_t*)colData); + } + if (colDataIsNull_s(pColData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + void* colData = colDataGetData(pColData, j); + if (IS_STR_DATA_TYPE(pCol->type)) { + SValue sv = + (SValue){.nData = varDataLen(colData), .pData = varDataVal(colData)}; // address copy, no value + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } else { + SValue sv; + memcpy(&sv.val, colData, tDataTypes[pCol->type].bytes); + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } + } + } + SRow* pRow = NULL; + if ((terrno = tRowBuild(pVals, (STSchema*)pTSchema, &pRow)) < 0) { + tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + goto _end; + } + ASSERT(pRow); + taosArrayPush(tbData.aRowP, &pRow); + } + + SSubmitReq2 submitReq = {0}; + if (!(submitReq.aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { + goto _end; + } + + taosArrayPush(submitReq.aSubmitTbData, &tbData); + + // encode + int32_t len; + int32_t code; + tEncodeSize(tEncodeSSubmitReq2, &submitReq, len, code); + SEncoder encoder; + len += sizeof(SMsgHead); + pBuf = rpcMallocCont(len); + if (NULL == pBuf) { + goto _end; + } + ((SMsgHead*)pBuf)->vgId = TD_VID(pVnode); + ((SMsgHead*)pBuf)->contLen = htonl(len); + tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SMsgHead)), len - sizeof(SMsgHead)); + if (tEncodeSSubmitReq2(&encoder, &submitReq) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("failed to encode submit req since %s", terrstr()); + tEncoderClear(&encoder); + rpcFreeCont(pBuf); + continue; + } + tEncoderClear(&encoder); + tDestroySSubmitReq2(&submitReq, TSDB_MSG_FLG_ENCODE); + + SRpcMsg msg = { + .msgType = TDMT_VND_SUBMIT, + .pCont = pBuf, + .contLen = len, + }; + + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + tqDebug("failed to put into write-queue since %s", terrstr()); + } + } + } +_end: + taosArrayDestroy(tagArray); + taosArrayDestroy(pVals); + // TODO: change +} + #if 0 void tqSinkToTableMerge(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { const SArray* pRes = (const SArray*)data; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 6a8251706721cc4ac5dd178d258728c4d9f0b2d2..c7b36ebb226014115d4fc08f63a92747c389f3f9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -91,12 +91,10 @@ int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { } } + taosLRUCacheRelease(pCache, h, invalidate); if (invalidate) { - taosLRUCacheRelease(pCache, h, true); - } else { - taosLRUCacheRelease(pCache, h, false); + taosLRUCacheErase(pCache, key, keyLen); } - // void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen); } return code; @@ -124,12 +122,10 @@ int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { } } + taosLRUCacheRelease(pCache, h, invalidate); if (invalidate) { - taosLRUCacheRelease(pCache, h, true); - } else { - taosLRUCacheRelease(pCache, h, false); + taosLRUCacheErase(pCache, key, keyLen); } - // void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen); } return code; @@ -190,7 +186,7 @@ int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { return code; } -int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, STSRow *row, bool dup) { +int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, TSDBROW *row, bool dup) { int32_t code = 0; STSRow *cacheRow = NULL; char key[32] = {0}; @@ -201,13 +197,51 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); if (h) { STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); - TSKEY keyTs = row->ts; + TSKEY keyTs = TSDBROW_TS(row); bool invalidate = false; SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h); int16_t nCol = taosArrayGetSize(pLast); int16_t iCol = 0; + if (nCol <= 0) { + nCol = pTSchema->numOfCols; + + STColumn *pTColumn = &pTSchema->columns[0]; + SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = keyTs}); + if (taosArrayPush(pLast, &(SLastCol){.ts = keyTs, .colVal = tColVal}) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + + for (iCol = 1; iCol < nCol; ++iCol) { + SColVal colVal = {0}; + tsdbRowGetColVal(row, pTSchema, iCol, &colVal); + + SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; + if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { + SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); + taosMemoryFree(pLastCol->colVal.value.pData); + + lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); + if (lastCol.colVal.value.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); + } + + if (taosArrayPush(pLast, &lastCol) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + } + + goto _invalidate; + } + SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); if (keyTs > tTsVal->ts) { STColumn *pTColumn = &pTSchema->columns[0]; @@ -222,7 +256,7 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST SColVal *tColVal = &tTsVal1->colVal; SColVal colVal = {0}; - tTSRowGetVal(row, pTSchema, iCol, &colVal); + tsdbRowGetColVal(row, pTSchema, iCol, &colVal); if (!COL_VAL_IS_NONE(&colVal)) { if (keyTs == tTsVal1->ts && !COL_VAL_IS_NONE(tColVal)) { invalidate = true; @@ -261,7 +295,7 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST return code; } -int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb) { +int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, TSDBROW *row, STsdb *pTsdb) { int32_t code = 0; STSRow *cacheRow = NULL; char key[32] = {0}; @@ -272,13 +306,51 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); if (h) { STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); - TSKEY keyTs = row->ts; + TSKEY keyTs = TSDBROW_TS(row); bool invalidate = false; SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h); int16_t nCol = taosArrayGetSize(pLast); int16_t iCol = 0; + if (nCol <= 0) { + nCol = pTSchema->numOfCols; + + STColumn *pTColumn = &pTSchema->columns[0]; + SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = keyTs}); + if (taosArrayPush(pLast, &(SLastCol){.ts = keyTs, .colVal = tColVal}) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + + for (iCol = 1; iCol < nCol; ++iCol) { + SColVal colVal = {0}; + tsdbRowGetColVal(row, pTSchema, iCol, &colVal); + + SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; + if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { + SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); + taosMemoryFree(pLastCol->colVal.value.pData); + + lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); + if (lastCol.colVal.value.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); + } + + if (taosArrayPush(pLast, &lastCol) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + } + + goto _invalidate; + } + SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); if (keyTs > tTsVal->ts) { STColumn *pTColumn = &pTSchema->columns[0]; @@ -293,7 +365,7 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb SColVal *tColVal = &tTsVal1->colVal; SColVal colVal = {0}; - tTSRowGetVal(row, pTSchema, iCol, &colVal); + tsdbRowGetColVal(row, pTSchema, iCol, &colVal); if (COL_VAL_IS_VALUE(&colVal)) { if (keyTs == tTsVal1->ts && COL_VAL_IS_VALUE(tColVal)) { invalidate = true; @@ -617,7 +689,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { } else { // tBlockDataDestroy(&state->blockData, 1); if (state->pBlockData) { - tBlockDataDestroy(state->pBlockData, 1); + tBlockDataDestroy(state->pBlockData); state->pBlockData = NULL; } @@ -685,7 +757,8 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { tBlockDataReset(state->pBlockData); tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetDataBlk); - /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); */ + /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); + */ tBlockDataReset(state->pBlockData); TABLEID tid = {.suid = state->suid, .uid = state->uid}; code = tBlockDataInit(state->pBlockData, &tid, state->pTSchema, NULL, 0); @@ -739,7 +812,7 @@ _err: state->aBlockIdx = NULL; } if (state->pBlockData) { - tBlockDataDestroy(state->pBlockData, 1); + tBlockDataDestroy(state->pBlockData); state->pBlockData = NULL; } @@ -766,7 +839,7 @@ int32_t clearNextRowFromFS(void *iter) { } if (state->pBlockData) { // tBlockDataDestroy(&state->blockData, 1); - tBlockDataDestroy(state->pBlockData, 1); + tBlockDataDestroy(state->pBlockData); state->pBlockData = NULL; } @@ -1219,12 +1292,12 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo // build the result ts row here *dup = false; - if (taosArrayGetSize(pColArray) != nCol) { - *ppColArray = NULL; - taosArrayDestroy(pColArray); - } else { - *ppColArray = pColArray; - } + // if (taosArrayGetSize(pColArray) != nCol) { + //*ppColArray = NULL; + // taosArrayDestroy(pColArray); + //} else { + *ppColArray = pColArray; + //} nextRowIterClose(&iter); // taosMemoryFreeClear(pTSchema); @@ -1337,12 +1410,12 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach } } while (setNoneCol); - if (taosArrayGetSize(pColArray) <= 0) { - *ppLastArray = NULL; - taosArrayDestroy(pColArray); - } else { - *ppLastArray = pColArray; - } + // if (taosArrayGetSize(pColArray) <= 0) { + //*ppLastArray = NULL; + // taosArrayDestroy(pColArray); + //} else { + *ppLastArray = pColArray; + //} nextRowIterClose(&iter); // taosMemoryFreeClear(pTSchema); @@ -1373,8 +1446,8 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader * SArray *pArray = NULL; bool dup = false; // which is always false for now code = mergeLastRow(uid, pTsdb, &dup, &pArray, pr); - // if table's empty or error, return code of -1 - if (code < 0 || pArray == NULL) { + // if table's empty or error, set handle NULL and return + if (code < 0 /* || pArray == NULL*/) { if (!dup && pArray) { taosArrayDestroy(pArray); } @@ -1392,13 +1465,8 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader * if (status != TAOS_LRU_STATUS_OK) { code = -1; } - - // taosThreadMutexUnlock(&pTsdb->lruMutex); - - // h = taosLRUCacheLookup(pCache, key, keyLen); - } // else { + } taosThreadMutexUnlock(&pTsdb->lruMutex); - //} } *handle = h; @@ -1422,8 +1490,8 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, if (!h) { SArray *pLastArray = NULL; code = mergeLast(uid, pTsdb, &pLastArray, pr); - // if table's empty or error, return code of -1 - if (code < 0 || pLastArray == NULL) { + // if table's empty or error, set handle NULL and return + if (code < 0 /* || pLastArray == NULL*/) { taosThreadMutexUnlock(&pTsdb->lruMutex); *handle = NULL; @@ -1437,13 +1505,8 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, if (status != TAOS_LRU_STATUS_OK) { code = -1; } - - // taosThreadMutexUnlock(&pTsdb->lruMutex); - - // h = taosLRUCacheLookup(pCache, key, keyLen); - } // else { + } taosThreadMutexUnlock(&pTsdb->lruMutex); - //} } *handle = h; diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index f05f5d5c88f7ef19c8598410019326edc9b1c5c6..9f48125638befad30a8277d9aa637b9074ea95f2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -258,6 +258,10 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 if (h == NULL) { continue; } + if (taosArrayGetSize(pRow) <= 0) { + tsdbCacheRelease(lruCache, h); + continue; + } { for (int32_t k = 0; k < pr->numOfCols; ++k) { @@ -327,6 +331,10 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 if (h == NULL) { continue; } + if (taosArrayGetSize(pRow) <= 0) { + tsdbCacheRelease(lruCache, h); + continue; + } saveOneRow(pRow, pResBlock, pr, slotIds, pRes, pr->idstr); // TODO reset the pRes diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 8ec59ea95949a4bccda38c7cc2fda25e2f769fb4..92451dcf5805d67fe053513f958470a5b118b4a2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -913,24 +913,24 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { // reader taosArrayDestroy(pCommitter->dReader.aBlockIdx); tMapDataClear(&pCommitter->dReader.mBlock); - tBlockDataDestroy(&pCommitter->dReader.bData, 1); + tBlockDataDestroy(&pCommitter->dReader.bData); // merger for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) { SDataIter *pIter = &pCommitter->aDataIter[iStt]; taosArrayDestroy(pIter->aSttBlk); - tBlockDataDestroy(&pIter->bData, 1); + tBlockDataDestroy(&pIter->bData); } // writer taosArrayDestroy(pCommitter->dWriter.aBlockIdx); taosArrayDestroy(pCommitter->dWriter.aSttBlk); tMapDataClear(&pCommitter->dWriter.mBlock); - tBlockDataDestroy(&pCommitter->dWriter.bData, 1); + tBlockDataDestroy(&pCommitter->dWriter.bData); #if USE_STREAM_COMPRESSION tDiskDataBuilderDestroy(pCommitter->dWriter.pBuilder); #else - tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1); + tBlockDataDestroy(&pCommitter->dWriter.bDatal); #endif tDestroyTSchema(pCommitter->skmTable.pTSchema); tDestroyTSchema(pCommitter->skmRow.pTSchema); @@ -1183,7 +1183,6 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) tBlockDataClear(pBlockData); while (pRowInfo) { - ASSERT(pRowInfo->row.type == 0); code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); TSDB_CHECK_CODE(code, lino, _exit); @@ -1251,7 +1250,6 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) pRow = NULL; } } else if (c > 0) { - ASSERT(pRowInfo->row.type == 0); code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); TSDB_CHECK_CODE(code, lino, _exit); @@ -1493,7 +1491,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { while (pRowInfo) { STSchema *pTSchema = NULL; - if (pRowInfo->row.type == 0) { + if (pRowInfo->row.type == TSDBROW_ROW_FMT) { code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); TSDB_CHECK_CODE(code, lino, _exit); pTSchema = pCommitter->skmRow.pTSchema; @@ -1536,7 +1534,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { while (pRowInfo) { STSchema *pTSchema = NULL; - if (pRowInfo->row.type == 0) { + if (pRowInfo->row.type == TSDBROW_ROW_FMT) { code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); TSDB_CHECK_CODE(code, lino, _exit); pTSchema = pCommitter->skmRow.pTSchema; diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index b46a00363817666cdff7bc7756ff204483787c36..ae9af11f5ae8226c1fceefb57b4d83167800772a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -596,7 +596,7 @@ int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTS if (pBuilder->bi.maxKey < kRow.ts) pBuilder->bi.maxKey = kRow.ts; STSDBRowIter iter = {0}; - tsdbRowIterInit(&iter, pRow, pTSchema); + tsdbRowIterOpen(&iter, pRow, pTSchema); SColVal *pColVal = tsdbRowIterNext(&iter); for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) { diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 0a7f59e429142a8a8cdd52627a36ff6c8856acaf..007ae871a0e20d84d729f2d8123c0e7500591423 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -28,8 +28,10 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags); static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData); -static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, int64_t version, - SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubmitBlkRsp *pRsp); +static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version, + SSubmitTbData *pSubmitTbData, int32_t *affectedRows); +static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version, + SSubmitTbData *pSubmitTbData, int32_t *affectedRows); int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { int32_t code = 0; @@ -95,14 +97,14 @@ STbData *tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t return pTbData; } -int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, - SSubmitBlkRsp *pRsp) { +int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitTbData *pSubmitTbData, int32_t *affectedRows) { int32_t code = 0; SMemTable *pMemTable = pTsdb->mem; STbData *pTbData = NULL; - tb_uid_t suid = pMsgIter->suid; - tb_uid_t uid = pMsgIter->uid; + tb_uid_t suid = pSubmitTbData->suid; + tb_uid_t uid = pSubmitTbData->uid; +#if 0 SMetaInfo info; code = metaGetInfo(pTsdb->pVnode->pMeta, uid, &info, NULL); if (code) { @@ -116,14 +118,15 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI if (info.suid) { metaGetInfo(pTsdb->pVnode->pMeta, info.suid, &info, NULL); } - if (pMsgIter->sversion != info.skmVer) { + if (pSubmitTbData->sver != info.skmVer) { tsdbError("vgId:%d, req sver:%d, skmVer:%d suid:%" PRId64 " uid:%" PRId64, TD_VID(pTsdb->pVnode), - pMsgIter->sversion, info.skmVer, suid, uid); + pSubmitTbData->sver, info.skmVer, suid, uid); code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; goto _err; } - pRsp->sver = info.skmVer; + if (pRsp) pRsp->sver = info.skmVer; +#endif // create/get STbData to op code = tsdbGetOrCreateTbData(pMemTable, suid, uid, &pTbData); @@ -132,10 +135,12 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI } // do insert impl - code = tsdbInsertTableDataImpl(pMemTable, pTbData, version, pMsgIter, pBlock, pRsp); - if (code) { - goto _err; + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + code = tsdbInsertColDataToTable(pMemTable, pTbData, version, pSubmitTbData, affectedRows); + } else { + code = tsdbInsertRowDataToTable(pMemTable, pTbData, version, pSubmitTbData, affectedRows); } + if (code) goto _err; return code; @@ -242,7 +247,6 @@ void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDa pIter->pTbData = pTbData; pIter->backward = backward; pIter->pRow = NULL; - pIter->row.type = 0; if (pFrom == NULL) { // create from head or tail if (backward) { @@ -410,8 +414,13 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p for (int8_t iLevel = pTbData->sl.level - 1; iLevel >= 0; iLevel--) { pn = SL_NODE_BACKWARD(px, iLevel); while (pn != pTbData->sl.pHead) { - tKey.version = pn->version; - tKey.ts = pn->pTSRow->ts; + if (pn->flag == TSDBROW_ROW_FMT) { + tKey.version = pn->version; + tKey.ts = ((SRow *)pn->pData)->ts; + } else if (pn->flag == TSDBROW_COL_FMT) { + tKey.version = ((SBlockData *)pn->pData)->aVersion[pn->iRow]; + tKey.ts = ((SBlockData *)pn->pData)->aTSKEY[pn->iRow]; + } int32_t c = tsdbKeyCmprFn(&tKey, pKey); if (c <= 0) { @@ -440,8 +449,13 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p for (int8_t iLevel = pTbData->sl.level - 1; iLevel >= 0; iLevel--) { pn = SL_NODE_FORWARD(px, iLevel); while (pn != pTbData->sl.pTail) { - tKey.version = pn->version; - tKey.ts = pn->pTSRow->ts; + if (pn->flag == TSDBROW_ROW_FMT) { + tKey.version = pn->version; + tKey.ts = ((SRow *)pn->pData)->ts; + } else if (pn->flag == TSDBROW_COL_FMT) { + tKey.version = ((SBlockData *)pn->pData)->aVersion[pn->iRow]; + tKey.ts = ((SBlockData *)pn->pData)->aTSKEY[pn->iRow]; + } int32_t c = tsdbKeyCmprFn(&tKey, pKey); if (c >= 0) { @@ -468,29 +482,39 @@ static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) { return level; } -static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListNode **pos, int64_t version, - STSRow *pRow, int8_t forward) { +static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListNode **pos, TSDBROW *pRow, + int8_t forward) { int32_t code = 0; int8_t level; SMemSkipListNode *pNode; SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; + ASSERT(pPool != NULL); + // node level = tsdbMemSkipListRandLevel(&pTbData->sl); - ASSERT(pPool != NULL); pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level)); if (pNode == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } pNode->level = level; - pNode->version = version; - pNode->pTSRow = vnodeBufPoolMalloc(pPool, pRow->len); - if (NULL == pNode->pTSRow) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + pNode->flag = pRow->type; + + if (pRow->type == TSDBROW_ROW_FMT) { + pNode->version = pRow->version; + pNode->pData = vnodeBufPoolMalloc(pPool, pRow->pTSRow->len); + if (NULL == pNode->pData) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(pNode->pData, pRow->pTSRow, pRow->pTSRow->len); + } else if (pRow->type == TSDBROW_COL_FMT) { + pNode->iRow = pRow->iRow; + pNode->pData = pRow->pBlockData; + } else { + ASSERT(0); } - memcpy(pNode->pTSRow, pRow, pRow->len); for (int8_t iLevel = level - 1; iLevel >= 0; iLevel--) { SMemSkipListNode *pn = pos[iLevel]; @@ -537,69 +561,166 @@ _exit: return code; } -static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, int64_t version, - SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubmitBlkRsp *pRsp) { - int32_t code = 0; - SSubmitBlkIter blkIter = {0}; - TSDBKEY key = {.version = version}; - SMemSkipListNode *pos[SL_MAX_LEVEL]; - TSDBROW row = tsdbRowFromTSRow(version, NULL); - int32_t nRow = 0; - STSRow *pLastRow = NULL; +static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version, + SSubmitTbData *pSubmitTbData, int32_t *affectedRows) { + int32_t code = 0; - tInitSubmitBlkIter(pMsgIter, pBlock, &blkIter); + SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; + int32_t nColData = TARRAY_SIZE(pSubmitTbData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pSubmitTbData->aCol); - // backward put first data - row.pTSRow = tGetSubmitBlkNext(&blkIter); - if (row.pTSRow == NULL) return code; + ASSERT(aColData[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); + ASSERT(aColData[0].type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(aColData[0].flag == HAS_VALUE); - key.ts = row.pTSRow->ts; - nRow++; - tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD); - code = tbDataDoPut(pMemTable, pTbData, pos, version, row.pTSRow, 0); - if (code) { - goto _err; + // copy and construct block data + SBlockData *pBlockData = vnodeBufPoolMalloc(pPool, sizeof(*pBlockData)); + if (pBlockData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + pBlockData->suid = pTbData->suid; + pBlockData->uid = pTbData->uid; + pBlockData->nRow = aColData[0].nVal; + pBlockData->aUid = NULL; + pBlockData->aVersion = vnodeBufPoolMalloc(pPool, aColData[0].nData); + if (pBlockData->aVersion == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + for (int32_t i = 0; i < pBlockData->nRow; i++) { // todo: here can be optimized + pBlockData->aVersion[i] = version; + } + + pBlockData->aTSKEY = vnodeBufPoolMalloc(pPool, aColData[0].nData); + if (pBlockData->aTSKEY == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(pBlockData->aTSKEY, aColData[0].pData, aColData[0].nData); + + pBlockData->nColData = nColData - 1; + pBlockData->aColData = vnodeBufPoolMalloc(pPool, sizeof(SColData) * pBlockData->nColData); + if (pBlockData->aColData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + for (int32_t iColData = 0; iColData < pBlockData->nColData; ++iColData) { + code = tColDataCopy(&aColData[iColData + 1], &pBlockData->aColData[iColData], (xMallocFn)vnodeBufPoolMalloc, pPool); + if (code) goto _exit; } + // loop to add each row to the skiplist + SMemSkipListNode *pos[SL_MAX_LEVEL]; + TSDBROW tRow = tsdbRowFromBlockData(pBlockData, 0); + TSDBKEY key = {.version = version, .ts = pBlockData->aTSKEY[0]}; + TSDBROW lRow; // last row + + // first row + tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD); + if ((code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 0))) goto _exit; pTbData->minKey = TMIN(pTbData->minKey, key.ts); + lRow = tRow; + + // remain row + ++tRow.iRow; + if (tRow.iRow < pBlockData->nRow) { + for (int8_t iLevel = pos[0]->level; iLevel < pTbData->sl.maxLevel; iLevel++) { + pos[iLevel] = SL_NODE_BACKWARD(pos[iLevel], iLevel); + } + + while (tRow.iRow < pBlockData->nRow) { + key.ts = pBlockData->aTSKEY[tRow.iRow]; - pLastRow = row.pTSRow; + if (SL_NODE_FORWARD(pos[0], 0) != pTbData->sl.pTail) { + tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_FROM_POS); + } + + if ((code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 1))) goto _exit; + lRow = tRow; + + ++tRow.iRow; + } + } + + if (key.ts >= pTbData->maxKey) { + pTbData->maxKey = key.ts; + + if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config)) { + tsdbCacheInsertLastrow(pMemTable->pTsdb->lruCache, pMemTable->pTsdb, pTbData->uid, &lRow, true); + } + } + + if (TSDB_CACHE_LAST(pMemTable->pTsdb->pVnode->config)) { + tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, &lRow, pMemTable->pTsdb); + } + + // SMemTable + pMemTable->minKey = TMIN(pMemTable->minKey, pTbData->minKey); + pMemTable->maxKey = TMAX(pMemTable->maxKey, pTbData->maxKey); + pMemTable->nRow += pBlockData->nRow; + + if (affectedRows) *affectedRows = pBlockData->nRow; + +_exit: + return code; +} + +static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version, + SSubmitTbData *pSubmitTbData, int32_t *affectedRows) { + int32_t code = 0; + + int32_t nRow = TARRAY_SIZE(pSubmitTbData->aRowP); + SRow **aRow = (SRow **)TARRAY_DATA(pSubmitTbData->aRowP); + TSDBKEY key = {.version = version}; + SMemSkipListNode *pos[SL_MAX_LEVEL]; + TSDBROW tRow = {.type = TSDBROW_ROW_FMT, .version = version}; + int32_t iRow = 0; + TSDBROW lRow; + + // backward put first data + tRow.pTSRow = aRow[iRow++]; + key.ts = tRow.pTSRow->ts; + tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD); + code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 0); + if (code) goto _exit; + lRow = tRow; + + pTbData->minKey = TMIN(pTbData->minKey, key.ts); // forward put rest data - row.pTSRow = tGetSubmitBlkNext(&blkIter); - if (row.pTSRow) { + if (iRow < nRow) { for (int8_t iLevel = pos[0]->level; iLevel < pTbData->sl.maxLevel; iLevel++) { pos[iLevel] = SL_NODE_BACKWARD(pos[iLevel], iLevel); } - do { - key.ts = row.pTSRow->ts; - nRow++; + + while (iRow < nRow) { + tRow.pTSRow = aRow[iRow]; + key.ts = tRow.pTSRow->ts; + if (SL_NODE_FORWARD(pos[0], 0) != pTbData->sl.pTail) { tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_FROM_POS); } - code = tbDataDoPut(pMemTable, pTbData, pos, version, row.pTSRow, 1); - if (code) { - goto _err; - } - pLastRow = row.pTSRow; + code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 1); + if (code) goto _exit; + + lRow = tRow; - row.pTSRow = tGetSubmitBlkNext(&blkIter); - } while (row.pTSRow); + iRow++; + } } if (key.ts >= pTbData->maxKey) { - if (key.ts > pTbData->maxKey) { - pTbData->maxKey = key.ts; - } + pTbData->maxKey = key.ts; - if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config) && pLastRow != NULL) { - tsdbCacheInsertLastrow(pMemTable->pTsdb->lruCache, pMemTable->pTsdb, pTbData->uid, pLastRow, true); + if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config)) { + tsdbCacheInsertLastrow(pMemTable->pTsdb->lruCache, pMemTable->pTsdb, pTbData->uid, &lRow, true); } } if (TSDB_CACHE_LAST(pMemTable->pTsdb->pVnode->config)) { - tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, pLastRow, pMemTable->pTsdb); + tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, &lRow, pMemTable->pTsdb); } // SMemTable @@ -607,12 +728,9 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i pMemTable->maxKey = TMAX(pMemTable->maxKey, pTbData->maxKey); pMemTable->nRow += nRow; - pRsp->numOfRows = nRow; - pRsp->affectedRows = nRow; - - return code; + if (affectedRows) *affectedRows = nRow; -_err: +_exit: return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index a5ad18a22d3467a9232b35bb99cb76c10269117a..1f99444cf3cc655be656ce86a70ea22920c26fe8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -89,8 +89,8 @@ void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { pLoadInfo[i].blockIndex[0] = -1; pLoadInfo[i].blockIndex[1] = -1; - tBlockDataDestroy(&pLoadInfo[i].blockData[0], true); - tBlockDataDestroy(&pLoadInfo[i].blockData[1], true); + tBlockDataDestroy(&pLoadInfo[i].blockData[0]); + tBlockDataDestroy(&pLoadInfo[i].blockData[1]); taosArrayDestroy(pLoadInfo[i].aSttBlk); } @@ -567,7 +567,6 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead pMTree->pLoadInfo = pBlockLoadInfo; pMTree->destroyLoadInfo = destroyLoadInfo; - ASSERT(pMTree->pLoadInfo != NULL); for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file struct SLDataIter *pIter = NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 6634340d75a0061be6908b8927dee508efef44e8..d629be2fef308fd2cbbb1fe9cf74c9df834666b9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -82,13 +82,13 @@ typedef struct SIOCostSummary { } SIOCostSummary; typedef struct SBlockLoadSuppInfo { - SArray* pColAgg; - SColumnDataAgg tsColAgg; - int16_t* colId; - int16_t* slotId; - int32_t numOfCols; - char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. - bool smaValid; // the sma on all queried columns are activated + SArray* pColAgg; + SColumnDataAgg tsColAgg; + int16_t* colId; + int16_t* slotId; + int32_t numOfCols; + char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. + bool smaValid; // the sma on all queried columns are activated } SBlockLoadSuppInfo; typedef struct SLastBlockReader { @@ -168,11 +168,11 @@ struct STsdbReader { SBlockLoadSuppInfo suppInfo; STsdbReadSnap* pReadSnap; SIOCostSummary cost; - STSchema* pSchema; // the newest version schema - STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times - SDataFReader* pFileReader; // the file reader - SDelFReader* pDelFReader; // the del file reader - SArray* pDelIdx; // del file block index; + STSchema* pSchema; // the newest version schema + STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times + SDataFReader* pFileReader; // the file reader + SDelFReader* pDelFReader; // the del file reader + SArray* pDelIdx; // del file block index; SVersionRange verRange; SBlockInfoBuf blockInfoBuf; int32_t step; @@ -189,8 +189,8 @@ static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STabl SRowMerger* pMerger, SVersionRange* pVerRange); static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, STsdbReader* pReader); -static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, - STableBlockScanInfo* pScanInfo); +static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, + STableBlockScanInfo* pInfo); static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex); static void setComposedBlockFlag(STsdbReader* pReader, bool composed); @@ -198,9 +198,9 @@ static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* SVersionRange* pVerRange); static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, - STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow); + TSDBROW* pTSRow, STsdbReader* pReader, bool* freeTSRow); static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, - STsdbReader* pReader, STSRow** pTSRow); + STsdbReader* pReader, SRow** pTSRow); static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader); @@ -218,17 +218,18 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBl static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } -static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, int32_t numOfCols) { +static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, + int32_t numOfCols) { pSupInfo->smaValid = true; pSupInfo->numOfCols = numOfCols; - pSupInfo->colId = taosMemoryMalloc(numOfCols * (sizeof(int16_t)*2 + POINTER_BYTES)); + pSupInfo->colId = taosMemoryMalloc(numOfCols * (sizeof(int16_t) * 2 + POINTER_BYTES)); if (pSupInfo->colId == NULL) { taosMemoryFree(pSupInfo->colId); return TSDB_CODE_OUT_OF_MEMORY; } pSupInfo->slotId = (int16_t*)((char*)pSupInfo->colId + (sizeof(int16_t) * numOfCols)); - pSupInfo->buildBuf = (char**) ((char*)pSupInfo->slotId + (sizeof(int16_t) * numOfCols)); + pSupInfo->buildBuf = (char**)((char*)pSupInfo->slotId + (sizeof(int16_t) * numOfCols)); for (int32_t i = 0; i < numOfCols; ++i) { pSupInfo->colId[i] = pCols[i].colId; pSupInfo->slotId[i] = pSlotIdList[i]; @@ -243,15 +244,15 @@ static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pC return TSDB_CODE_SUCCESS; } -static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) { +static int32_t updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) { int32_t i = 0, j = 0; - while(i < pSchema->numOfCols && j < pSupInfo->numOfCols) { + while (i < pSchema->numOfCols && j < pSupInfo->numOfCols) { STColumn* pTCol = &pSchema->columns[i]; if (pTCol->colId == pSupInfo->colId[j]) { if (!IS_BSMA_ON(pTCol)) { pSupInfo->smaValid = false; - return; + return TSDB_CODE_SUCCESS; } i += 1; @@ -260,9 +261,11 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) // do nothing i += 1; } else { - ASSERT(0); + return TSDB_CODE_INVALID_PARA; } } + + return TSDB_CODE_SUCCESS; } static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) { @@ -309,7 +312,8 @@ static void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index) { } // NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model -static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, int32_t numOfTables) { +static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, + int32_t numOfTables) { // allocate buffer in order to load data blocks from file // todo use simple hash instead, optimize the memory consumption SHashObj* pTableMap = @@ -579,7 +583,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd } if (VND_IS_TSMA(pVnode)) { - tsdbDebug("vgId:%d, tsma is selected to query", TD_VID(pVnode)); + tsdbDebug("vgId:%d, tsma is selected to query, %s", TD_VID(pVnode), idstr); } initReaderStatus(&pReader->status); @@ -594,7 +598,6 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->type = pCond->type; pReader->window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); pReader->blockInfoBuf.numPerBucket = 1000; // 1000 tables per bucket - ASSERT(pCond->numOfCols > 0); if (pReader->pResBlock == NULL) { pReader->freeBlock = true; @@ -605,6 +608,12 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd } } + if (pCond->numOfCols <= 0) { + tsdbError("vgId:%d, invalid column number %d in query cond, %s", TD_VID(pVnode), pCond->numOfCols, idstr); + code = TSDB_CODE_INVALID_PARA; + goto _end; + } + // todo refactor. limitOutputBufferSize(pCond, &pReader->capacity); @@ -761,7 +770,6 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastFiles, sizeInDisk / 1000.0, el, pReader->idStr); - pReader->cost.numOfBlocks += total; pReader->cost.headFileLoadTime += el; @@ -794,8 +802,9 @@ static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_ } static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { - if (taosArrayGetSize(pBlockIter->blockList) == 0) { - ASSERT(pBlockIter->numOfBlocks == taosArrayGetSize(pBlockIter->blockList)); + size_t num = taosArrayGetSize(pBlockIter->blockList); + if (num == 0) { + ASSERT(pBlockIter->numOfBlocks == num); return NULL; } @@ -805,73 +814,6 @@ static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { static SDataBlk* getCurrentBlock(SDataBlockIter* pBlockIter) { return &pBlockIter->block; } -int32_t binarySearchForTs(char* pValue, int num, TSKEY key, int order) { - int32_t midPos = -1; - int32_t numOfRows; - - ASSERT(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - - TSKEY* keyList = (TSKEY*)pValue; - int32_t firstPos = 0; - int32_t lastPos = num - 1; - - if (order == TSDB_ORDER_DESC) { - // find the first position which is smaller than the key - while (1) { - if (key >= keyList[firstPos]) return firstPos; - if (key == keyList[lastPos]) return lastPos; - - if (key < keyList[lastPos]) { - lastPos += 1; - if (lastPos >= num) { - return -1; - } else { - return lastPos; - } - } - - numOfRows = lastPos - firstPos + 1; - midPos = (numOfRows >> 1) + firstPos; - - if (key < keyList[midPos]) { - firstPos = midPos + 1; - } else if (key > keyList[midPos]) { - lastPos = midPos - 1; - } else { - break; - } - } - - } else { - // find the first position which is bigger than the key - while (1) { - if (key <= keyList[firstPos]) return firstPos; - if (key == keyList[lastPos]) return lastPos; - - if (key > keyList[lastPos]) { - lastPos = lastPos + 1; - if (lastPos >= num) - return -1; - else - return lastPos; - } - - numOfRows = lastPos - firstPos + 1; - midPos = (numOfRows >> 1u) + firstPos; - - if (key < keyList[midPos]) { - lastPos = midPos - 1; - } else if (key > keyList[midPos]) { - firstPos = midPos + 1; - } else { - break; - } - } - } - - return midPos; -} - static int doBinarySearchKey(TSKEY* keyList, int num, int pos, TSKEY key, int order) { // start end position int s, e; @@ -961,7 +903,7 @@ static void copyPrimaryTsCol(const SBlockData* pBlockData, SFileBlockDumpInfo* p // a faster version of copy procedure. static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData, - int32_t dumpedRows, bool asc) { + int32_t dumpedRows, bool asc) { uint8_t* p = NULL; if (asc) { p = pData->pData + tDataTypes[pData->type].bytes * pDumpInfo->rowIndex; @@ -970,22 +912,21 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo p = pData->pData + tDataTypes[pData->type].bytes * startIndex; } - int32_t step = asc? 1:-1; + int32_t step = asc ? 1 : -1; - // make sure it is aligned to 8bit - ASSERT((((uint64_t)pColData->pData) & (0x8 - 1)) == 0); + // make sure it is aligned to 8bit, the allocated memory address is aligned to 256bit +// ASSERT((((uint64_t)pColData->pData) & (0x8 - 1)) == 0); // 1. copy data in a batch model memcpy(pColData->pData, p, dumpedRows * tDataTypes[pData->type].bytes); // 2. reverse the array list in case of descending order scan data block if (!asc) { - switch(pColData->info.type) { + switch (pColData->info.type) { case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - { + case TSDB_DATA_TYPE_UBIGINT: { int32_t mid = dumpedRows >> 1u; int64_t* pts = (int64_t*)pColData->pData; for (int32_t j = 0; j < mid; ++j) { @@ -999,7 +940,7 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_UTINYINT: { - int32_t mid = dumpedRows >> 1u; + int32_t mid = dumpedRows >> 1u; int8_t* pts = (int8_t*)pColData->pData; for (int32_t j = 0; j < mid; ++j) { int8_t t = pts[j]; @@ -1072,11 +1013,20 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn // pDumpInfo->rowIndex = 0; } else if (!asc && pReader->window.ekey >= pBlock->maxKey.ts) { // pDumpInfo->rowIndex = pBlock->nRow - 1; - } else { + } else { // find the appropriate the start position in current block, and set it to be the current rowIndex int32_t pos = asc ? pBlock->nRow - 1 : 0; int32_t order = asc ? TSDB_ORDER_DESC : TSDB_ORDER_ASC; int64_t key = asc ? pReader->window.skey : pReader->window.ekey; pDumpInfo->rowIndex = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, key, order); + + if (pDumpInfo->rowIndex < 0) { + tsdbError( + "%p failed to locate the start position in current block, global index:%d, table index:%d, brange:%" PRId64 + "-%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 " %s", + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->minVer, + pBlock->maxVer, pReader->idStr); + return TSDB_CODE_INVALID_PARA; + } } } @@ -1157,6 +1107,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn setBlockAllDumped(pDumpInfo, ts, pReader->order); } + pBlockScanInfo->lastKey = pDumpInfo->lastKey; + double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; pReader->cost.blockLoadTime += elapsedTime; @@ -1183,7 +1135,6 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - ASSERT(pBlockInfo != NULL); SDataBlk* pBlock = getCurrentBlock(pBlockIter); code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData); @@ -1221,8 +1172,6 @@ static void cleanupBlockOrderSupporter(SBlockOrderSupporter* pSup) { } static int32_t initBlockOrderSupporter(SBlockOrderSupporter* pSup, int32_t numOfTables) { - ASSERT(numOfTables >= 1); - pSup->numOfBlocksPerTable = taosMemoryCalloc(1, sizeof(int32_t) * numOfTables); pSup->indexPerTable = taosMemoryCalloc(1, sizeof(int32_t) * numOfTables); pSup->pDataBlockInfo = taosMemoryCalloc(1, POINTER_BYTES * numOfTables); @@ -1329,7 +1278,10 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte sup.numOfTables += 1; } - ASSERT(numOfBlocks == cnt); + if (numOfBlocks != cnt && sup.numOfTables != numOfTables) { + cleanupBlockOrderSupporter(&sup); + return TSDB_CODE_INVALID_PARA; + } // since there is only one table qualified, blocks are not sorted if (sup.numOfTables == 1) { @@ -1351,10 +1303,9 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables %s", pReader, cnt, sup.numOfTables, pReader->idStr); - ASSERT(cnt <= numOfBlocks && sup.numOfTables <= numOfTables); - SMultiwayMergeTreeInfo* pTree = NULL; - uint8_t ret = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, fileDataBlockOrderCompar); + + uint8_t ret = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, fileDataBlockOrderCompar); if (ret != TSDB_CODE_SUCCESS) { cleanupBlockOrderSupporter(&sup); return TSDB_CODE_OUT_OF_MEMORY; @@ -1432,8 +1383,6 @@ static bool getNeighborBlockOfSameTable(SFileDataBlockInfo* pBlockInfo, STableBl } static int32_t findFileBlockInfoIndex(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pFBlockInfo) { - ASSERT(pBlockIter != NULL && pFBlockInfo != NULL); - int32_t step = ASCENDING_TRAVERSE(pBlockIter->order) ? 1 : -1; int32_t index = pBlockIter->index; @@ -1741,7 +1690,7 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, SIterInfo* pIter, int64_t key, SLastBlockReader* pLastBlockReader) { SRowMerger merge = {0}; - STSRow* pTSRow = NULL; + SRow* pTSRow = NULL; SBlockData* pBlockData = &pReader->status.fileBlockData; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -1789,7 +1738,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (pReader->order == TSDB_ORDER_ASC) { if (minKey == key) { init = true; - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1799,10 +1748,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); } else { init = true; - int32_t code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1812,11 +1761,11 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { if (init) { - tRowMerge(&merge, pRow); + tsdbRowMerge(&merge, pRow); } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1830,7 +1779,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1844,10 +1793,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); } else { init = true; - int32_t code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1857,10 +1806,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { if (init) { - tRowMerge(&merge, &fRow); + tsdbRowMerge(&merge, &fRow); } else { init = true; - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1869,7 +1818,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } } - int32_t code = tRowMergerGetRow(&merge, &pTSRow); + int32_t code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1877,7 +1826,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); return TSDB_CODE_SUCCESS; } @@ -1887,7 +1836,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); - STSRow* pTSRow = NULL; + SRow* pTSRow = NULL; SRowMerger merge = {0}; TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", fRow.pBlockData, fRow.iRow, pLastBlockReader->uid, pReader->idStr); @@ -1898,16 +1847,16 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; } else { - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge, &pReader->verRange); - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1915,23 +1864,22 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); } } else { // not merge block data - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge, &pReader->verRange); - ASSERT(mergeBlockData); // merge with block data if ts == key if (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex]) { doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1939,7 +1887,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); } return TSDB_CODE_SUCCESS; @@ -1964,10 +1912,10 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader if (key < ts) { // imem, mem are all empty, file blocks (data blocks and last block) exist return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); } else if (key == ts) { - STSRow* pTSRow = NULL; + SRow* pTSRow = NULL; SRowMerger merge = {0}; - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1975,11 +1923,11 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge, &pReader->verRange); - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1987,10 +1935,9 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); return code; } else { - ASSERT(0); return TSDB_CODE_SUCCESS; } } else { // desc order @@ -2004,14 +1951,13 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { SRowMerger merge = {0}; - STSRow* pTSRow = NULL; + SRow* pTSRow = NULL; int32_t code = TSDB_CODE_SUCCESS; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SArray* pDelList = pBlockScanInfo->delSkyline; TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pDelList, pReader); TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader); - ASSERT(pRow != NULL && piRow != NULL); int64_t tsLast = INT64_MIN; if (hasDataInLastBlock(pLastBlockReader)) { @@ -2068,7 +2014,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { init = true; TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2079,10 +2025,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); } else { init = true; - code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2093,7 +2039,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == ik.ts) { if (init) { - tRowMerge(&merge, piRow); + tsdbRowMerge(&merge, piRow); } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); @@ -2101,7 +2047,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - code = tRowMergerInit(&merge, piRow, pSchema); + code = tsdbRowMergerInit(&merge, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2120,10 +2066,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - tRowMerge(&merge, pRow); + tsdbRowMerge(&merge, pRow); } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tRowMergerInit(&merge, pRow, pSchema); + code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2138,7 +2084,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tRowMergerInit(&merge, pRow, pSchema); + code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2152,11 +2098,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == ik.ts) { if (init) { - tRowMerge(&merge, piRow); + tsdbRowMerge(&merge, piRow); } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - code = tRowMergerInit(&merge, piRow, pSchema); + code = tsdbRowMergerInit(&merge, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2171,10 +2117,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); } else { init = true; - code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2185,7 +2131,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); if (!init) { - code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2193,7 +2139,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (merge.pTSchema == NULL) { return code; } - tRowMerge(&merge, &fRow); + tsdbRowMerge(&merge, &fRow); } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } @@ -2203,7 +2149,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2211,7 +2157,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); return code; } @@ -2235,7 +2181,6 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea if (pReader->pReadSnap->pMem != NULL) { d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid); if (d != NULL) { - ASSERT(pBlockScanInfo->iter.iter == NULL); code = tsdbTbDataIterCreate(d, &startKey, backward, &pBlockScanInfo->iter.iter); if (code == TSDB_CODE_SUCCESS) { pBlockScanInfo->iter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iter.iter) != NULL); @@ -2349,10 +2294,9 @@ static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->mergeTree.pIter != NULL; } bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo) { - if (pBlockData->nRow > 0) { - ASSERT(pBlockData->nRow == pDumpInfo->totalRows); + if ((pBlockData->nRow > 0) && (pBlockData->nRow != pDumpInfo->totalRows)) { + return false; // this is an invalid result. } - return pBlockData->nRow > 0 && (!pDumpInfo->allDumped); } @@ -2365,16 +2309,16 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc } else { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - STSRow* pTSRow = NULL; + SRow* pTSRow = NULL; SRowMerger merge = {0}; - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2382,7 +2326,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); return TSDB_CODE_SUCCESS; } } @@ -2487,7 +2431,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); // it is a clean block, load it directly - if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader) && + if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader) && pBlock->nRow <= pReader->capacity) { if (asc || ((!asc) && (!hasDataInLastBlock(pLastBlockReader)))) { copyBlockDataToSDataBlock(pReader, pBlockScanInfo); @@ -2507,7 +2451,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { while (1) { bool hasBlockData = false; { - while (pBlockData->nRow > 0) { // find the first qualified row in data block + while (pBlockData->nRow > 0 && pBlockData->uid == pBlockScanInfo->uid) { // find the first qualified row in data block if (isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) { hasBlockData = true; break; @@ -2583,7 +2527,6 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* int32_t code = 0; SArray* pDelData = taosArrayInit(4, sizeof(SDelData)); - ASSERT(pReader->pReadSnap != NULL); SDelFile* pDelFile = pReader->pReadSnap->fs.pDelFile; if (pDelFile && taosArrayGetSize(pReader->pDelIdx) > 0) { @@ -2691,7 +2634,6 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { taosArrayDestroy(pIndexList); if (pReader->pReadSnap != NULL) { - SDelFile* pDelFile = pReader->pReadSnap->fs.pDelFile; if (pReader->pDelFReader == NULL && pDelFile != NULL) { int32_t code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pReader->pTsdb); @@ -2868,7 +2810,6 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); if (pBlockInfo == NULL) { // build data block from last data file - ASSERT(pBlockIter->numOfBlocks == 0); code = buildComposedDataBlock(pReader); } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) { code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); @@ -3042,7 +2983,12 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { } else { if (pReader->status.pCurrentFileset->nSttF > 0) { // data blocks in current file are exhausted, let's try the next file now - tBlockDataReset(&pReader->status.fileBlockData); + SBlockData* pBlockData = &pReader->status.fileBlockData; + if (pBlockData->uid != 0) { + tBlockDataClear(pBlockData); + } + + tBlockDataReset(pBlockData); resetDataBlockIterator(pBlockIter, pReader->order); goto _begin; } else { @@ -3240,7 +3186,8 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p } TSDBROW* pRow = tsdbTbDataIterGet(pIter->iter); - TSDBKEY key = {.ts = pRow->pTSRow->ts, .version = pRow->version}; + TSDBKEY key = TSDBROW_KEY(pRow); + if (outOfTimeWindow(key.ts, &pReader->window)) { pIter->hasVal = false; return NULL; @@ -3293,12 +3240,16 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDe break; } - STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid); - if (pTSchema == NULL) { - return terrno; - } + if (pRow->type == TSDBROW_ROW_FMT) { + STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid); + if (pTSchema == NULL) { + return terrno; + } - tRowMergerAdd(pMerger, pRow, pTSchema); + tsdbRowMergerAdd(pMerger, pRow, pTSchema); + } else { // column format + tsdbRowMerge(pMerger, pRow); + } } return TSDB_CODE_SUCCESS; @@ -3313,7 +3264,7 @@ static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t rowInd } TSDBROW fRow = tsdbRowFromBlockData(pBlockData, rowIndex); - tRowMerge(pMerger, &fRow); + tsdbRowMerge(pMerger, &fRow); rowIndex += step; } @@ -3369,6 +3320,11 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc SFileDataBlockInfo* pFileBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); SDataBlk* pCurrentBlock = getCurrentBlock(&pReader->status.blockIter); + if (pFileBlockInfo == NULL) { + st = CHECK_FILEBLOCK_QUIT; + break; + } + checkForNeighborFileBlock(pReader, pScanInfo, pCurrentBlock, pFileBlockInfo, pMerger, key, &st); if (st == CHECK_FILEBLOCK_QUIT) { break; @@ -3385,7 +3341,7 @@ int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockSc int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 == ts) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tRowMerge(pMerger, &fRow1); + tsdbRowMerge(pMerger, &fRow1); } else { break; } @@ -3394,7 +3350,7 @@ int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockSc return TSDB_CODE_SUCCESS; } -int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, +int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, TSDBROW* pResRow, STsdbReader* pReader, bool* freeTSRow) { TSDBROW* pNextRow = NULL; TSDBROW current = *pRow; @@ -3403,19 +3359,19 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, pIter->hasVal = tsdbTbDataIterNext(pIter->iter); if (!pIter->hasVal) { - *pTSRow = current.pTSRow; + *pResRow = *pRow; *freeTSRow = false; return TSDB_CODE_SUCCESS; } else { // has next point in mem/imem pNextRow = getValidMemRow(pIter, pDelList, pReader); if (pNextRow == NULL) { - *pTSRow = current.pTSRow; + *pResRow = current; *freeTSRow = false; return TSDB_CODE_SUCCESS; } - if (current.pTSRow->ts != pNextRow->pTSRow->ts) { - *pTSRow = current.pTSRow; + if (TSDBROW_TS(¤t) != TSDBROW_TS(pNextRow)) { + *pResRow = current; *freeTSRow = false; return TSDB_CODE_SUCCESS; } @@ -3423,47 +3379,60 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, } SRowMerger merge = {0}; - - // get the correct schema for data in memory terrno = 0; - STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(¤t), pReader, uid); - if (pTSchema == NULL) { - return terrno; - } + int32_t code = 0; - if (pReader->pSchema == NULL) { - pReader->pSchema = pTSchema; - } + // start to merge duplicated rows + if (current.type == TSDBROW_ROW_FMT) { + // get the correct schema for data in memory + STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(¤t), pReader, uid); + if (pTSchema == NULL) { + return terrno; + } - int32_t code = tRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + if (pReader->pSchema == NULL) { + pReader->pSchema = pTSchema; + } - STSchema* pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid); - if (pTSchema1 == NULL) { - return terrno; - } + code = tsdbRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - tRowMergerAdd(&merge, pNextRow, pTSchema1); + STSchema* pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid); + if (pTSchema1 == NULL) { + return terrno; + } - code = doMergeRowsInBuf(pIter, uid, current.pTSRow->ts, pDelList, &merge, pReader); + tsdbRowMergerAdd(&merge, pNextRow, pTSchema1); + } else { // let's merge rows in file block + code = tsdbRowMergerInit(&merge, ¤t, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + tsdbRowMerge(&merge, pNextRow); + } + + code = doMergeRowsInBuf(pIter, uid, TSDBROW_TS(¤t), pDelList, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } - code = tRowMergerGetRow(&merge, pTSRow); + code = tsdbRowMergerGetRow(&merge, &pResRow->pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } - tRowMergerClear(&merge); + pResRow->type = TSDBROW_ROW_FMT; + tsdbRowMergerClear(&merge); *freeTSRow = true; + return TSDB_CODE_SUCCESS; } int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, - STSRow** pTSRow) { + SRow** pTSRow) { SRowMerger merge = {0}; TSDBKEY k = TSDBROW_KEY(pRow); @@ -3472,7 +3441,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, piRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3483,7 +3452,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p return code; } - tRowMerge(&merge, pRow); + tsdbRowMerge(&merge, pRow); code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { @@ -3493,7 +3462,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS || merge.pTSchema == NULL) { return code; } @@ -3504,7 +3473,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p return code; } - tRowMerge(&merge, piRow); + tsdbRowMerge(&merge, piRow); code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { @@ -3512,13 +3481,12 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p } } - int32_t code = tRowMergerGetRow(&merge, pTSRow); - tRowMergerClear(&merge); - + int32_t code = tsdbRowMergerGetRow(&merge, pTSRow); + tsdbRowMergerClear(&merge); return code; } -int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STSRow** pTSRow, int64_t endKey, +int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, TSDBROW* pResRow, int64_t endKey, bool* freeTSRow) { TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); @@ -3548,13 +3516,14 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR int32_t code = TSDB_CODE_SUCCESS; if (ik.ts != k.ts) { if (((ik.ts < k.ts) && asc) || ((ik.ts > k.ts) && (!asc))) { // ik.ts < k.ts - code = doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader, freeTSRow); + code = doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pResRow, pReader, freeTSRow); } else if (((k.ts < ik.ts) && asc) || ((k.ts > ik.ts) && (!asc))) { - code = doMergeMemTableMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, freeTSRow); + code = doMergeMemTableMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, pResRow, pReader, freeTSRow); } } else { // ik.ts == k.ts *freeTSRow = true; - code = doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, pTSRow); + pResRow->type = TSDBROW_ROW_FMT; + code = doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, &pResRow->pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3564,29 +3533,30 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR } if (pBlockScanInfo->iter.hasVal && pRow != NULL) { - return doMergeMemTableMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, + return doMergeMemTableMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pResRow, pReader, freeTSRow); } if (pBlockScanInfo->iiter.hasVal && piRow != NULL) { - return doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader, freeTSRow); + return doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pResRow, pReader, freeTSRow); } return TSDB_CODE_SUCCESS; } -int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, - STableBlockScanInfo* pScanInfo) { +int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, STableBlockScanInfo* pScanInfo) { int32_t outputRowIndex = pBlock->info.rows; int64_t uid = pScanInfo->uid; + int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); + SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; STSchema* pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid); SColVal colVal = {0}; int32_t i = 0, j = 0; - if (pSupInfo->colId[i]== PRIMARYKEY_TIMESTAMP_COL_ID) { + if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts; i += 1; @@ -3598,7 +3568,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* if (colId == pSchema->columns[j].colId) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); - tTSRowGetVal(pTSRow, pSchema, j, &colVal); + tRowGet(pTSRow, pSchema, j, &colVal); doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo); i += 1; j += 1; @@ -3631,7 +3601,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S int32_t outputRowIndex = pResBlock->info.rows; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - if (pReader->suppInfo.colId[i]== PRIMARYKEY_TIMESTAMP_COL_ID) { + if (pReader->suppInfo.colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; i += 1; @@ -3677,17 +3647,22 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e SSDataBlock* pBlock = pReader->pResBlock; do { - STSRow* pTSRow = NULL; + // SRow* pTSRow = NULL; + TSDBROW row = {.type = -1}; bool freeTSRow = false; - tsdbGetNextRowInMem(pBlockScanInfo, pReader, &pTSRow, endKey, &freeTSRow); - if (pTSRow == NULL) { + tsdbGetNextRowInMem(pBlockScanInfo, pReader, &row, endKey, &freeTSRow); + if (row.type == -1) { break; } - doAppendRowFromTSRow(pBlock, pReader, pTSRow, pBlockScanInfo); + if (row.type == TSDBROW_ROW_FMT) { + doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo); - if (freeTSRow) { - taosMemoryFree(pTSRow); + if (freeTSRow) { + taosMemoryFree(row.pTSRow); + } + } else { + doAppendRowFromFileBlock(pBlock, pReader, row.pBlockData, row.iRow); } // no data in buffer, return immediately @@ -3837,15 +3812,16 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL } if (pReader->pSchema != NULL) { - updateBlockSMAInfo(pReader->pSchema, &pReader->suppInfo); + code = updateBlockSMAInfo(pReader->pSchema, &pReader->suppInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } } STsdbReader* p = (pReader->innerReader[0] != NULL) ? pReader->innerReader[0] : pReader; pReader->status.pTableMap = createDataBlockScanInfo(p, &pReader->blockInfoBuf, pTableList, numOfTables); if (pReader->status.pTableMap == NULL) { - tsdbReaderClose(p); *ppReader = NULL; - code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -3934,7 +3910,7 @@ void tsdbReaderClose(STsdbReader* pReader) { } taosMemoryFree(pSupInfo->colId); - tBlockDataDestroy(&pReader->status.fileBlockData, true); + tBlockDataDestroy(&pReader->status.fileBlockData); cleanupDataBlockIterator(&pReader->status.blockIter); size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); @@ -3997,7 +3973,7 @@ static bool doTsdbNextDataBlock(STsdbReader* pReader) { blockDataCleanup(pBlock); SReaderStatus* pStatus = &pReader->status; - if (taosHashGetSize(pStatus->pTableMap) == 0){ + if (taosHashGetSize(pStatus->pTableMap) == 0) { return false; } @@ -4087,12 +4063,10 @@ void tsdbRetrieveDataBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64 } } - -static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, - SColumnDataAgg* pTsAgg) { +static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) { // do fill all null column value SMA info int32_t i = 0, j = 0; - int32_t size = (int32_t) taosArrayGetSize(pSup->pColAgg); + int32_t size = (int32_t)taosArrayGetSize(pSup->pColAgg); taosArrayInsert(pSup->pColAgg, 0, pTsAgg); while (j < numOfCols && i < size) { @@ -4105,7 +4079,7 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ } else if (pSup->colId[j] < pAgg->colId) { if (pSup->colId[j] != PRIMARYKEY_TIMESTAMP_COL_ID) { SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows}; - taosArrayInsert(pSup->pColAgg, i ,&nullColAgg); + taosArrayInsert(pSup->pColAgg, i, &nullColAgg); } j += 1; } @@ -4113,25 +4087,27 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ } int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave) { + SColumnDataAgg*** pBlockSMA = &pDataBlock->pBlockAgg; + int32_t code = 0; - SColumnDataAgg ***pBlockSMA = &pDataBlock->pBlockAgg; *allHave = false; + *pBlockSMA = NULL; if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { - *pBlockSMA = NULL; return TSDB_CODE_SUCCESS; } // there is no statistics data for composed block if (pReader->status.composedDataBlock || (!pReader->suppInfo.smaValid)) { - *pBlockSMA = NULL; return TSDB_CODE_SUCCESS; } SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter); SBlockLoadSuppInfo* pSup = &pReader->suppInfo; - ASSERT(pReader->pResBlock->info.id.uid == pFBlock->uid); + if (pReader->pResBlock->info.id.uid != pFBlock->uid) { + return TSDB_CODE_SUCCESS; + } SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); if (tDataBlkHasSma(pBlock)) { @@ -4159,13 +4135,10 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, // update the number of NULL data rows size_t numOfCols = pSup->numOfCols; - int32_t i = 0, j = 0; - size_t size = taosArrayGetSize(pSup->pColAgg); - // ensure capacity - if(pDataBlock->pDataBlock) { - size_t colsNum = taosArrayGetSize(pDataBlock->pDataBlock); - taosArrayEnsureCap(pSup->pColAgg, colsNum); + if (pDataBlock->pDataBlock) { + size_t colsNum = taosArrayGetSize(pDataBlock->pDataBlock); + taosArrayEnsureCap(pSup->pColAgg, colsNum); } SSDataBlock* pResBlock = pReader->pResBlock; @@ -4176,8 +4149,9 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, // do fill all null column value SMA info doFillNullColSMA(pSup, pBlock->nRow, numOfCols, pTsAgg); + size_t size = taosArrayGetSize(pSup->pColAgg); - i = 0, j = 0; + int32_t i = 0, j = 0; while (j < numOfCols && i < size) { SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i); if (pAgg->colId == pSup->colId[j]) { @@ -4187,7 +4161,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, } else if (pAgg->colId < pSup->colId[j]) { i += 1; } else if (pSup->colId[j] < pAgg->colId) { - ASSERT(pSup->colId[j] == PRIMARYKEY_TIMESTAMP_COL_ID); + // ASSERT(pSup->colId[j] == PRIMARYKEY_TIMESTAMP_COL_ID); pResBlock->pBlockAgg[pSup->slotId[j]] = &pSup->tsColAgg; j += 1; } @@ -4219,7 +4193,7 @@ static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { - tBlockDataDestroy(&pStatus->fileBlockData, 1); + tBlockDataDestroy(&pStatus->fileBlockData); terrno = code; return NULL; } @@ -4420,9 +4394,12 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 return terrno; } sversion = mr.me.stbEntry.schemaRow.version; - } else { - ASSERT(mr.me.type == TSDB_NORMAL_TABLE); + } else if (mr.me.type == TSDB_NORMAL_TABLE) { sversion = mr.me.ntbEntry.schemaRow.version; + } else { + terrno = TSDB_CODE_INVALID_PARA; + metaReaderClear(&mr); + return terrno; } metaReaderClear(&mr); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 99acc249926f001fb795f5bc0302a1d1cb3e1139..db29d75f174d6a09506f039152ff182bd7e0ac39 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -538,7 +538,7 @@ _exit: if (pReader) { taosArrayDestroy(pReader->aDelData); taosArrayDestroy(pReader->aDelIdx); - tBlockDataDestroy(&pReader->bData, 1); + tBlockDataDestroy(&pReader->bData); tsdbFSDestroy(&pReader->fs); taosMemoryFree(pReader); } @@ -565,10 +565,10 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { taosArrayDestroy(pIter->aSttBlk); } - tBlockDataDestroy(&pIter->bData, 1); + tBlockDataDestroy(&pIter->bData); } - tBlockDataDestroy(&pReader->bData, 1); + tBlockDataDestroy(&pReader->bData); tDestroyTSchema(pReader->skmTable.pTSchema); // del @@ -1359,13 +1359,13 @@ _exit: if (pWriter->aDelIdxW) taosArrayDestroy(pWriter->aDelIdxW); if (pWriter->aDelData) taosArrayDestroy(pWriter->aDelData); if (pWriter->aDelIdxR) taosArrayDestroy(pWriter->aDelIdxR); - tBlockDataDestroy(&pWriter->dWriter.sData, 1); - tBlockDataDestroy(&pWriter->dWriter.bData, 1); + tBlockDataDestroy(&pWriter->dWriter.sData); + tBlockDataDestroy(&pWriter->dWriter.bData); if (pWriter->dWriter.aSttBlk) taosArrayDestroy(pWriter->dWriter.aSttBlk); if (pWriter->dWriter.aBlockIdx) taosArrayDestroy(pWriter->dWriter.aBlockIdx); - tBlockDataDestroy(&pWriter->dReader.bData, 1); + tBlockDataDestroy(&pWriter->dReader.bData); if (pWriter->dReader.aBlockIdx) taosArrayDestroy(pWriter->dReader.aBlockIdx); - tBlockDataDestroy(&pWriter->bData, 1); + tBlockDataDestroy(&pWriter->bData); tsdbFSDestroy(&pWriter->fs); taosMemoryFree(pWriter); } @@ -1425,18 +1425,18 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { // SNAP_DATA_TSDB // Writer - tBlockDataDestroy(&pWriter->dWriter.sData, 1); - tBlockDataDestroy(&pWriter->dWriter.bData, 1); + tBlockDataDestroy(&pWriter->dWriter.sData); + tBlockDataDestroy(&pWriter->dWriter.bData); taosArrayDestroy(pWriter->dWriter.aSttBlk); tMapDataClear(&pWriter->dWriter.mDataBlk); taosArrayDestroy(pWriter->dWriter.aBlockIdx); // Reader - tBlockDataDestroy(&pWriter->dReader.bData, 1); + tBlockDataDestroy(&pWriter->dReader.bData); tMapDataClear(&pWriter->dReader.mDataBlk); taosArrayDestroy(pWriter->dReader.aBlockIdx); - tBlockDataDestroy(&pWriter->bData, 1); + tBlockDataDestroy(&pWriter->bData); tDestroyTSchema(pWriter->skmTable.pTSchema); for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) { diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 55703002b80a518d3d572dab6dc5368910dcfbc2..1c92e922b0004c6ff9cc94b5e6fafa80b638cc0f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -23,7 +23,7 @@ void tMapDataReset(SMapData *pMapData) { } void tMapDataClear(SMapData *pMapData) { - tFree((uint8_t *)pMapData->aOffset); + tFree(pMapData->aOffset); tFree(pMapData->pData); pMapData->pData = NULL; pMapData->aOffset = NULL; @@ -572,9 +572,9 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal * ASSERT(iCol > 0); - if (pRow->type == 0) { - tTSRowGetVal(pRow->pTSRow, pTSchema, iCol, pColVal); - } else if (pRow->type == 1) { + if (pRow->type == TSDBROW_ROW_FMT) { + tRowGet(pRow->pTSRow, pTSchema, iCol, pColVal); + } else if (pRow->type == TSDBROW_COL_FMT) { SColData *pColData; tBlockDataGetColData(pRow->pBlockData, pTColumn->colId, &pColData); @@ -589,60 +589,61 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal * } } -// int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow) { -// int32_t n = 0; - -// n += tPutI64(p, pRow->version); -// if (p) memcpy(p + n, pRow->pTSRow, pRow->pTSRow->len); -// n += pRow->pTSRow->len; - -// return n; -// } - int32_t tsdbRowCmprFn(const void *p1, const void *p2) { return tsdbKeyCmprFn(&TSDBROW_KEY((TSDBROW *)p1), &TSDBROW_KEY((TSDBROW *)p2)); } // STSDBRowIter ====================================================== -void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { + int32_t code = 0; + pIter->pRow = pRow; - if (pRow->type == 0) { - ASSERT(pTSchema); - pIter->pTSchema = pTSchema; - pIter->i = 1; - } else if (pRow->type == 1) { - pIter->pTSchema = NULL; - pIter->i = 0; + if (pRow->type == TSDBROW_ROW_FMT) { + code = tRowIterOpen(pRow->pTSRow, pTSchema, &pIter->pIter); + if (code) goto _exit; + } else if (pRow->type == TSDBROW_COL_FMT) { + pIter->iColData = 0; } else { ASSERT(0); } + +_exit: + return code; } -SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { - if (pIter->pRow->type == 0) { - if (pIter->i < pIter->pTSchema->numOfCols) { - tTSRowGetVal(pIter->pRow->pTSRow, pIter->pTSchema, pIter->i, &pIter->colVal); - pIter->i++; +void tsdbRowClose(STSDBRowIter *pIter) { + if (pIter->pRow->type == TSDBROW_ROW_FMT) { + tRowIterClose(&pIter->pIter); + } +} - return &pIter->colVal; +SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { + if (pIter->pRow->type == TSDBROW_ROW_FMT) { + return tRowIterNext(pIter->pIter); + } else if (pIter->pRow->type == TSDBROW_COL_FMT) { + if (pIter->iColData == 0) { + pIter->cv = COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, + (SValue){.val = pIter->pRow->pBlockData->aTSKEY[pIter->pRow->iRow]}); + ++pIter->iColData; + return &pIter->cv; } - } else { - if (pIter->i < pIter->pRow->pBlockData->nColData) { - SColData *pColData = tBlockDataGetColDataByIdx(pIter->pRow->pBlockData, pIter->i); - - tColDataGetValue(pColData, pIter->pRow->iRow, &pIter->colVal); - pIter->i++; - return &pIter->colVal; + if (pIter->iColData < pIter->pRow->pBlockData->nColData) { + tColDataGetValue(&pIter->pRow->pBlockData->aColData[pIter->iColData], pIter->pRow->iRow, &pIter->cv); + ++pIter->iColData; + return &pIter->cv; + } else { + return NULL; } + } else { + ASSERT(0); + return NULL; // suppress error report by compiler } - - return NULL; } // SRowMerger ====================================================== -int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; @@ -697,7 +698,7 @@ _exit: return code; } -int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; @@ -736,7 +737,7 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { return code; } -int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; @@ -775,9 +776,9 @@ _exit: return code; } -void tRowMergerClear(SRowMerger *pMerger) { taosArrayDestroy(pMerger->pArray); } +void tsdbRowMergerClear(SRowMerger *pMerger) { taosArrayDestroy(pMerger->pArray); } -int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { +int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; @@ -807,12 +808,8 @@ _exit: return code; } -int32_t tRowMergerGetRow(SRowMerger *pMerger, STSRow **ppRow) { - int32_t code = 0; - - code = tdSTSRowNew(pMerger->pArray, pMerger->pTSchema, ppRow); - - return code; +int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow) { + return tRowBuild(pMerger->pArray, pMerger->pTSchema, ppRow); } // delete skyline ====================================================== @@ -931,27 +928,49 @@ int32_t tBlockDataCreate(SBlockData *pBlockData) { pBlockData->aVersion = NULL; pBlockData->aTSKEY = NULL; pBlockData->nColData = 0; - pBlockData->aColData = taosArrayInit(0, sizeof(SColData)); - if (pBlockData->aColData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } + pBlockData->aColData = NULL; _exit: return code; } -void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear) { - tFree((uint8_t *)pBlockData->aUid); - tFree((uint8_t *)pBlockData->aVersion); - tFree((uint8_t *)pBlockData->aTSKEY); - taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataDestroy : NULL); - pBlockData->aUid = NULL; - pBlockData->aVersion = NULL; - pBlockData->aTSKEY = NULL; - pBlockData->aColData = NULL; +void tBlockDataDestroy(SBlockData *pBlockData) { + tFree(pBlockData->aUid); + tFree(pBlockData->aVersion); + tFree(pBlockData->aTSKEY); + + for (int32_t i = 0; i < pBlockData->nColData; i++) { + tColDataDestroy(&pBlockData->aColData[i]); + } + + if (pBlockData->aColData) { + taosMemoryFree(pBlockData->aColData); + pBlockData->aColData = NULL; + } } +static int32_t tBlockDataAdjustColData(SBlockData *pBlockData, int32_t nColData) { + int32_t code = 0; + + if (pBlockData->nColData > nColData) { + for (int32_t i = nColData; i < pBlockData->nColData; i++) { + tColDataDestroy(&pBlockData->aColData[i]); + } + } else if (pBlockData->nColData < nColData) { + SColData *aColData = taosMemoryRealloc(pBlockData->aColData, sizeof(SBlockData) * nColData); + if (aColData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + pBlockData->aColData = aColData; + memset(&pBlockData->aColData[pBlockData->nColData], 0, sizeof(SBlockData) * (nColData - pBlockData->nColData)); + } + pBlockData->nColData = nColData; + +_exit: + return code; +} int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid) { int32_t code = 0; @@ -961,37 +980,35 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, pBlockData->uid = pId->uid; pBlockData->nRow = 0; - pBlockData->nColData = 0; if (aCid) { + code = tBlockDataAdjustColData(pBlockData, nCid); + if (code) goto _exit; + int32_t iColumn = 1; STColumn *pTColumn = &pTSchema->columns[iColumn]; for (int32_t iCid = 0; iCid < nCid; iCid++) { - while (pTColumn && pTColumn->colId < aCid[iCid]) { + ASSERT(pTColumn); + while (pTColumn->colId < aCid[iCid]) { iColumn++; - pTColumn = (iColumn < pTSchema->numOfCols) ? &pTSchema->columns[iColumn] : NULL; + ASSERT(iColumn < pTSchema->numOfCols); + pTColumn = &pTSchema->columns[iColumn]; } - if (pTColumn == NULL) { - break; - } else if (pTColumn->colId == aCid[iCid]) { - SColData *pColData; - code = tBlockDataAddColData(pBlockData, &pColData); - if (code) goto _exit; - tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) ? 1 : 0); + ASSERT(pTColumn->colId == aCid[iCid]); + tColDataInit(&pBlockData->aColData[iCid], pTColumn->colId, pTColumn->type, + (pTColumn->flags & COL_SMA_ON) ? 1 : 0); - iColumn++; - pTColumn = (iColumn < pTSchema->numOfCols) ? &pTSchema->columns[iColumn] : NULL; - } + iColumn++; + pTColumn = (iColumn < pTSchema->numOfCols) ? &pTSchema->columns[iColumn] : NULL; } } else { - for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { - STColumn *pTColumn = &pTSchema->columns[iColumn]; - - SColData *pColData; - code = tBlockDataAddColData(pBlockData, &pColData); - if (code) goto _exit; + code = tBlockDataAdjustColData(pBlockData, pTSchema->numOfCols - 1); + if (code) goto _exit; - tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) ? 1 : 0); + for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { + STColumn *pTColumn = &pTSchema->columns[iColData + 1]; + tColDataInit(&pBlockData->aColData[iColData], pTColumn->colId, pTColumn->type, + (pTColumn->flags & COL_SMA_ON) ? 1 : 0); } } @@ -1002,8 +1019,6 @@ _exit: void tBlockDataReset(SBlockData *pBlockData) { pBlockData->suid = 0; pBlockData->uid = 0; - pBlockData->nRow = 0; - pBlockData->nColData = 0; } void tBlockDataClear(SBlockData *pBlockData) { @@ -1011,49 +1026,22 @@ void tBlockDataClear(SBlockData *pBlockData) { pBlockData->nRow = 0; for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - tColDataClear(pColData); + tColDataClear(tBlockDataGetColDataByIdx(pBlockData, iColData)); } } -int32_t tBlockDataAddColData(SBlockData *pBlockData, SColData **ppColData) { - int32_t code = 0; - SColData *pColData = NULL; - - if (pBlockData->nColData >= taosArrayGetSize(pBlockData->aColData)) { - if (taosArrayPush(pBlockData->aColData, &((SColData){0})) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - } - pColData = (SColData *)taosArrayGet(pBlockData->aColData, pBlockData->nColData); - - pBlockData->nColData++; - - *ppColData = pColData; - return code; - -_err: - *ppColData = NULL; - return code; -} - static int32_t tBlockDataAppendBlockRow(SBlockData *pBlockData, SBlockData *pBlockDataFrom, int32_t iRow) { int32_t code = 0; SColVal cv = {0}; int32_t iColDataFrom = 0; - SColData *pColDataFrom = - (iColDataFrom < pBlockDataFrom->nColData) ? &((SColData *)pBlockDataFrom->aColData->pData)[iColDataFrom] : NULL; + SColData *pColDataFrom = (iColDataFrom < pBlockDataFrom->nColData) ? &pBlockDataFrom->aColData[iColDataFrom] : NULL; for (int32_t iColDataTo = 0; iColDataTo < pBlockData->nColData; iColDataTo++) { - SColData *pColDataTo = &((SColData *)pBlockData->aColData->pData)[iColDataTo]; + SColData *pColDataTo = &pBlockData->aColData[iColDataTo]; while (pColDataFrom && pColDataFrom->cid < pColDataTo->cid) { - iColDataFrom++; - pColDataFrom = (iColDataFrom < pBlockDataFrom->nColData) - ? &((SColData *)pBlockDataFrom->aColData->pData)[iColDataFrom] - : NULL; + pColDataFrom = (++iColDataFrom < pBlockDataFrom->nColData) ? &pBlockDataFrom->aColData[iColDataFrom] : NULL; } if (pColDataFrom == NULL || pColDataFrom->cid > pColDataTo->cid) { @@ -1065,157 +1053,7 @@ static int32_t tBlockDataAppendBlockRow(SBlockData *pBlockData, SBlockData *pBlo code = tColDataAppendValue(pColDataTo, &cv); if (code) goto _exit; - iColDataFrom++; - pColDataFrom = (iColDataFrom < pBlockDataFrom->nColData) - ? &((SColData *)pBlockDataFrom->aColData->pData)[iColDataFrom] - : NULL; - } - } - -_exit: - return code; -} - -static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSchema *pTSchema) { - int32_t code = 0; - - int32_t iTColumn = 1; - STColumn *pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - void *pBitmap = pRow->statis ? tdGetBitmapAddrTp(pRow, pTSchema->flen) : NULL; - - for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = &((SColData *)pBlockData->aColData->pData)[iColData]; - - while (pTColumn && pTColumn->colId < pColData->cid) { - iTColumn++; - pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - } - - if (pTColumn == NULL || pTColumn->colId > pColData->cid) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } else { - ASSERT(pTColumn->type == pColData->type); - - SColVal cv = {.cid = pTColumn->colId, .type = pTColumn->type}; - - if (pRow->statis) { - TDRowValT vt = TD_VTYPE_MAX; - tdGetBitmapValTypeII(pBitmap, iTColumn - 1, &vt); - - if (vt == TD_VTYPE_NORM) { - cv.flag = CV_FLAG_VALUE; - - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); - cv.value.nData = varDataLen(pData); - cv.value.pData = varDataVal(pData); - } else { - memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); - } - - code = tColDataAppendValue(pColData, &cv); - if (code) goto _exit; - } else if (vt == TD_VTYPE_NONE) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } else if (vt == TD_VTYPE_NULL) { - code = tColDataAppendValue(pColData, &COL_VAL_NULL(pColData->cid, pColData->type)); - if (code) goto _exit; - } else { - ASSERT(0); - } - } else { - cv.flag = CV_FLAG_VALUE; - - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); - cv.value.nData = varDataLen(pData); - cv.value.pData = varDataVal(pData); - } else { - memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); - } - - code = tColDataAppendValue(pColData, &cv); - if (code) goto _exit; - } - - iTColumn++; - pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - } - } - -_exit: - return code; -} - -static int32_t tBlockDataAppendKVRow(SBlockData *pBlockData, STSRow *pRow, STSchema *pTSchema) { - int32_t code = 0; - - col_id_t kvIter = 0; - col_id_t nKvCols = tdRowGetNCols(pRow) - 1; - void *pColIdx = TD_ROW_COL_IDX(pRow); - void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow)); - int32_t iTColumn = 1; - STColumn *pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - - for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = &((SColData *)pBlockData->aColData->pData)[iColData]; - - while (pTColumn && pTColumn->colId < pColData->cid) { - iTColumn++; - pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - } - - if (pTColumn == NULL || pTColumn->colId > pColData->cid) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } else { - ASSERT(pTColumn->type == pColData->type); - - SColVal cv = {.cid = pTColumn->colId, .type = pTColumn->type}; - TDRowValT vt = TD_VTYPE_NONE; // default is NONE - SKvRowIdx *pKvIdx = NULL; - - while (kvIter < nKvCols) { - pKvIdx = (SKvRowIdx *)POINTER_SHIFT(pColIdx, kvIter * sizeof(SKvRowIdx)); - if (pKvIdx->colId == pTColumn->colId) { - tdGetBitmapValTypeII(pBitmap, kvIter, &vt); - ++kvIter; - break; - } else if (pKvIdx->colId > pTColumn->colId) { - vt = TD_VTYPE_NONE; - break; - } else { - ++kvIter; - } - } - - if (vt == TD_VTYPE_NORM) { - cv.flag = CV_FLAG_VALUE; - - void *pData = POINTER_SHIFT(pRow, pKvIdx->offset); - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - cv.value.nData = varDataLen(pData); - cv.value.pData = varDataVal(pData); - } else { - memcpy(&cv.value.val, pData, pTColumn->bytes); - } - - code = tColDataAppendValue(pColData, &cv); - if (code) goto _exit; - } else if (vt == TD_VTYPE_NONE) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } else if (vt == TD_VTYPE_NULL) { - code = tColDataAppendValue(pColData, &COL_VAL_NULL(pColData->cid, pColData->type)); - if (code) goto _exit; - } else { - ASSERT(0); - } - - iTColumn++; - pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + pColDataFrom = (++iColDataFrom < pBlockDataFrom->nColData) ? &pBlockDataFrom->aColData[iColDataFrom] : NULL; } } @@ -1245,19 +1083,14 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS pBlockData->aTSKEY[pBlockData->nRow] = TSDBROW_TS(pRow); SColVal cv = {0}; - if (pRow->type == 0) { - if (TD_IS_TP_ROW(pRow->pTSRow)) { - code = tBlockDataAppendTPRow(pBlockData, pRow->pTSRow, pTSchema); - if (code) goto _err; - } else if (TD_IS_KV_ROW(pRow->pTSRow)) { - code = tBlockDataAppendKVRow(pBlockData, pRow->pTSRow, pTSchema); - if (code) goto _err; - } else { - ASSERT(0); - } - } else { + if (pRow->type == TSDBROW_ROW_FMT) { + code = tRowAppendToColData(pRow->pTSRow, pTSchema, pBlockData->aColData, pBlockData->nColData); + if (code) goto _err; + } else if (pRow->type == TSDBROW_COL_FMT) { code = tBlockDataAppendBlockRow(pBlockData, pRow->pBlockData, pRow->iRow); if (code) goto _err; + } else { + ASSERT(0); } pBlockData->nRow++; @@ -1267,133 +1100,13 @@ _err: return code; } -int32_t tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFrom) { - int32_t code = 0; - - int32_t iColData = 0; - for (int32_t iColDataFrom = 0; iColDataFrom < pBlockDataFrom->nColData; iColDataFrom++) { - SColData *pColDataFrom = tBlockDataGetColDataByIdx(pBlockDataFrom, iColDataFrom); - - while (true) { - SColData *pColData; - if (iColData < pBlockData->nColData) { - pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - } else { - pColData = NULL; - } - - if (pColData == NULL || pColData->cid > pColDataFrom->cid) { - code = tBlockDataAddColData(pBlockData, &pColData); - if (code) goto _exit; - - tColDataInit(pColData, pColDataFrom->cid, pColDataFrom->type, pColDataFrom->smaOn); - for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } - - iColData++; - break; - } else if (pColData->cid == pColDataFrom->cid) { - iColData++; - break; - } else { - iColData++; - } - } - } - -_exit: - return code; -} - -int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData) { - int32_t code = 0; - - ASSERT(pBlockData->suid == pBlockData1->suid); - ASSERT(pBlockData->uid == pBlockData1->uid); - ASSERT(pBlockData1->nRow > 0); - ASSERT(pBlockData2->nRow > 0); - - tBlockDataClear(pBlockData); - - TSDBROW row1 = tsdbRowFromBlockData(pBlockData1, 0); - TSDBROW row2 = tsdbRowFromBlockData(pBlockData2, 0); - TSDBROW *pRow1 = &row1; - TSDBROW *pRow2 = &row2; - - while (pRow1 && pRow2) { - int32_t c = tsdbRowCmprFn(pRow1, pRow2); - - if (c < 0) { - code = tBlockDataAppendRow(pBlockData, pRow1, NULL, - pBlockData1->uid ? pBlockData1->uid : pBlockData1->aUid[pRow1->iRow]); - if (code) goto _exit; - - pRow1->iRow++; - if (pRow1->iRow < pBlockData1->nRow) { - *pRow1 = tsdbRowFromBlockData(pBlockData1, pRow1->iRow); - } else { - pRow1 = NULL; - } - } else if (c > 0) { - code = tBlockDataAppendRow(pBlockData, pRow2, NULL, - pBlockData2->uid ? pBlockData2->uid : pBlockData2->aUid[pRow2->iRow]); - if (code) goto _exit; - - pRow2->iRow++; - if (pRow2->iRow < pBlockData2->nRow) { - *pRow2 = tsdbRowFromBlockData(pBlockData2, pRow2->iRow); - } else { - pRow2 = NULL; - } - } else { - ASSERT(0); - } - } - - while (pRow1) { - code = tBlockDataAppendRow(pBlockData, pRow1, NULL, - pBlockData1->uid ? pBlockData1->uid : pBlockData1->aUid[pRow1->iRow]); - if (code) goto _exit; - - pRow1->iRow++; - if (pRow1->iRow < pBlockData1->nRow) { - *pRow1 = tsdbRowFromBlockData(pBlockData1, pRow1->iRow); - } else { - pRow1 = NULL; - } - } - - while (pRow2) { - code = tBlockDataAppendRow(pBlockData, pRow2, NULL, - pBlockData2->uid ? pBlockData2->uid : pBlockData2->aUid[pRow2->iRow]); - if (code) goto _exit; - - pRow2->iRow++; - if (pRow2->iRow < pBlockData2->nRow) { - *pRow2 = tsdbRowFromBlockData(pBlockData2, pRow2->iRow); - } else { - pRow2 = NULL; - } - } - -_exit: - return code; -} - -SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx) { - ASSERT(idx >= 0 && idx < pBlockData->nColData); - return (SColData *)taosArrayGet(pBlockData->aColData, idx); -} - void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData) { ASSERT(cid != PRIMARYKEY_TIMESTAMP_COL_ID); int32_t lidx = 0; int32_t ridx = pBlockData->nColData - 1; while (lidx <= ridx) { - int32_t midx = (lidx + ridx) / 2; + int32_t midx = (lidx + ridx) >> 1; SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, midx); int32_t c = (pColData->cid == cid) ? 0 : ((pColData->cid > cid) ? 1 : -1); @@ -1540,15 +1253,25 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin // loop to decode each column data if (hdr.szBlkCol == 0) goto _exit; + int32_t nColData = 0; int32_t nt = 0; while (nt < hdr.szBlkCol) { SBlockCol blockCol = {0}; nt += tGetBlockCol(pIn + n + nt, &blockCol); - ASSERT(nt <= hdr.szBlkCol); + ++nColData; + } + ASSERT(nt == hdr.szBlkCol); - SColData *pColData; - code = tBlockDataAddColData(pBlockData, &pColData); - if (code) goto _exit; + code = tBlockDataAdjustColData(pBlockData, nColData); + if (code) goto _exit; + + nt = 0; + int32_t iColData = 0; + while (nt < hdr.szBlkCol) { + SBlockCol blockCol = {0}; + nt += tGetBlockCol(pIn + n + nt, &blockCol); + + SColData *pColData = &pBlockData->aColData[iColData++]; tColDataInit(pColData, blockCol.cid, blockCol.type, blockCol.smaOn); if (blockCol.flag == HAS_NULL) { diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 49d5eaac43c6dbd2356bf95ca4cfb9c70cb350c7..7329bd65fbf804dbb16af721e975f8f772512e97 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -26,14 +26,17 @@ static int64_t tsMaxKeyByPrecision[] = {31556995200000L, 31556995200000000L, 921 // static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); -int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *pRsp) { - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - int32_t affectedrows = 0; - int32_t numOfRows = 0; +int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2 *pRsp) { + int32_t arrSize = 0; + int32_t affectedrows = 0; + int32_t numOfRows = 0; ASSERT(pTsdb->mem != NULL); + if (pMsg) { + arrSize = taosArrayGetSize(pMsg->aSubmitTbData); + } + // scan and convert if (tsdbScanAndConvertSubmitMsg(pTsdb, pMsg) < 0) { if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) { @@ -43,18 +46,10 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp * } // loop to insert - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { - return -1; - } - while (true) { - SSubmitBlkRsp r = {0}; - tGetSubmitMsgNext(&msgIter, &pBlock); - if (pBlock == NULL) break; - if ((terrno = tsdbInsertTableData(pTsdb, version, &msgIter, pBlock, &r)) < 0) { + for (int32_t i = 0; i < arrSize; ++i) { + if ((terrno = tsdbInsertTableData(pTsdb, version, taosArrayGet(pMsg->aSubmitTbData, i), &affectedrows)) < 0) { return -1; } - - numOfRows += msgIter.numOfRows; } if (pRsp != NULL) { @@ -82,9 +77,8 @@ static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, STable *pTable, STSRow * } #endif -static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, STSRow *row, TSKEY minKey, TSKEY maxKey, +static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, TSKEY rowKey, TSKEY minKey, TSKEY maxKey, TSKEY now) { - TSKEY rowKey = TD_ROW_KEY(row); if (rowKey < minKey || rowKey > maxKey) { tsdbError("vgId:%d, table uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 " maxKey %" PRId64 " row key %" PRId64, @@ -96,6 +90,7 @@ static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, STSRow *ro return 0; } +#if 0 int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { ASSERT(pMsg != NULL); // STsdbMeta * pMeta = pTsdb->tsdbMeta; @@ -163,6 +158,46 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { } } + if (terrno != TSDB_CODE_SUCCESS) return -1; + return 0; +} +#endif + +int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) { + ASSERT(pMsg != NULL); + STsdbKeepCfg *pCfg = &pTsdb->keepCfg; + TSKEY now = taosGetTimestamp(pCfg->precision); + TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2; + TSKEY maxKey = tsMaxKeyByPrecision[pCfg->precision]; + int32_t size = taosArrayGetSize(pMsg->aSubmitTbData); + + terrno = TSDB_CODE_SUCCESS; + + for (int32_t i = 0; i < size; ++i) { + SSubmitTbData *pData = TARRAY_GET_ELEM(pMsg->aSubmitTbData, i); + if (pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + uint64_t nColData = TARRAY_SIZE(pData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pData->aCol); + if (nColData > 0) { + int32_t nRows = aColData[0].nVal; + TSKEY *aKey = (TSKEY *)aColData[0].pData; + for (int32_t r = 0; r < nRows; ++r) { + if (tsdbCheckRowRange(pTsdb, pData->uid, aKey[r], minKey, maxKey, now) < 0) { + return -1; + } + } + } + } else { + int32_t nRows = taosArrayGetSize(pData->aRowP); + for (int32_t r = 0; r < nRows; ++r) { + SRow *pRow = (SRow *)taosArrayGetP(pData->aRowP, r); + if (tsdbCheckRowRange(pTsdb, pData->uid, pRow->ts, minKey, maxKey, now) < 0) { + return -1; + } + } + } + } + if (terrno != TSDB_CODE_SUCCESS) return -1; return 0; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 83a414dae08bd48e41e12682922daf7055e1f5a0..e67f2cc59ae27da623add915c0cab3f30bd7e75f 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -123,6 +123,7 @@ void vnodeBufPoolReset(SVBufPool *pPool) { pPool->ptr = pPool->node.data; } + void *vnodeBufPoolMallocAligned(SVBufPool *pPool, int size) { SVBufPoolNode *pNode; void *p = NULL; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 2e0370a81cd128f5ba42122a417d7faa96525544..0874a88cfe733be6d2f63a8310a76070b1b426e4 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -184,18 +184,51 @@ _err: return -1; } -static int32_t vnodePrepareCommit(SVnode *pVnode) { +static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) { int32_t code = 0; + int32_t lino = 0; + char dir[TSDB_FILENAME_LEN] = {0}; + tsem_wait(&pVnode->canCommit); + pVnode->state.commitTerm = pVnode->state.applyTerm; + + pInfo->info.config = pVnode->config; + pInfo->info.state.committed = pVnode->state.applied; + pInfo->info.state.commitTerm = pVnode->state.applyTerm; + pInfo->info.state.commitID = pVnode->state.commitID; + pInfo->pVnode = pVnode; + pInfo->txn = metaGetTxn(pVnode->pMeta); + + // save info + if (pVnode->pTfs) { + snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); + } else { + snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); + } + if (vnodeSaveInfo(dir, &pInfo->info) < 0) { + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + } + tsdbPrepareCommit(pVnode->pTsdb); + metaPrepareAsyncCommit(pVnode->pMeta); + code = smaPrepareAsyncCommit(pVnode->pSma); if (code) goto _exit; -_exit: vnodeBufPoolUnRef(pVnode->inUse); pVnode->inUse = NULL; + +_exit: + if (code) { + vError("vgId:%d, %s failed at line %d since %s, commit id:%" PRId64, TD_VID(pVnode), __func__, lino, + tstrerror(code), pVnode->state.commitID); + } else { + vDebug("vgId:%d, %s done", TD_VID(pVnode), __func__); + } + return code; } @@ -217,33 +250,32 @@ _exit: int vnodeAsyncCommit(SVnode *pVnode) { int32_t code = 0; - // prepare to commit - code = vnodePrepareCommit(pVnode); - if (code) goto _exit; - - // schedule the task - pVnode->state.commitTerm = pVnode->state.applyTerm; - SCommitInfo *pInfo = (SCommitInfo *)taosMemoryCalloc(1, sizeof(*pInfo)); if (NULL == pInfo) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - pInfo->info.config = pVnode->config; - pInfo->info.state.committed = pVnode->state.applied; - pInfo->info.state.commitTerm = pVnode->state.applyTerm; - pInfo->info.state.commitID = pVnode->state.commitID; - pInfo->pVnode = pVnode; - pInfo->txn = metaGetTxn(pVnode->pMeta); + + // prepare to commit + code = vnodePrepareCommit(pVnode, pInfo); + if (TSDB_CODE_SUCCESS != code) { + goto _exit; + } + + // schedule the task code = vnodeScheduleTask(vnodeCommitTask, pInfo); _exit: if (code) { + if (NULL != pInfo) { + taosMemoryFree(pInfo); + } tsem_post(&pVnode->canCommit); vError("vgId:%d, %s failed since %s, commit id:%" PRId64, TD_VID(pVnode), __func__, tstrerror(code), pVnode->state.commitID); } else { - vDebug("vgId:%d, %s done", TD_VID(pVnode), __func__); + vInfo("vgId:%d, vnode async commit done, commitId:%" PRId64 " term:%" PRId64 " applied:%" PRId64, TD_VID(pVnode), + pVnode->state.commitID, pVnode->state.applyTerm, pVnode->state.applied); } return code; } @@ -262,7 +294,7 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) { char dir[TSDB_FILENAME_LEN] = {0}; SVnode *pVnode = pInfo->pVnode; - vInfo("vgId:%d, start to commit, commit ID:%" PRId64 " version:%" PRId64 " term: %" PRId64, TD_VID(pVnode), + vInfo("vgId:%d, start to commit, commitId:%" PRId64 " version:%" PRId64 " term: %" PRId64, TD_VID(pVnode), pVnode->state.commitID, pVnode->state.applied, pVnode->state.applyTerm); // persist wal before starting @@ -271,16 +303,11 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) { return -1; } - // save info if (pVnode->pTfs) { snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); } else { snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); } - if (vnodeSaveInfo(dir, &pInfo->info) < 0) { - code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); - } // walBeginSnapshot(pVnode->pWal, pVnode->state.applied); syncBeginSnapshot(pVnode->sync, pVnode->state.applied); diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index fcfacd1ca9d6a369e0290863b86c96c925b4174c..2fc06fba86331882df27197bd9405cf72c98088c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -257,8 +257,8 @@ _exit: pReader->index++; *nData = sizeof(SSnapDataHdr) + pHdr->size; pHdr->index = pReader->index; - vInfo("vgId:%d, vnode snapshot read data,index:%" PRId64 " type:%d nData:%d ", TD_VID(pReader->pVnode), - pReader->index, pHdr->type, *nData); + vDebug("vgId:%d, vnode snapshot read data, index:%" PRId64 " type:%d blockLen:%d ", TD_VID(pReader->pVnode), + pReader->index, pHdr->type, *nData); } else { vInfo("vgId:%d, vnode snapshot read data end, index:%" PRId64, TD_VID(pReader->pVnode), pReader->index); } @@ -423,8 +423,8 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { ASSERT(pHdr->index == pWriter->index + 1); pWriter->index = pHdr->index; - vInfo("vgId:%d, vnode snapshot write data, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode), pHdr->index, - pHdr->type, nData); + vDebug("vgId:%d, vnode snapshot write data, index:%" PRId64 " type:%d blockLen:%d", TD_VID(pVnode), pHdr->index, + pHdr->type, nData); switch (pHdr->type) { case SNAP_DATA_CFG: { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index d719a349671c8ba0d32f12e00ff068b113e2add5..ec92316505739c7b25f79aff297af62cf89ffad3 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -77,52 +77,72 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { tDecoderClear(&dc); } break; case TDMT_VND_SUBMIT: { - SSubmitMsgIter msgIter = {0}; - SSubmitReq *pSubmitReq = (SSubmitReq *)pMsg->pCont; - SSubmitBlk *pBlock = NULL; - int64_t ctime = taosGetTimestampMs(); - tb_uid_t uid; - - if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { - code = terrno; + int64_t ctime = taosGetTimestampMs(); + + tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); + tStartDecode(&dc); + + uint64_t nSubmitTbData; + if (tDecodeU64v(&dc, &nSubmitTbData) < 0) { + code = TSDB_CODE_INVALID_MSG; goto _err; } - for (;;) { - tGetSubmitMsgNext(&msgIter, &pBlock); - if (pBlock == NULL) break; + for (int32_t i = 0; i < nSubmitTbData; i++) { + if (tStartDecode(&dc) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _err; + } - if (msgIter.schemaLen > 0) { - char *name = NULL; + int32_t flags; + if (tDecodeI32v(&dc, &flags) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _err; + } - tDecoderInit(&dc, pBlock->data, msgIter.schemaLen); + if (flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { + // SVCreateTbReq if (tStartDecode(&dc) < 0) { code = TSDB_CODE_INVALID_MSG; - return code; + goto _err; } if (tDecodeI32v(&dc, NULL) < 0) { code = TSDB_CODE_INVALID_MSG; - return code; + goto _err; } + + char *name = NULL; if (tDecodeCStr(&dc, &name) < 0) { code = TSDB_CODE_INVALID_MSG; - return code; + goto _err; } - uid = metaGetTableEntryUidByName(pVnode->pMeta, name); + int64_t uid = metaGetTableEntryUidByName(pVnode->pMeta, name); if (uid == 0) { uid = tGenIdPI64(); } + *(int64_t *)(dc.data + dc.pos) = uid; *(int64_t *)(dc.data + dc.pos + 8) = ctime; - pBlock->uid = htobe64(uid); tEndDecode(&dc); - tDecoderClear(&dc); + + // SSubmitTbData + int64_t suid; + if (tDecodeI64(&dc, &suid) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _err; + } + + *(int64_t *)(dc.data + dc.pos) = uid; } + + tEndDecode(&dc); } + tEndDecode(&dc); + tDecoderClear(&dc); } break; case TDMT_VND_DELETE: { int32_t size; @@ -197,7 +217,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp if (!syncUtilUserCommit(pMsg->msgType)) goto _exit; - if (pMsg->msgType == TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE) { + if (pMsg->msgType == TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE || pMsg->msgType == TDMT_STREAM_TASK_CHECK_RSP) { if (tqCheckLogInWal(pVnode->pTq, version)) return 0; } @@ -236,7 +256,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp break; /* TSDB */ case TDMT_VND_SUBMIT: - if (vnodeProcessSubmitReq(pVnode, version, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err; + if (vnodeProcessSubmitReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DELETE: if (vnodeProcessDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; @@ -855,57 +875,198 @@ static int32_t vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, return TSDB_CODE_SUCCESS; } -static int32_t vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char *tags) { - ASSERT(pMsg != NULL); - SSubmitMsgIter msgIter = {0}; - SMeta *pMeta = pVnode->pMeta; - SSubmitBlk *pBlock = NULL; +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +#if 1 + int32_t code = 0; + terrno = 0; - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; - while (true) { - if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; - if (pBlock == NULL) break; + SSubmitReq2 *pSubmitReq = &(SSubmitReq2){0}; + SSubmitRsp2 *pSubmitRsp = &(SSubmitRsp2){0}; + SArray *newTbUids = NULL; + int32_t ret; + SEncoder ec = {0}; + + pRsp->code = TSDB_CODE_SUCCESS; + + // decode + SDecoder dc = {0}; + tDecoderInit(&dc, pReq, len); + if (tDecodeSSubmitReq2(&dc, pSubmitReq) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + tDecoderClear(&dc); - vnodeDebugPrintSingleSubmitMsg(pMeta, pBlock, &msgIter, tags); + // check + code = tsdbScanAndConvertSubmitMsg(pVnode->pTsdb, pSubmitReq); + if (code) { + goto _exit; } - return 0; -} + for (int32_t i = 0; i < TARRAY_SIZE(pSubmitReq->aSubmitTbData); ++i) { + SSubmitTbData *pSubmitTbData = taosArrayGet(pSubmitReq->aSubmitTbData, i); -static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; - SSubmitRsp submitRsp = {0}; - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SVCreateTbReq createTbReq = {0}; - SDecoder decoder = {0}; - int32_t nRows = 0; - int32_t tsize, ret; - SEncoder encoder = {0}; - SArray *newTbUids = NULL; - SVStatis statis = {0}; - bool tbCreated = false; + if (pSubmitTbData->pCreateTbReq) { + pSubmitTbData->uid = pSubmitTbData->pCreateTbReq->uid; + } else { + SMetaInfo info = {0}; + + code = metaGetInfo(pVnode->pMeta, pSubmitTbData->uid, &info, NULL); + if (code) { + code = TSDB_CODE_TDB_TABLE_NOT_EXIST; + vWarn("vgId:%d, table uid:%" PRId64 " not exists", TD_VID(pVnode), pSubmitTbData->uid); + goto _exit; + } + + if (info.suid != pSubmitTbData->suid) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (info.suid) { + metaGetInfo(pVnode->pMeta, info.suid, &info, NULL); + } + + if (pSubmitTbData->sver != info.skmVer) { + code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; + goto _exit; + } + } + + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + int32_t nColData = TARRAY_SIZE(pSubmitTbData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pSubmitTbData->aCol); + + if (nColData <= 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (aColData[0].cid != PRIMARYKEY_TIMESTAMP_COL_ID || aColData[0].type != TSDB_DATA_TYPE_TIMESTAMP || + aColData[0].nVal <= 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + for (int32_t i = 1; i < nColData; i++) { + if (aColData[i].nVal != aColData[0].nVal) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + } + } + } + + vDebug("vgId:%d, submit block size %d", TD_VID(pVnode), (int32_t)taosArrayGetSize(pSubmitReq->aSubmitTbData)); + + // loop to handle + for (int32_t i = 0; i < TARRAY_SIZE(pSubmitReq->aSubmitTbData); ++i) { + SSubmitTbData *pSubmitTbData = taosArrayGet(pSubmitReq->aSubmitTbData, i); + + // create table + if (pSubmitTbData->pCreateTbReq) { + // check (TODO: move check to create table) + code = grantCheck(TSDB_GRANT_TIMESERIES); + if (code) goto _exit; + + code = grantCheck(TSDB_GRANT_TABLE); + if (code) goto _exit; + + // alloc if need + if (pSubmitRsp->aCreateTbRsp == NULL && + (pSubmitRsp->aCreateTbRsp = taosArrayInit(TARRAY_SIZE(pSubmitReq->aSubmitTbData), sizeof(SVCreateTbRsp))) == + NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + SVCreateTbRsp *pCreateTbRsp = taosArrayReserve(pSubmitRsp->aCreateTbRsp, 1); + + // create table + if (metaCreateTable(pVnode->pMeta, version, pSubmitTbData->pCreateTbReq, &pCreateTbRsp->pMeta) == + 0) { // create table success + + if (newTbUids == NULL && + (newTbUids = taosArrayInit(TARRAY_SIZE(pSubmitReq->aSubmitTbData), sizeof(int64_t))) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + taosArrayPush(newTbUids, &pSubmitTbData->uid); + + if (pCreateTbRsp->pMeta) { + vnodeUpdateMetaRsp(pVnode, pCreateTbRsp->pMeta); + } + } else { // create table failed + if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { + code = terrno; + goto _exit; + } + } + } + + // insert data + int32_t affectedRows; + code = tsdbInsertTableData(pVnode->pTsdb, version, pSubmitTbData, &affectedRows); + if (code) goto _exit; + + pSubmitRsp->affectedRows += affectedRows; + } + + // update table uid list + if (taosArrayGetSize(newTbUids) > 0) { + vDebug("vgId:%d, add %d table into query table list in handling submit", TD_VID(pVnode), + (int32_t)taosArrayGetSize(newTbUids)); + tqUpdateTbUidList(pVnode->pTq, newTbUids, true); + } + +_exit: + // message + pRsp->code = code; + tEncodeSize(tEncodeSSubmitRsp2, pSubmitRsp, pRsp->contLen, ret); + pRsp->pCont = rpcMallocCont(pRsp->contLen); + tEncoderInit(&ec, pRsp->pCont, pRsp->contLen); + tEncodeSSubmitRsp2(&ec, pSubmitRsp); + tEncoderClear(&ec); + + // update statistics + atomic_add_fetch_64(&pVnode->statis.nInsert, pSubmitRsp->affectedRows); + atomic_add_fetch_64(&pVnode->statis.nInsertSuccess, pSubmitRsp->affectedRows); + atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1); + if (code == 0) { + atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1); + tdProcessRSmaSubmit(pVnode->pSma, version, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT); + } + + // clear + taosArrayDestroy(newTbUids); + tDestroySSubmitReq2(pSubmitReq, TSDB_MSG_FLG_DECODE); + tDestroySSubmitRsp2(pSubmitRsp, TSDB_MSG_FLG_ENCODE); + + if (code) terrno = code; + + return code; + +#else + SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; + SSubmitRsp submitRsp = {0}; + int32_t nRows = 0; + int32_t tsize, ret; + SEncoder encoder = {0}; + SArray *newTbUids = NULL; + SVStatis statis = {0}; + bool tbCreated = false; terrno = TSDB_CODE_SUCCESS; pRsp->code = 0; pSubmitReq->version = version; statis.nBatchInsert = 1; -#ifdef TD_DEBUG_PRINT_ROW - vnodeDebugPrintSubmitMsg(pVnode, pReq, __func__); -#endif - if (tsdbScanAndConvertSubmitMsg(pVnode->pTsdb, pSubmitReq) < 0) { pRsp->code = terrno; goto _exit; } - // handle the request - if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { - pRsp->code = TSDB_CODE_INVALID_MSG; - goto _exit; - } - submitRsp.pArray = taosArrayInit(msgIter.numOfBlocks, sizeof(SSubmitBlkRsp)); newTbUids = taosArrayInit(msgIter.numOfBlocks, sizeof(int64_t)); if (!submitRsp.pArray || !newTbUids) { @@ -922,42 +1083,42 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq // create table for auto create table mode if (msgIter.schemaLen > 0) { - tDecoderInit(&decoder, pBlock->data, msgIter.schemaLen); - if (tDecodeSVCreateTbReq(&decoder, &createTbReq) < 0) { - pRsp->code = TSDB_CODE_INVALID_MSG; - tDecoderClear(&decoder); - taosArrayDestroy(createTbReq.ctb.tagName); - goto _exit; - } - - if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0) { - pRsp->code = terrno; - tDecoderClear(&decoder); - taosArrayDestroy(createTbReq.ctb.tagName); - goto _exit; - } - - if ((terrno = grantCheck(TSDB_GRANT_TABLE)) < 0) { - pRsp->code = terrno; - tDecoderClear(&decoder); - taosArrayDestroy(createTbReq.ctb.tagName); - goto _exit; - } + // tDecoderInit(&decoder, pBlock->data, msgIter.schemaLen); + // if (tDecodeSVCreateTbReq(&decoder, &createTbReq) < 0) { + // pRsp->code = TSDB_CODE_INVALID_MSG; + // tDecoderClear(&decoder); + // taosArrayDestroy(createTbReq.ctb.tagName); + // goto _exit; + // } + + // if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0) { + // pRsp->code = terrno; + // tDecoderClear(&decoder); + // taosArrayDestroy(createTbReq.ctb.tagName); + // goto _exit; + // } + + // if ((terrno = grantCheck(TSDB_GRANT_TABLE)) < 0) { + // pRsp->code = terrno; + // tDecoderClear(&decoder); + // taosArrayDestroy(createTbReq.ctb.tagName); + // goto _exit; + // } if (metaCreateTable(pVnode->pMeta, version, &createTbReq, &submitBlkRsp.pMeta) < 0) { - if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { - submitBlkRsp.code = terrno; - pRsp->code = terrno; - tDecoderClear(&decoder); - taosArrayDestroy(createTbReq.ctb.tagName); - goto _exit; - } + // if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { + // submitBlkRsp.code = terrno; + // pRsp->code = terrno; + // tDecoderClear(&decoder); + // taosArrayDestroy(createTbReq.ctb.tagName); + // goto _exit; + // } } else { if (NULL != submitBlkRsp.pMeta) { vnodeUpdateMetaRsp(pVnode, submitBlkRsp.pMeta); } - taosArrayPush(newTbUids, &createTbReq.uid); + // taosArrayPush(newTbUids, &createTbReq.uid); submitBlkRsp.uid = createTbReq.uid; submitBlkRsp.tblFName = taosMemoryMalloc(strlen(pVnode->config.dbname) + strlen(createTbReq.name) + 2); @@ -965,18 +1126,15 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq tbCreated = true; } - msgIter.uid = createTbReq.uid; - if (createTbReq.type == TSDB_CHILD_TABLE) { - msgIter.suid = createTbReq.ctb.suid; - } else { - msgIter.suid = 0; - } + // msgIter.uid = createTbReq.uid; + // if (createTbReq.type == TSDB_CHILD_TABLE) { + // msgIter.suid = createTbReq.ctb.suid; + // } else { + // msgIter.suid = 0; + // } -#ifdef TD_DEBUG_PRINT_ROW - vnodeDebugPrintSingleSubmitMsg(pVnode->pMeta, pBlock, &msgIter, "real uid"); -#endif - tDecoderClear(&decoder); - taosArrayDestroy(createTbReq.ctb.tagName); + // tDecoderClear(&decoder); + // taosArrayDestroy(createTbReq.ctb.tagName); } if (tsdbInsertTableData(pVnode->pTsdb, version, &msgIter, pBlock, &submitBlkRsp) < 0) { @@ -990,21 +1148,21 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq } } - if (taosArrayGetSize(newTbUids) > 0) { - vDebug("vgId:%d, add %d table into query table list in handling submit", TD_VID(pVnode), - (int32_t)taosArrayGetSize(newTbUids)); - } + // if (taosArrayGetSize(newTbUids) > 0) { + // vDebug("vgId:%d, add %d table into query table list in handling submit", TD_VID(pVnode), + // (int32_t)taosArrayGetSize(newTbUids)); + // } - tqUpdateTbUidList(pVnode->pTq, newTbUids, true); + // tqUpdateTbUidList(pVnode->pTq, newTbUids, true); _exit: taosArrayDestroy(newTbUids); - tEncodeSize(tEncodeSSubmitRsp, &submitRsp, tsize, ret); - pRsp->pCont = rpcMallocCont(tsize); - pRsp->contLen = tsize; - tEncoderInit(&encoder, pRsp->pCont, tsize); - tEncodeSSubmitRsp(&encoder, &submitRsp); - tEncoderClear(&encoder); + // tEncodeSize(tEncodeSSubmitRsp, &submitRsp, tsize, ret); + // pRsp->pCont = rpcMallocCont(tsize); + // pRsp->contLen = tsize; + // tEncoderInit(&encoder, pRsp->pCont, tsize); + // tEncodeSSubmitRsp(&encoder, &submitRsp); + // tEncoderClear(&encoder); taosArrayDestroyEx(submitRsp.pArray, tFreeSSubmitBlkRsp); @@ -1025,6 +1183,8 @@ _exit: vDebug("vgId:%d, submit success, index:%" PRId64, pVnode->config.vgId, version); return 0; +#endif + return 0; } static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 5caaae502f14f58b6cb464a4c503ce526fc1c8cb..5697487743d30c5c1f76da4907ffd30bf30d1c7f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -391,9 +391,9 @@ static int32_t vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsm const STraceId *trace = &pMsg->info.traceId; vGTrace("vgId:%d, commit-cb is excuted, fsm:%p, index:%" PRId64 ", term:%" PRIu64 ", msg-index:%" PRId64 - ", weak:%d, code:%d, state:%d %s, type:%s", + ", weak:%d, code:%d, state:%d %s, type:%s code:0x%x", pVnode->config.vgId, pFsm, pMeta->index, pMeta->term, pMsg->info.conn.applyIndex, pMeta->isWeak, pMeta->code, - pMeta->state, syncStr(pMeta->state), TMSG_INFO(pMsg->msgType)); + pMeta->state, syncStr(pMeta->state), TMSG_INFO(pMsg->msgType), pMsg->code); return tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, pMsg); } @@ -465,9 +465,9 @@ static int32_t vnodeSnapshotStopWrite(const SSyncFSM *pFsm, void *pWriter, bool static int32_t vnodeSnapshotDoWrite(const SSyncFSM *pFsm, void *pWriter, void *pBuf, int32_t len) { SVnode *pVnode = pFsm->data; - vDebug("vgId:%d, continue write vnode snapshot, len:%d", pVnode->config.vgId, len); + vDebug("vgId:%d, continue write vnode snapshot, blockLen:%d", pVnode->config.vgId, len); int32_t code = vnodeSnapWrite(pWriter, pBuf, len); - vDebug("vgId:%d, continue write vnode snapshot finished, len:%d", pVnode->config.vgId, len); + vDebug("vgId:%d, continue write vnode snapshot finished, blockLen:%d", pVnode->config.vgId, len); return code; } diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index a6ce613882daeda0ea99898e104cf184b4c5add6..9cbad139bfe957b81c0afcbd7ad9be48999b60b0 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -582,34 +582,34 @@ typedef struct SCtgOperation { #define CTG_LOCK(type, _lock) \ do { \ if (CTG_READ == (type)) { \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before read lock"); \ CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRLockLatch(_lock); \ CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) > 0); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value after read lock"); \ } else { \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before write lock"); \ CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWLockLatch(_lock); \ CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value after write lock"); \ } \ } while (0) #define CTG_UNLOCK(type, _lock) \ do { \ if (CTG_READ == (type)) { \ - assert(atomic_load_32((_lock)) > 0); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value before read unlock"); \ CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRUnLockLatch(_lock); \ CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after read unlock"); \ } else { \ - assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value before write unlock"); \ CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWUnLockLatch(_lock); \ CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after write unlock"); \ } \ } while (0) diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index acd18fcca5108335a11e731c72e26df80020b993..b8590c9255047b3c53ff90b82ffb3862ac8e2bdc 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -471,17 +471,20 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, con } int32_t ctgInitTask(SCtgJob* pJob, CTG_TASK_TYPE type, void* param, int32_t* taskId) { + int32_t code = 0; int32_t tid = atomic_fetch_add_32(&pJob->taskIdx, 1); CTG_LOCK(CTG_WRITE, &pJob->taskLock); - CTG_ERR_RET((*gCtgAsyncFps[type].initFp)(pJob, tid, param)); - CTG_UNLOCK(CTG_WRITE, &pJob->taskLock); + CTG_ERR_JRET((*gCtgAsyncFps[type].initFp)(pJob, tid, param)); if (taskId) { *taskId = tid; } - return TSDB_CODE_SUCCESS; +_return: + CTG_UNLOCK(CTG_WRITE, &pJob->taskLock); + + return code; } int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const SCatalogReq* pReq, catalogCallback fp, diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index fe83854a911a26729beaa0e6626ed44c3be4f2b4..c266cc1df9fc0ab5bd61574f519920df327630c9 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -2500,6 +2500,7 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe CTG_LOCK(CTG_READ, &pCache->metaLock); if (NULL == pCache->pMeta) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); ctgDebug("tb %s meta not in cache, dbFName:%s", pName->tname, dbFName); ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index b6ff1c8b987278ee8ef223577e91771c12cefde8..a6c0d1c40157f31f8002b11e7d4ce0e95a6f3f21 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -22,7 +22,6 @@ extern SCatalogMgmt gCtgMgmt; SCtgDebug gCTGDebug = {0}; void ctgdUserCallback(SMetaData *pResult, void *param, int32_t code) { - ASSERT(*(int32_t *)param == 1); taosMemoryFree(param); qDebug("async call result: %s", tstrerror(code)); diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 7cc6c90d303c4c425358800eb646f986417cefdd..3b037e206290ff767a1f879e0b46aadf8b3e4f20 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -40,7 +40,9 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu msgNum = taosArrayGetSize(batchRsp.pRsps); } - ASSERT(taskNum == msgNum || 0 == msgNum); + if (ASSERTS(taskNum == msgNum || 0 == msgNum, "taskNum %d mis-match msgNum %d", taskNum, msgNum)) { + msgNum = 0; + } ctgDebug("QID:0x%" PRIx64 " ctg got batch %d rsp %s", pJob->queryId, cbParam->batchId, TMSG_INFO(cbParam->reqType + 1)); @@ -58,11 +60,19 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu if (msgNum > 0) { pRsp = taosArrayGet(batchRsp.pRsps, i); - taskMsg.msgType = pRsp->reqType; - taskMsg.pData = pRsp->msg; - taskMsg.len = pRsp->msgLen; - - ASSERT(pRsp->msgIdx == *msgIdx); + if (ASSERTS(pRsp->msgIdx == *msgIdx, "rsp msgIdx %d mis-match msgIdx %d", pRsp->msgIdx, *msgIdx)) { + pRsp = &rsp; + pRsp->msgIdx = *msgIdx; + pRsp->reqType = -1; + pRsp->rspCode = 0; + taskMsg.msgType = -1; + taskMsg.pData = NULL; + taskMsg.len = 0; + } else { + taskMsg.msgType = pRsp->reqType; + taskMsg.pData = pRsp->msg; + taskMsg.len = pRsp->msgLen; + } } else { pRsp = &rsp; pRsp->msgIdx = *msgIdx; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index a179ec24f91cf7b5564862796633d4ad4f965f85..7967da2248b178daff8f87a3becde6508fcbf7ab 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -40,7 +40,6 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->numOfCols = htonl(numOfCols); int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols); - ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 253718048dcebb71b64ca4d4b48b3004c6d9ea3a..38222112a19a447a4a7294a6082bf8bd9edd18ce 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1666,7 +1666,6 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->numOfRows = htobe64((int64_t)rowNum); int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock)); - ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); rsp->compLen = htonl(len); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 647da78a78895c21e6dd10da278d95b92bf85051..44202b541288cccf6dc38246069a30bda655ef11 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -126,16 +126,19 @@ enum { typedef struct { // TODO remove prepareStatus - STqOffsetVal prepareStatus; // for tmq - STqOffsetVal lastStatus; // for tmq - SMqMetaRsp metaRsp; // for tmq fetching meta - int8_t returned; - int64_t snapshotVer; - const SSubmitReq* pReq; + STqOffsetVal prepareStatus; // for tmq + STqOffsetVal lastStatus; // for tmq + SMqMetaRsp metaRsp; // for tmq fetching meta + int8_t returned; + int64_t snapshotVer; + // const SSubmitReq* pReq; + + SPackedData submit; SSchemaWrapper* schema; char tbName[TSDB_TABLE_NAME_LEN]; int8_t recoverStep; + int8_t recoverScanFinished; SQueryTableDataCond tableCond; int64_t fillHistoryVer1; int64_t fillHistoryVer2; @@ -182,7 +185,7 @@ struct SExecTaskInfo { SSubplan* pSubplan; struct SOperatorInfo* pRoot; SLocalFetch localFetch; - SArray* pResultBlockList;// result block list + SArray* pResultBlockList; // result block list STaskStopInfo stopInfo; }; @@ -199,7 +202,7 @@ typedef struct SOperatorFpSet { __optr_fn_t getNextFn; __optr_fn_t cleanupFn; // call this function to release the allocated resources ASAP __optr_close_fn_t closeFn; - __optr_reqBuf_fn_t reqBufFn; // total used buffer for blocking operator + __optr_reqBuf_fn_t reqBufFn; // total used buffer for blocking operator __optr_encode_fn_t encodeResultRow; __optr_decode_fn_t decodeResultRow; __optr_explain_fn_t getExplainFn; @@ -255,22 +258,22 @@ typedef struct SLimitInfo { } SLimitInfo; typedef struct SExchangeInfo { - SArray* pSources; - SArray* pSourceDataInfo; - tsem_t ready; - void* pTransporter; + SArray* pSources; + SArray* pSourceDataInfo; + tsem_t ready; + void* pTransporter; // SArray, result block list, used to keep the multi-block that // passed by downstream operator - SArray* pResultBlockList; - SArray* pRecycledBlocks;// build a pool for small data block to avoid to repeatly create and then destroy. - SSDataBlock* pDummyBlock; // dummy block, not keep data - bool seqLoadData; // sequential load data or not, false by default - int32_t current; + SArray* pResultBlockList; + SArray* pRecycledBlocks; // build a pool for small data block to avoid to repeatly create and then destroy. + SSDataBlock* pDummyBlock; // dummy block, not keep data + bool seqLoadData; // sequential load data or not, false by default + int32_t current; SLoadRemoteDataInfo loadInfo; uint64_t self; SLimitInfo limitInfo; - int64_t openedTs; // start exec time stamp, todo: move to SLoadRemoteDataInfo + int64_t openedTs; // start exec time stamp, todo: move to SLoadRemoteDataInfo } SExchangeInfo; typedef struct SScanInfo { @@ -305,9 +308,9 @@ typedef struct { } SAggOptrPushDownInfo; typedef struct STableMetaCacheInfo { - SLRUCache* pTableMetaEntryCache; // 100 by default - uint64_t metaFetch; - uint64_t cacheHit; + SLRUCache* pTableMetaEntryCache; // 100 by default + uint64_t metaFetch; + uint64_t cacheHit; } STableMetaCacheInfo; typedef struct STableScanBase { @@ -325,46 +328,47 @@ typedef struct STableScanBase { } STableScanBase; typedef struct STableScanInfo { - STableScanBase base; - SScanInfo scanInfo; - int32_t scanTimes; - SSDataBlock* pResBlock; - SSampleExecInfo sample; // sample execution info - int32_t currentGroupId; - int32_t currentTable; - int8_t scanMode; - int8_t assignBlockUid; - bool hasGroupByTag; + STableScanBase base; + SScanInfo scanInfo; + int32_t scanTimes; + SSDataBlock* pResBlock; + SSampleExecInfo sample; // sample execution info + int32_t currentGroupId; + int32_t currentTable; + int8_t scanMode; + int8_t assignBlockUid; + bool hasGroupByTag; } STableScanInfo; typedef struct STableMergeScanInfo { - int32_t tableStartIndex; - int32_t tableEndIndex; - bool hasGroupId; - uint64_t groupId; - SArray* queryConds; // array of queryTableDataCond - STableScanBase base; - int32_t bufPageSize; - uint32_t sortBufSize; // max buffer size for in-memory sort - SArray* pSortInfo; - SSortHandle* pSortHandle; - SSDataBlock* pSortInputBlock; - int64_t startTs; // sort start time - SArray* sortSourceParams; - SLimitInfo limitInfo; - int64_t numOfRows; - SScanInfo scanInfo; - SSDataBlock* pResBlock; - SSampleExecInfo sample; // sample execution info - SSortExecInfo sortExecInfo; + int32_t tableStartIndex; + int32_t tableEndIndex; + bool hasGroupId; + uint64_t groupId; + SArray* queryConds; // array of queryTableDataCond + STableScanBase base; + int32_t bufPageSize; + uint32_t sortBufSize; // max buffer size for in-memory sort + SArray* pSortInfo; + SSortHandle* pSortHandle; + SSDataBlock* pSortInputBlock; + int64_t startTs; // sort start time + SArray* sortSourceParams; + SLimitInfo limitInfo; + int64_t numOfRows; + SScanInfo scanInfo; + int32_t scanTimes; + SSDataBlock* pResBlock; + SSampleExecInfo sample; // sample execution info + SSortExecInfo sortExecInfo; } STableMergeScanInfo; typedef struct STagScanInfo { - SColumnInfo* pCols; - SSDataBlock* pRes; - SColMatchInfo matchInfo; - int32_t curPos; - SReadHandle readHandle; + SColumnInfo* pCols; + SSDataBlock* pRes; + SColMatchInfo matchInfo; + int32_t curPos; + SReadHandle readHandle; } STagScanInfo; typedef enum EStreamScanMode { @@ -468,6 +472,11 @@ typedef struct SStreamScanInfo { SNodeList* pGroupTags; SNode* pTagCond; SNode* pTagIndexCond; + + // recover + int32_t blockRecoverContiCnt; + int32_t blockRecoverTotCnt; + } SStreamScanInfo; typedef struct { @@ -499,8 +508,8 @@ typedef struct STableCountScanOperatorInfo { STableCountScanSupp supp; - int32_t currGrpIdx; - SArray* stbUidList; // when group by db_name and/or stable_name + int32_t currGrpIdx; + SArray* stbUidList; // when group by db_name and/or stable_name } STableCountScanOperatorInfo; typedef struct SOptrBasicInfo { @@ -678,19 +687,19 @@ void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32 void destroyOperatorInfo(SOperatorInfo* pOperator); int32_t optrDefaultBufFn(SOperatorInfo* pOperator); -void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock); -void cleanupBasicInfo(SOptrBasicInfo* pInfo); +void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock); +void cleanupBasicInfo(SOptrBasicInfo* pInfo); int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr); void cleanupExprSupp(SExprSupp* pSup); -void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs); +void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs); int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, const char* pkey); void cleanupAggSup(SAggSupporter* pAggSup); -void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows); +void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows); void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf); @@ -803,10 +812,10 @@ void setInputDataBlock(SExprSupp* pExprSupp, SSDataBlock* pBlock, int32_t order, int32_t checkForQueryBuf(size_t numOfTables); -bool isTaskKilled(SExecTaskInfo* pTaskInfo); -void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode); -void doDestroyTask(SExecTaskInfo* pTaskInfo); -void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); +bool isTaskKilled(SExecTaskInfo* pTaskInfo); +void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode); +void doDestroyTask(SExecTaskInfo* pTaskInfo); +void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, char* sql, EOPTR_EXEC_MODEL model); @@ -828,8 +837,8 @@ bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup); bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup); void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp, void* pTbName); -uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); -void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock); +uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); +void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock); int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SExprSupp* pSup, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index eff7a5ef938379a1d6e6a078c302e3f91ca9f544..a8051ea7c3eb294e7beb9ce6b5a23be601e635ef 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -62,8 +62,8 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp pEntry->numOfCols = taosArrayGetSize(pInput->pData->pDataBlock); pEntry->dataLen = sizeof(SDeleterRes); - ASSERT(1 == pEntry->numOfRows); - ASSERT(3 == pEntry->numOfCols); +// ASSERT(1 == pEntry->numOfRows); +// ASSERT(3 == pEntry->numOfCols); pBuf->useSize = sizeof(SDataCacheEntry); @@ -167,7 +167,6 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE SDataDeleterBuf* pBuf = NULL; taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf); - ASSERT(NULL != pBuf); memcpy(&pDeleter->nextOutput, pBuf, sizeof(SDataDeleterBuf)); taosFreeQitem(pBuf); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index c2fa438c808d42854794a56695d73ae192834247..a603bffba5f37b79ee9f4a42546b8bbb3a0c32ea 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -77,8 +77,8 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn pBuf->useSize = sizeof(SDataCacheEntry); pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols); - ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); - ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); +// ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); +// ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); pBuf->useSize += pEntry->dataLen; @@ -162,15 +162,14 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE SDataDispatchBuf* pBuf = NULL; taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); - ASSERT(NULL != pBuf); memcpy(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf)); taosFreeQitem(pBuf); SDataCacheEntry* pEntry = (SDataCacheEntry*)pDispatcher->nextOutput.pData; *pLen = pEntry->dataLen; - ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); - ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); +// ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); +// ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); *pQueryEnd = pDispatcher->queryEnd; qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, @@ -193,8 +192,8 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->numOfCols = pEntry->numOfCols; pOutput->compressed = pEntry->compressed; - ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); - ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); +// ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); +// ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen); atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 346fcc9729d30e896a24dc63bc3f996e14606da3..ca6149d42c90b21a7b1359348220d9611351dbf6 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -27,7 +27,7 @@ extern SDataSinkStat gDataSinkStat; typedef struct SSubmitRes { int64_t affectedRows; int32_t code; - SSubmitRsp* pRsp; + SSubmitRsp2* pRsp; } SSubmitRes; typedef struct SDataInserterHandle { @@ -58,22 +58,25 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) { pInserter->submitRes.code = code; if (code == TSDB_CODE_SUCCESS) { - pInserter->submitRes.pRsp = taosMemoryCalloc(1, sizeof(SSubmitRsp)); + pInserter->submitRes.pRsp = taosMemoryCalloc(1, sizeof(SSubmitRsp2)); SDecoder coder = {0}; tDecoderInit(&coder, pMsg->pData, pMsg->len); - code = tDecodeSSubmitRsp(&coder, pInserter->submitRes.pRsp); + code = tDecodeSSubmitRsp2(&coder, pInserter->submitRes.pRsp); if (code) { - tFreeSSubmitRsp(pInserter->submitRes.pRsp); + taosMemoryFree(pInserter->submitRes.pRsp); pInserter->submitRes.code = code; goto _return; } - if (pInserter->submitRes.pRsp->nBlocks > 0) { - for (int32_t i = 0; i < pInserter->submitRes.pRsp->nBlocks; ++i) { - SSubmitBlkRsp* blk = pInserter->submitRes.pRsp->pBlocks + i; - if (TSDB_CODE_SUCCESS != blk->code) { - code = blk->code; - tFreeSSubmitRsp(pInserter->submitRes.pRsp); + if (pInserter->submitRes.pRsp->affectedRows > 0) { + SArray* pCreateTbList = pInserter->submitRes.pRsp->aCreateTbRsp; + int32_t numOfTables = taosArrayGetSize(pCreateTbList); + + for (int32_t i = 0; i < numOfTables; ++i) { + SVCreateTbRsp* pRsp = taosArrayGet(pCreateTbList, i); + if (TSDB_CODE_SUCCESS != pRsp->code) { + code = pRsp->code; + taosMemoryFree(pInserter->submitRes.pRsp); pInserter->submitRes.code = code; goto _return; } @@ -83,20 +86,17 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) { pInserter->submitRes.affectedRows += pInserter->submitRes.pRsp->affectedRows; qDebug("submit rsp received, affectedRows:%d, total:%"PRId64, pInserter->submitRes.pRsp->affectedRows, pInserter->submitRes.affectedRows); - - tFreeSSubmitRsp(pInserter->submitRes.pRsp); + tDecoderClear(&coder); + taosMemoryFree(pInserter->submitRes.pRsp); } _return: - tsem_post(&pInserter->ready); - taosMemoryFree(pMsg->pData); - return TSDB_CODE_SUCCESS; } -static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMsg, void* pTransporter, SEpSet* pEpset) { +static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, void* pMsg, int32_t msgLen, void* pTransporter, SEpSet* pEpset) { // send the fetch remote task result reques SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { @@ -111,7 +111,7 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMs pMsgSendInfo->param = pParam; pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->msgInfo.pData = pMsg; - pMsgSendInfo->msgInfo.len = ntohl(pMsg->length); + pMsgSendInfo->msgInfo.len = msgLen; pMsgSendInfo->msgType = TDMT_VND_SUBMIT; pMsgSendInfo->fp = inserterCallback; @@ -119,140 +119,233 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMs return asyncSendMsgToServer(pTransporter, pEpset, &transporterId, pMsgSendInfo); } -int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) { - const SArray* pBlocks = pInserter->pDataBlocks; - const STSchema* pTSchema = pInserter->pSchema; - int64_t uid = pInserter->pNode->tableId; - int64_t suid = pInserter->pNode->stableId; - int32_t vgId = pInserter->pNode->vgId; - bool fullCol = (pInserter->pNode->pCols->length == pTSchema->numOfCols); +static int32_t submitReqToMsg(int32_t vgId, SSubmitReq2* pReq, void** pData, int32_t* pLen) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t len = 0; + void* pBuf = NULL; + tEncodeSize(tEncodeSSubmitReq2, pReq, len, code); + if (TSDB_CODE_SUCCESS == code) { + SEncoder encoder; + len += sizeof(SMsgHead); + pBuf = taosMemoryMalloc(len); + if (NULL == pBuf) { + return TSDB_CODE_OUT_OF_MEMORY; + } + ((SMsgHead*)pBuf)->vgId = htonl(vgId); + ((SMsgHead*)pBuf)->contLen = htonl(len); + tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SMsgHead)), len - sizeof(SMsgHead)); + code = tEncodeSSubmitReq2(&encoder, pReq); + tEncoderClear(&encoder); + } - SSubmitReq* ret = NULL; - int32_t sz = taosArrayGetSize(pBlocks); + if (TSDB_CODE_SUCCESS == code) { + *pData = pBuf; + *pLen = len; + } else { + taosMemoryFree(pBuf); + } + return code; +} - // cal size - int32_t cap = sizeof(SSubmitReq); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i); - int32_t rows = pDataBlock->info.rows; - // TODO min - int32_t rowSize = pDataBlock->info.rowSize; - int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); - cap += sizeof(SSubmitBlk) + rows * maxLen; +int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** ppReq, const SSDataBlock* pDataBlock, const STSchema* pTSchema, + int64_t uid, int32_t vgId, tb_uid_t suid) { + SSubmitReq2* pReq = *ppReq; + SArray* pVals = NULL; + int32_t numOfBlks = 0; + bool fullCol = (pInserter->pNode->pCols->length == pTSchema->numOfCols); + + terrno = TSDB_CODE_SUCCESS; + + if (NULL == pReq) { + if (!(pReq = taosMemoryMalloc(sizeof(SSubmitReq2)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } } - // assign data - // TODO - ret = taosMemoryCalloc(1, cap); - ret->header.vgId = htonl(vgId); - ret->version = htonl(pTSchema->version); - ret->length = sizeof(SSubmitReq); - ret->numOfBlocks = htonl(sz); + int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); + int32_t rows = pDataBlock->info.rows; - SSubmitBlk* blkHead = POINTER_SHIFT(ret, sizeof(SSubmitReq)); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i); + SSubmitTbData tbData = {0}; + if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow*)))) { + goto _end; + } + tbData.suid = suid; + tbData.uid = uid; + tbData.sver = pTSchema->version; - blkHead->sversion = htonl(pTSchema->version); - // TODO - blkHead->suid = htobe64(suid); - blkHead->uid = htobe64(uid); - blkHead->schemaLen = htonl(0); - - int32_t rows = 0; - int32_t dataLen = 0; - STSRow* rowData = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); - int64_t lastTs = TSKEY_MIN; - bool ignoreRow = false; - for (int32_t j = 0; j < pDataBlock->info.rows; j++) { - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTSchema->version); - tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); - tdSRowResetBuf(&rb, rowData); + if (!pVals && !(pVals = taosArrayInit(colNum, sizeof(SColVal)))) { + taosArrayDestroy(tbData.aRowP); + goto _end; + } - ignoreRow = false; - for (int32_t k = 0; k < pTSchema->numOfCols; k++) { - const STColumn* pColumn = &pTSchema->columns[k]; - SColumnInfoData* pColData = NULL; - int16_t colIdx = k; - if (!fullCol) { - int16_t* slotId = taosHashGet(pInserter->pCols, &pColumn->colId, sizeof(pColumn->colId)); - if (NULL == slotId) { - continue; - } + int64_t lastTs = TSKEY_MIN; + bool ignoreRow = false; + bool disorderTs = false; - colIdx = *slotId; - } + for (int32_t j = 0; j < rows; ++j) { // iterate by row + taosArrayClear(pVals); - pColData = taosArrayGet(pDataBlock->pDataBlock, colIdx); - if (pColData->info.type != pColumn->type) { - qError("col type mis-match, schema type:%d, type in block:%d", pColumn->type, pColData->info.type); - terrno = TSDB_CODE_APP_ERROR; - return TSDB_CODE_APP_ERROR; + int32_t offset = 0; + for (int32_t k = 0; k < pTSchema->numOfCols; ++k) { // iterate by column + int16_t colIdx = k; + const STColumn* pCol = &pTSchema->columns[k]; + if (!fullCol) { + int16_t* slotId = taosHashGet(pInserter->pCols, &pCol->colId, sizeof(pCol->colId)); + if (NULL == slotId) { + continue; } - if (colDataIsNull_s(pColData, j)) { - if (0 == k && TSDB_DATA_TYPE_TIMESTAMP == pColumn->type) { - ignoreRow = true; - break; + colIdx = *slotId; + } + + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, colIdx); + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY + ASSERT(pColInfoData->info.type == pCol->type); + if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + void* data = colDataGetVarData(pColInfoData, j); + SValue sv = (SValue){.nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); } - - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, pColumn->offset, k); - } else { - void* data = colDataGetData(pColData, j); - if (0 == k && TSDB_DATA_TYPE_TIMESTAMP == pColumn->type) { - if (*(int64_t*)data == lastTs) { - ignoreRow = true; - break; + break; + } + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_MEDIUMBLOB: + uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); + ASSERT(0); + break; + default: + if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { + if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); // should use pCol->type + taosArrayPush(pVals, &cv); } else { - lastTs = *(int64_t*)data; + if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId) { + if (*(int64_t*)var == lastTs) { + ignoreRow = true; + } else if (*(int64_t*)var < lastTs) { + disorderTs = true; + } else { + lastTs = *(int64_t*)var; + } + } + + SValue sv; + memcpy(&sv.val, var, tDataTypes[pCol->type].bytes); + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); } + } else { + uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); + ASSERT(0); } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); - } - } - if (!fullCol) { - rb.hasNone = true; + break; } - tdSRowEnd(&rb); if (ignoreRow) { - continue; + break; } - - rows++; - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - dataLen += rowLen; } - blkHead->dataLen = htonl(dataLen); - blkHead->numOfRows = htonl(rows); + if (ignoreRow) { + ignoreRow = false; + continue; + } + + SRow* pRow = NULL; + if ((terrno = tRowBuild(pVals, pTSchema, &pRow)) < 0) { + tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + goto _end; + } + taosArrayPush(tbData.aRowP, &pRow); + } - ret->length += sizeof(SSubmitBlk) + dataLen; - blkHead = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk) + dataLen); + if (disorderTs) { + tRowSort(tbData.aRowP); + if ((terrno = tRowMerge(tbData.aRowP, (STSchema*)pTSchema, 0)) != 0) { + goto _end; + } } - ret->length = htonl(ret->length); + taosArrayPush(pReq->aSubmitTbData, &tbData); - *pReq = ret; +_end: + taosArrayDestroy(pVals); + if (terrno != 0) { + *ppReq = NULL; + if (pReq) { + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); + } + return TSDB_CODE_FAILED; + } + *ppReq = pReq; return TSDB_CODE_SUCCESS; } + +int32_t dataBlocksToSubmitReq(SDataInserterHandle* pInserter, void** pMsg, int32_t* msgLen) { + const SArray* pBlocks = pInserter->pDataBlocks; + const STSchema* pTSchema = pInserter->pSchema; + int64_t uid = pInserter->pNode->tableId; + int64_t suid = pInserter->pNode->stableId; + int32_t vgId = pInserter->pNode->vgId; + int32_t sz = taosArrayGetSize(pBlocks); + int32_t code = 0; + SSubmitReq2 *pReq = NULL; + + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i); + + code = buildSubmitReqFromBlock(pInserter, &pReq, pDataBlock, pTSchema, uid, vgId, suid); + if (code) { + if (pReq) { + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); + } + + return code; + } + } + + code = submitReqToMsg(vgId, pReq, pMsg, msgLen); + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); + + return code; +} + static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle; taosArrayPush(pInserter->pDataBlocks, &pInput->pData); - SSubmitReq* pMsg = NULL; - int32_t code = dataBlockToSubmit(pInserter, &pMsg); + void* pMsg = NULL; + int32_t msgLen = 0; + int32_t code = dataBlocksToSubmitReq(pInserter, &pMsg, &msgLen); if (code) { return code; } taosArrayClear(pInserter->pDataBlocks); - code = sendSubmitRequest(pInserter, pMsg, pInserter->pParam->readHandle->pMsgCb->clientRpc, &pInserter->pNode->epSet); + code = sendSubmitRequest(pInserter, pMsg, msgLen, pInserter->pParam->readHandle->pMsgCb->clientRpc, &pInserter->pNode->epSet); if (code) { return code; } diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 4103ca82dcc7b122f925207717ea15698bf7b904..9873c520060d1a645255ef05b1542c39143c2623 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -373,7 +373,6 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pRsp->useconds = htobe64(pRsp->useconds); pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); - ASSERT(pRsp != NULL); qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%" PRId64 ", %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, pRsp->numOfRows, pExchangeInfo); } else { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 75de0129470963c34edb9922f5d8e3c6cd9fc40d..0a2802ba375cd3a3efa186ad3bc3ba852a83164b 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -51,8 +51,8 @@ static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOf if (type == STREAM_INPUT__MERGED_SUBMIT) { for (int32_t i = 0; i < numOfBlocks; i++) { - SSubmitReq* pReq = *(void**)POINTER_SHIFT(input, i * sizeof(void*)); - taosArrayPush(pInfo->pBlockLists, &pReq); + SPackedData* pReq = POINTER_SHIFT(input, i * sizeof(SPackedData)); + taosArrayPush(pInfo->pBlockLists, pReq); } pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; } else if (type == STREAM_INPUT__DATA_SUBMIT) { @@ -61,7 +61,10 @@ static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOf } else if (type == STREAM_INPUT__DATA_BLOCK) { for (int32_t i = 0; i < numOfBlocks; ++i) { SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i]; - taosArrayPush(pInfo->pBlockLists, &pDataBlock); + SPackedData tmp = { + .pDataBlock = pDataBlock, + }; + taosArrayPush(pInfo->pBlockLists, &tmp); } pInfo->blockType = STREAM_INPUT__DATA_BLOCK; } @@ -115,18 +118,21 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu if (type == STREAM_INPUT__MERGED_SUBMIT) { // ASSERT(numOfBlocks > 1); for (int32_t i = 0; i < numOfBlocks; i++) { - SSubmitReq* pReq = *(void**)POINTER_SHIFT(input, i * sizeof(void*)); - taosArrayPush(pInfo->pBlockLists, &pReq); + SPackedData* pReq = POINTER_SHIFT(input, i * sizeof(SPackedData)); + taosArrayPush(pInfo->pBlockLists, pReq); } pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; } else if (type == STREAM_INPUT__DATA_SUBMIT) { ASSERT(numOfBlocks == 1); - taosArrayPush(pInfo->pBlockLists, &input); + taosArrayPush(pInfo->pBlockLists, input); pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; } else if (type == STREAM_INPUT__DATA_BLOCK) { for (int32_t i = 0; i < numOfBlocks; ++i) { SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i]; - taosArrayPush(pInfo->pBlockLists, &pDataBlock); + SPackedData tmp = { + .pDataBlock = pDataBlock, + }; + taosArrayPush(pInfo->pBlockLists, &tmp); } pInfo->blockType = STREAM_INPUT__DATA_BLOCK; } else { @@ -936,6 +942,10 @@ int32_t qStreamRestoreParam(qTaskInfo_t tinfo) { } return 0; } +bool qStreamRecoverScanFinished(qTaskInfo_t tinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + return pTaskInfo->streamInfo.recoverScanFinished; +} void* qExtractReaderFromStreamScanner(void* scanner) { SStreamScanInfo* pInfo = scanner; @@ -1002,11 +1012,22 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s return TSDB_CODE_SUCCESS; } -int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq) { +#if 0 +int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq, int64_t scanVer) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); ASSERT(pTaskInfo->streamInfo.pReq == NULL); pTaskInfo->streamInfo.pReq = pReq; + pTaskInfo->streamInfo.scanVer = scanVer; + return 0; +} +#endif + +int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); + ASSERT(pTaskInfo->streamInfo.submit.msgStr == NULL); + pTaskInfo->streamInfo.submit = submit; return 0; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index fecdcd8fa322c9636e12830f58188e5dd74aa98a..149efef884b10149bf005407e150380f9f1f0979 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -104,8 +104,6 @@ static int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, void setOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; - ASSERT(pOperator->pTaskInfo != NULL); - pOperator->cost.totalCost = (taosGetTimestampUs() - pOperator->pTaskInfo->cost.start) / 1000.0; setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); } @@ -524,7 +522,7 @@ bool functionNeedToExecute(SqlFunctionCtx* pCtx) { return true; } -static int32_t doCreateConstantValColumnAggInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, int32_t type, +static int32_t doCreateConstantValColumnSMAInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, int32_t type, int32_t paramIndex, int32_t numOfRows) { if (pInput->pData[paramIndex] == NULL) { pInput->pData[paramIndex] = taosMemoryCalloc(1, sizeof(SColumnInfoData)); @@ -548,8 +546,6 @@ static int32_t doCreateConstantValColumnAggInfo(SInputColumnInfoData* pInput, SF da = pInput->pColumnDataAgg[paramIndex]; } - ASSERT(!IS_VAR_DATA_TYPE(type)); - if (type == TSDB_DATA_TYPE_BIGINT) { int64_t v = pFuncParam->param.i; *da = (SColumnDataAgg){.numOfNull = 0, .min = v, .max = v, .sum = v * numOfRows}; @@ -570,7 +566,7 @@ static int32_t doCreateConstantValColumnAggInfo(SInputColumnInfoData* pInput, SF } else if (type == TSDB_DATA_TYPE_TIMESTAMP) { // do nothing } else { - ASSERT(0); + qError("invalid constant type for sma info"); } return TSDB_CODE_SUCCESS; @@ -600,7 +596,7 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB // the data in the corresponding SColumnInfoData will not be used. pInput->pData[j] = taosArrayGet(pBlock->pDataBlock, slotId); } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { - doCreateConstantValColumnAggInfo(pInput, pFuncParam, pFuncParam->param.nType, j, pBlock->info.rows); + doCreateConstantValColumnSMAInfo(pInput, pFuncParam, pFuncParam->param.nType, j, pBlock->info.rows); } } } else { @@ -1577,8 +1573,7 @@ void destroyOperatorInfo(SOperatorInfo* pOperator) { // each operator should be set their own function to return total cost buffer int32_t optrDefaultBufFn(SOperatorInfo* pOperator) { if (pOperator->blocking) { - ASSERT(0); - return 0; + return -1; } else { return 0; } @@ -2205,7 +2200,6 @@ static int32_t extractTbscanInStreamOpTree(SOperatorInfo* pOperator, STableScanI return extractTbscanInStreamOpTree(pOperator->pDownstream[0], ppInfo); } else { SStreamScanInfo* pInfo = pOperator->info; - ASSERT(pInfo->pTableScanOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); *ppInfo = pInfo->pTableScanOp->info; return 0; } @@ -2217,13 +2211,11 @@ int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) { *ppNode = (STableScanPhysiNode*)pNode; return 0; } else { - ASSERT(0); terrno = TSDB_CODE_APP_ERROR; return -1; } } else { if (LIST_LENGTH(pNode->pChildren) != 1) { - ASSERT(0); terrno = TSDB_CODE_APP_ERROR; return -1; } @@ -2233,32 +2225,6 @@ int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) { return -1; } -#if 0 -int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHandle, int64_t uid, int64_t ts) { - STableScanInfo* pTableScanInfo = NULL; - if (extractTbscanInStreamOpTree(pOperator, &pTableScanInfo) < 0) { - return -1; - } - - STableScanPhysiNode* pNode = NULL; - if (extractTableScanNode(plan->pNode, &pNode) < 0) { - ASSERT(0); - } - - tsdbReaderClose(pTableScanInfo->dataReader); - - STableListInfo info = {0}; - pTableScanInfo->dataReader = doCreateDataReader(pNode, pHandle, &info, NULL); - if (pTableScanInfo->dataReader == NULL) { - ASSERT(0); - qError("failed to create data reader"); - return TSDB_CODE_APP_ERROR; - } - // TODO: set uid and ts to data reader - return 0; -} -#endif - int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle) { SExecTaskInfo* pTask = *(SExecTaskInfo**)pTaskInfo; diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 8a097a23ceeee705907cb8460a5b34f43985effa..88ed9eccb323561718a3a7cc1bc8dfd737bef53c 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -42,38 +42,40 @@ typedef struct SJoinOperatorInfo { static void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode); static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator); static void destroyMergeJoinOperator(void* param); -static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstream, int32_t numOfDownstream, - SSortMergeJoinPhysiNode* pJoinNode); +static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstream, int32_t num, + SSortMergeJoinPhysiNode* pJoinNode, const char* idStr); -static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstream, int32_t numOfDownstream, - SSortMergeJoinPhysiNode* pJoinNode) { +static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstream, int32_t num, + SSortMergeJoinPhysiNode* pJoinNode, const char* idStr) { SNode* pMergeCondition = pJoinNode->pMergeCondition; - if (nodeType(pMergeCondition) == QUERY_NODE_OPERATOR) { - SOperatorNode* pNode = (SOperatorNode*)pMergeCondition; - SColumnNode* col1 = (SColumnNode*)pNode->pLeft; - SColumnNode* col2 = (SColumnNode*)pNode->pRight; - SColumnNode* leftTsCol = NULL; - SColumnNode* rightTsCol = NULL; - if (col1->dataBlockId == col2->dataBlockId ) { + if (nodeType(pMergeCondition) != QUERY_NODE_OPERATOR) { + qError("not support this in join operator, %s", idStr); + return; // do not handle this + } + + SOperatorNode* pNode = (SOperatorNode*)pMergeCondition; + SColumnNode* col1 = (SColumnNode*)pNode->pLeft; + SColumnNode* col2 = (SColumnNode*)pNode->pRight; + SColumnNode* leftTsCol = NULL; + SColumnNode* rightTsCol = NULL; + if (col1->dataBlockId == col2->dataBlockId) { + leftTsCol = col1; + rightTsCol = col2; + } else { + if (col1->dataBlockId == pDownstream[0]->resultDataBlockId) { + ASSERT(col2->dataBlockId == pDownstream[1]->resultDataBlockId); leftTsCol = col1; rightTsCol = col2; } else { - if (col1->dataBlockId == pDownstream[0]->resultDataBlockId) { - ASSERT(col2->dataBlockId == pDownstream[1]->resultDataBlockId); - leftTsCol = col1; - rightTsCol = col2; - } else { - ASSERT(col1->dataBlockId == pDownstream[1]->resultDataBlockId); - ASSERT(col2->dataBlockId == pDownstream[0]->resultDataBlockId); - leftTsCol = col2; - rightTsCol = col1; - } + ASSERT(col1->dataBlockId == pDownstream[1]->resultDataBlockId); + ASSERT(col2->dataBlockId == pDownstream[0]->resultDataBlockId); + leftTsCol = col2; + rightTsCol = col1; } - setJoinColumnInfo(&pInfo->leftCol, leftTsCol); - setJoinColumnInfo(&pInfo->rightCol, rightTsCol); - } else { - ASSERT(false); - }} + } + setJoinColumnInfo(&pInfo->leftCol, leftTsCol); + setJoinColumnInfo(&pInfo->rightCol, rightTsCol); +} SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) { @@ -97,7 +99,7 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->exprSupp.numOfExprs = numOfCols; - extractTimeCondition(pInfo, pDownstream, numOfDownstream, pJoinNode); + extractTimeCondition(pInfo, pDownstream, numOfDownstream, pJoinNode, GET_TASKID(pTaskInfo)); if (pJoinNode->pOnConditions != NULL && pJoinNode->node.pConditions != NULL) { pInfo->pCondAfterMerge = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); @@ -364,8 +366,6 @@ static bool mergeJoinGetNextTimestamp(SOperatorInfo* pOperator, int64_t* pLeftTs char* pRightVal = colDataGetData(pRightCol, pJoinInfo->rightPos); *pRightTs = *(int64_t*)pRightVal; - ASSERT(pLeftCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(pRightCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); return true; } diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index de353b4bac837ecc544c179f40acede17741cc38..0cccc75ef56a96e590e09d478d908106e8cc5492 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -221,7 +221,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { blockDataCleanup(pFinalRes); SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - if (pTaskInfo->streamInfo.pReq) { + if (pTaskInfo->streamInfo.submit.msgStr) { pOperator->status = OP_OPENED; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 1414d3b4ab310b427491f51f982c60f4ddb21107..b2aa2269a2fc5f23ed8e7fa86b2fd355bdf5cead 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -232,30 +232,6 @@ static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, if (!allColumnsHaveAgg) { return false; } - -#if 0 - // if (allColumnsHaveAgg == true) { - int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - - // todo create this buffer during creating operator - if (pBlock->pBlockAgg == NULL) { - pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, POINTER_BYTES); - if (pBlock->pBlockAgg == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); - } - } - - size_t num = taosArrayGetSize(pTableScanInfo->matchInfo.pList); - for (int32_t i = 0; i < num; ++i) { - SColMatchItem* pColMatchInfo = taosArrayGet(pTableScanInfo->matchInfo.pList, i); - if (!pColMatchInfo->needOutput) { - continue; - } - - pBlock->pBlockAgg[pColMatchInfo->dstSlotId] = pColAgg[i]; - } -#endif - return true; } @@ -1560,14 +1536,18 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { qDebug("queue scan called"); - if (pTaskInfo->streamInfo.pReq != NULL) { - if (pInfo->tqReader->pMsg == NULL) { - pInfo->tqReader->pMsg = pTaskInfo->streamInfo.pReq; - const SSubmitReq* pSubmit = pInfo->tqReader->pMsg; - if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) { - qError("submit msg messed up when initing stream submit block %p", pSubmit); - pInfo->tqReader->pMsg = NULL; - pTaskInfo->streamInfo.pReq = NULL; + if (pTaskInfo->streamInfo.submit.msgStr != NULL) { + if (pInfo->tqReader->msg2.msgStr == NULL) { + /*pInfo->tqReader->pMsg = pTaskInfo->streamInfo.pReq;*/ + + /*const SSubmitReq* pSubmit = pInfo->tqReader->pMsg;*/ + /*if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) {*/ + /*void* msgStr = pTaskInfo->streamInfo.*/ + SPackedData submit = pTaskInfo->streamInfo.submit; + if (tqReaderSetSubmitReq2(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) { + qError("submit msg messed up when initing stream submit block %p", submit.msgStr); + pInfo->tqReader->msg2 = (SPackedData){0}; + pInfo->tqReader->setMsg = 0; ASSERT(0); } } @@ -1575,10 +1555,10 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { blockDataCleanup(pInfo->pRes); SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; - while (tqNextDataBlock(pInfo->tqReader)) { + while (tqNextDataBlock2(pInfo->tqReader)) { SSDataBlock block = {0}; - int32_t code = tqRetrieveDataBlock(&block, pInfo->tqReader); + int32_t code = tqRetrieveDataBlock2(&block, pInfo->tqReader); if (code != TSDB_CODE_SUCCESS || block.info.rows == 0) { continue; @@ -1591,8 +1571,9 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { } } - pInfo->tqReader->pMsg = NULL; - pTaskInfo->streamInfo.pReq = NULL; + pInfo->tqReader->msg2 = (SPackedData){0}; + pInfo->tqReader->setMsg = 0; + pTaskInfo->streamInfo.submit = (SPackedData){0}; return NULL; } @@ -1785,11 +1766,18 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pTSInfo->scanTimes = 0; pTSInfo->currentGroupId = -1; pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN; + pTaskInfo->streamInfo.recoverScanFinished = false; } if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__SCAN) { + if (pInfo->blockRecoverContiCnt > 100) { + pInfo->blockRecoverTotCnt += pInfo->blockRecoverContiCnt; + pInfo->blockRecoverContiCnt = 0; + return NULL; + } SSDataBlock* pBlock = doTableScan(pInfo->pTableScanOp); if (pBlock != NULL) { + pInfo->blockRecoverContiCnt++; calBlockTbName(pInfo, pBlock); if (pInfo->pUpdateInfo) { TSKEY maxTs = updateInfoFillBlockData(pInfo->pUpdateInfo, pBlock, pInfo->primaryTsIndex); @@ -1807,6 +1795,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pTSInfo->base.cond.startVersion = -1; pTSInfo->base.cond.endVersion = -1; + pTaskInfo->streamInfo.recoverScanFinished = true; return NULL; } @@ -1821,7 +1810,8 @@ FETCH_NEXT_BLOCK: } int32_t current = pInfo->validBlockIndex++; - SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current); + SPackedData* pPacked = taosArrayGet(pInfo->pBlockLists, current); + SSDataBlock* pBlock = pPacked->pDataBlock; if (pBlock->info.id.groupId && pBlock->info.parTbName[0]) { streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, pBlock->info.parTbName); } @@ -1951,7 +1941,7 @@ FETCH_NEXT_BLOCK: NEXT_SUBMIT_BLK: while (1) { - if (pInfo->tqReader->pMsg == NULL) { + if (pInfo->tqReader->msg2.msgStr == NULL) { if (pInfo->validBlockIndex >= totBlockNum) { updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); doClearBufferedBlocks(pInfo); @@ -1959,22 +1949,22 @@ FETCH_NEXT_BLOCK: return NULL; } - int32_t current = pInfo->validBlockIndex++; - SSubmitReq* pSubmit = taosArrayGetP(pInfo->pBlockLists, current); - if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) { + int32_t current = pInfo->validBlockIndex++; + SPackedData* pSubmit = taosArrayGet(pInfo->pBlockLists, current); + /*if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) {*/ + if (tqReaderSetSubmitReq2(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) { qError("submit msg messed up when initing stream submit block %p, current %d, total %d", pSubmit, current, totBlockNum); - pInfo->tqReader->pMsg = NULL; continue; } } blockDataCleanup(pInfo->pRes); - while (tqNextDataBlock(pInfo->tqReader)) { + while (tqNextDataBlock2(pInfo->tqReader)) { SSDataBlock block = {0}; - int32_t code = tqRetrieveDataBlock(&block, pInfo->tqReader); + int32_t code = tqRetrieveDataBlock2(&block, pInfo->tqReader); if (code != TSDB_CODE_SUCCESS || block.info.rows == 0) { continue; @@ -2016,7 +2006,6 @@ FETCH_NEXT_BLOCK: if (pBlockInfo->rows > 0 || pInfo->pUpdateDataRes->info.rows > 0) { break; } else { - pInfo->tqReader->pMsg = NULL; continue; } /*blockDataCleanup(pInfo->pRes);*/ @@ -2238,6 +2227,7 @@ static void destroyStreamScanOperatorInfo(void* param) { SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, SExecTaskInfo* pTaskInfo) { + SArray* pColIds = NULL; SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -2260,7 +2250,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys } int32_t numOfOutput = taosArrayGetSize(pInfo->matchInfo.pList); - SArray* pColIds = taosArrayInit(numOfOutput, sizeof(int16_t)); + pColIds = taosArrayInit(numOfOutput, sizeof(int16_t)); for (int32_t i = 0; i < numOfOutput; ++i) { SColMatchItem* id = taosArrayGet(pInfo->matchInfo.pList, i); @@ -2297,7 +2287,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys } } - pInfo->pBlockLists = taosArrayInit(4, POINTER_BYTES); + pInfo->pBlockLists = taosArrayInit(4, sizeof(SPackedData)); if (pInfo->pBlockLists == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _error; @@ -2357,6 +2347,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys memcpy(&pTaskInfo->streamInfo.tableCond, &pTSInfo->base.cond, sizeof(SQueryTableDataCond)); } else { taosArrayDestroy(pColIds); + pColIds = NULL; } // create the pseduo columns info diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 7ac007b7cb914f5b4fe3faa8110d3acd0b72a852..f5dc6cc623ca800cb459923cf69b495e70447360 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -46,13 +46,15 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pOperator->pTaskInfo = pTaskInfo; SDataBlockDescNode* pDescNode = pSortNode->node.pOutputDataBlockDesc; - int32_t numOfCols = 0; - SSDataBlock* pResBlock = createDataBlockFromDescNode(pDescNode); - SExprInfo* pExprInfo = createExprInfo(pSortNode->pExprs, NULL, &numOfCols); + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pSortNode->pExprs, NULL, &numOfCols); int32_t numOfOutputCols = 0; int32_t code = extractColMatchInfo(pSortNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset); initResultSizeInfo(&pOperator->resultInfo, 1024); @@ -61,7 +63,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* goto _error; } - pInfo->binfo.pRes = pResBlock; + pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode); pInfo->pSortInfo = createSortInfo(pSortNode->pSortKeys); initLimitInfo(pSortNode->node.pLimit, pSortNode->node.pSlimit, &pInfo->limitInfo); @@ -86,7 +88,10 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* _error: pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(pInfo); + if (pInfo != NULL) { + destroySortOperatorInfo(pInfo); + } + taosMemoryFree(pOperator); return NULL; } @@ -139,7 +144,6 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i int32_t numOfCols = taosArrayGetSize(pColMatchInfo); for (int32_t i = 0; i < numOfCols; ++i) { SColMatchItem* pmInfo = taosArrayGet(pColMatchInfo, i); - // ASSERT(pmInfo->matchType == COL_MATCH_FROM_SLOT_ID); SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->dstSlotId); @@ -272,7 +276,6 @@ void destroySortOperatorInfo(void* param) { } int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { - ASSERT(pOptr != NULL); SSortExecInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo)); SSortOperatorInfo* pOperatorInfo = (SSortOperatorInfo*)pOptr->info; @@ -329,7 +332,6 @@ SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo int32_t numOfCols = taosArrayGetSize(pColMatchInfo); for (int32_t i = 0; i < numOfCols; ++i) { SColMatchItem* pmInfo = taosArrayGet(pColMatchInfo, i); - // ASSERT(pmInfo->matchType == COL_MATCH_FROM_SLOT_ID); SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->dstSlotId); @@ -746,7 +748,6 @@ void destroyMultiwayMergeOperatorInfo(void* param) { } int32_t getMultiwayMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { - ASSERT(pOptr != NULL); SSortExecInfo* pSortExecInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo)); SMultiwayMergeOperatorInfo* pInfo = (SMultiwayMergeOperatorInfo*)pOptr->info; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 8107cea4a09f56fbd0458c098994034aa5b3a4e5..0de38fe50083c18bed26f0be5241507bb66e1724 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -907,7 +907,7 @@ static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) { } bool isOverdue(TSKEY ekey, STimeWindowAggSupp* pTwSup) { - ASSERT(pTwSup->maxTs == INT64_MIN || pTwSup->maxTs > 0); + ASSERTS(pTwSup->maxTs == INT64_MIN || pTwSup->maxTs > 0, "maxts should greater than 0"); return pTwSup->maxTs != INT64_MIN && ekey < pTwSup->maxTs - pTwSup->waterMark; } @@ -1396,7 +1396,6 @@ static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) { while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { void* key = tSimpleHashGetKey(pIte, &keyLen); uint64_t groupId = *(uint64_t*)key; - ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))); TSKEY ts = *(int64_t*)((char*)key + sizeof(uint64_t)); SResultRowPosition* pPos = (SResultRowPosition*)pIte; int32_t code = saveWinResult(ts, pPos->pageId, pPos->offset, groupId, resWins); @@ -1547,7 +1546,7 @@ static void closeChildIntervalWindow(SOperatorInfo* pOperator, SArray* pChildren for (int32_t i = 0; i < size; i++) { SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i); SStreamIntervalOperatorInfo* pChInfo = pChildOp->info; - ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE); + ASSERTS(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE, "children trigger type should be at once"); pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs); closeStreamIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL, NULL, pOperator); @@ -1767,8 +1766,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh .maxTs = INT64_MIN, }; - ASSERT(as.calTrigger != STREAM_TRIGGER_MAX_DELAY); - pInfo->win = pTaskInfo->window; pInfo->inputOrder = (pPhyNode->window.inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; pInfo->resultTsOrder = (pPhyNode->window.outputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; @@ -2252,7 +2249,6 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB return; } blockDataEnsureCapacity(pBlock, size - (*pIndex)); - ASSERT(3 <= taosArrayGetSize(pBlock->pDataBlock)); SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); @@ -2346,6 +2342,17 @@ void doBuildResult(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* buildDataBlockFromGroupRes(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo); } +static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, + TSKEY* primaryKeys, int32_t prevPosition) { + int32_t startPos = prevPosition + 1; + if (startPos == pDataBlockInfo->rows) { + startPos = -1; + } else { + *pNext = getFinalTimeWindow(primaryKeys[startPos], pInterval); + } + return startPos; +} + static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t groupId, SHashObj* pUpdatedMap) { SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info; @@ -2359,7 +2366,6 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p SResultRow* pResult = NULL; int32_t forwardRows = 0; - ASSERT(pSDataBlock->pDataBlock != NULL); SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; @@ -2456,8 +2462,12 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p } int32_t prevEndPos = (forwardRows - 1) * step + startPos; ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0); - startPos = - getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC); + if (IS_FINAL_OP(pInfo)) { + startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos); + } else { + startPos = + getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC); + } if (startPos < 0) { break; } @@ -2482,7 +2492,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes); if (pInfo->pPullDataRes->info.rows != 0) { // process the rest of the data - ASSERT(IS_FINAL_OP(pInfo)); printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); return pInfo->pPullDataRes; } @@ -2543,7 +2552,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { pInfo->numOfDatapack++; printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval final recv" : "interval semi recv"); - ASSERT(pBlock->info.type != STREAM_INVERT); if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) { pInfo->binfo.pRes->info.type = pBlock->info.type; } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || @@ -2633,7 +2641,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes); if (pInfo->pPullDataRes->info.rows != 0) { // process the rest of the data - ASSERT(IS_FINAL_OP(pInfo)); printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); return pInfo->pPullDataRes; } @@ -2688,7 +2695,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, .deleteMarkSaved = 0, .calTriggerSaved = 0, }; - ASSERT(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY); + ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay"); pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(&pOperator->resultInfo, 4096); @@ -2713,7 +2720,6 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, initStreamFunciton(pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs); - ASSERT(numOfCols > 0); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); @@ -2724,6 +2730,9 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->pChildren = NULL; if (numOfChild > 0) { pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*)); + if (!pInfo->pChildren) { + goto _error; + } for (int32_t i = 0; i < numOfChild; i++) { SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pPhyNode, pTaskInfo, 0); if (pChildOp) { @@ -2746,7 +2755,6 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, // semi interval operator does not catch result pInfo->isFinal = false; pOperator->name = "StreamSemiIntervalOperator"; - ASSERT(pInfo->aggSup.currentPageId == -1); } if (!IS_FINAL_OP(pInfo) || numOfChild == 0) { @@ -3162,15 +3170,6 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData } } -void deleteWindow(SArray* pWinInfos, int32_t index, FDelete fp) { - ASSERT(index >= 0 && index < taosArrayGetSize(pWinInfos)); - if (fp) { - void* ptr = taosArrayGet(pWinInfos, index); - fp(ptr); - } - taosArrayRemove(pWinInfos, index); -} - static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) { SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* startDatas = (TSKEY*)pStartTsCol->pData; @@ -3218,7 +3217,6 @@ static int32_t copyUpdateResult(SSHashObj* pStUpdated, SArray* pUpdated) { int32_t iter = 0; while ((pIte = tSimpleHashIterate(pStUpdated, pIte, &iter)) != NULL) { void* key = tSimpleHashGetKey(pIte, &keyLen); - ASSERT(keyLen == sizeof(SSessionKey)); taosArrayPush(pUpdated, key); } taosArraySort(pUpdated, sessionKeyCompareAsc); @@ -3279,7 +3277,6 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; int32_t numOfOutput = pSup->numOfExprs; int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren); - ASSERT(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { SSessionKey* pWinKey = taosArrayGet(pWinArray, i); @@ -3380,7 +3377,6 @@ static void copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted) { void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList) { pGroupResInfo->pRows = pArrayList; pGroupResInfo->index = 0; - ASSERT(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo)); } void doBuildSessionResult(SOperatorInfo* pOperator, SStreamState* pState, SGroupResInfo* pGroupResInfo, @@ -4811,7 +4807,6 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys int32_t code = TSDB_CODE_SUCCESS; int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols); - ASSERT(numOfCols > 0); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); SInterval interval = { @@ -4831,7 +4826,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys .deleteMark = getDeleteMark(pIntervalPhyNode), }; - ASSERT(twAggSupp.calTrigger != STREAM_TRIGGER_MAX_DELAY); + ASSERTS(twAggSupp.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay"); pOperator->pTaskInfo = pTaskInfo; pInfo->interval = interval; diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index 484d9170698a2ca84b21079cfc9964d261713805..fd6215e3a1a5b8b7b99ab3f2a94416fda9c8adff 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -49,7 +49,9 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { } SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn) { - ASSERT(fn != NULL); + if (fn == NULL) { + return NULL; + } if (capacity == 0) { capacity = 4; @@ -66,7 +68,6 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn) { pHashObj->equalFp = memcmp; pHashObj->hashFp = fn; - ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); pHashObj->hashList = (SHNode **)taosMemoryCalloc(pHashObj->capacity, sizeof(void *)); if (!pHashObj->hashList) { diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 30911887bbb42dd9e116f8c3cea2e587bc456144..fa0cdb394377d9308c7fcbb6dfa0b0b553663401 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -800,6 +800,7 @@ STupleHandle* tsortNextTuple(SSortHandle* pHandle) { } } + // all sources are completed. if (pHandle->cmpParam.numOfSources == pHandle->numOfCompletedSources) { return NULL; } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 0b309bc8f56abd3870a07eebdb94e61f357ee23c..acfd5c07af0cd5c54d28ef24d586b18fbab99a20 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -841,36 +841,42 @@ int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SS for (int32_t i = 0; i < numOfCols; ++i) { numOfRows = (input[i].numOfRows > numOfRows) ? input[i].numOfRows : numOfRows; } - output->info.rows = numOfRows; - output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - for (int32_t i = 0; i < numOfCols; ++i) { - if ((input+i)->numOfRows < numOfRows) { - SColumnInfoData* pColInfoData = (input+i)->columnData; - int32_t startRow = (input+i)->numOfRows; - int32_t expandRows = numOfRows - startRow; - colInfoDataEnsureCapacity(pColInfoData, numOfRows, false); + + // create the basic block info structure + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pInfo = input[i].columnData; + SColumnInfoData d = {0}; + d.info = pInfo->info; + + blockDataAppendColInfo(output, &d); + } + + blockDataEnsureCapacity(output, numOfRows); + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pDest = taosArrayGet(output->pDataBlock, i); + + SColumnInfoData* pColInfoData = input[i].columnData; + colDataAssign(pDest, pColInfoData, input[i].numOfRows, &output->info); + + if (input[i].numOfRows < numOfRows) { + int32_t startRow = input[i].numOfRows; + int expandRows = numOfRows - startRow; bool isNull = colDataIsNull_s(pColInfoData, (input+i)->numOfRows - 1); if (isNull) { - colDataAppendNNULL(pColInfoData, startRow, expandRows); + colDataAppendNNULL(pDest, startRow, expandRows); } else { char* src = colDataGetData(pColInfoData, (input + i)->numOfRows - 1); - int32_t bytes = pColInfoData->info.bytes; - char* data = taosMemoryMalloc(bytes); - memcpy(data, src, bytes); for (int j = 0; j < expandRows; ++j) { - colDataAppend(pColInfoData, startRow+j, data, false); + colDataAppend(pDest, startRow+j, src, false); } //colDataAppendNItems(pColInfoData, startRow, data, expandRows); - taosMemoryFree(data); } } + } - taosArrayPush(output->pDataBlock, (input + i)->columnData); + output->info.rows = numOfRows; - if (IS_VAR_DATA_TYPE((input + i)->columnData->info.type)) { - output->info.hasVarCol = true; - } - } return 0; } @@ -1824,8 +1830,8 @@ int32_t doCallUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t convertDataBlockToScalarParm(&resultBlock, output); taosArrayDestroy(resultBlock.pDataBlock); } - - taosArrayDestroy(inputBlock.pDataBlock); + + blockDataFreeRes(&inputBlock); return err; } diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 40c75ce6baacbaad258ad10874af16f9466ad2d4..6c88e4d5c8615d262db99d3e0ab5e476269820a4 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -606,6 +606,7 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { } static bool udfdRpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_SYN_NOT_LEADER || + code == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) { if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 38203e61b0c95dfabcb8aa8d1a5f05bb73daa26b..dd06326dcb54259e76e4f7e2ce2eb82e286cb64e 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -832,7 +832,8 @@ void nodesDestroyNode(SNode* pNode) { if (pStmt->freeArrayFunc) { pStmt->freeArrayFunc(pStmt->pVgDataBlocks); } - tdDestroySVCreateTbReq(&pStmt->createTblReq); + tdDestroySVCreateTbReq(pStmt->pCreateTblReq); + taosMemoryFreeClear(pStmt->pCreateTblReq); taosCloseFile(&pStmt->fp); break; } diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index c74ec9c1479b872c11daf65bf1a0df0caa093a44..018d20d3edbc241173cf061e47eb72a539a7d4f8 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -197,7 +197,7 @@ SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, cons SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery); SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, - const SToken* pSubDbName, bool withMeta); + SToken* pSubDbName, bool withMeta); SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pRealTable, bool withMeta); SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pTopicName); @@ -216,7 +216,7 @@ SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool SNode* createDropFunctionStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pFuncName); SNode* createStreamOptions(SAstCreateContext* pCxt); SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pStreamName, SNode* pRealTable, - SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery); + SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery, SNodeList* pCols); SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pStreamName); SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId); SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId); diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index 5cc72f86923762454c8cae66ef9bbc7828b9ef05..7b816359f9c77b5fe370177fae2aedc67d6cc9a3 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -20,8 +20,6 @@ struct SToken; -#define IS_DATA_COL_ORDERED(spd) ((spd->orderStatus) == (int8_t)ORDER_STATUS_ORDERED) - #define NEXT_TOKEN(pSql, sToken) \ do { \ int32_t index = 0; \ @@ -37,108 +35,27 @@ struct SToken; } \ } while (0) -typedef enum EOrderStatus { - ORDER_STATUS_UNKNOWN = 0, - ORDER_STATUS_ORDERED = 1, - ORDER_STATUS_DISORDERED = 2, -} EOrderStatus; - -typedef enum EValStat { - VAL_STAT_HAS = 0x0, // 0 means has val - VAL_STAT_NONE = 0x01, // 1 means no val -} EValStat; - -typedef struct SBoundColumn { - int32_t offset; // all column offset value - int32_t toffset; // first part offset for SDataRow TODO: get offset from STSchema on future - uint8_t valStat; // EValStat. denote if current column bound or not(0 means has val, 1 means no val) -} SBoundColumn; - -typedef struct { - col_id_t schemaColIdx; - col_id_t boundIdx; - col_id_t finalIdx; -} SBoundIdxInfo; - -typedef struct SParsedDataColInfo { - col_id_t numOfCols; - col_id_t numOfBound; - uint16_t flen; // TODO: get from STSchema - uint16_t allNullLen; // TODO: get from STSchema(base on SDataRow) - uint16_t extendedVarLen; - uint16_t boundNullLen; // bound column len with all NULL value(without VarDataOffsetT/SColIdx part) - col_id_t *boundColumns; // bound column idx according to schema - SBoundColumn *cols; - SBoundIdxInfo *colIdxInfo; - int8_t orderStatus; // bound columns -} SParsedDataColInfo; - -typedef struct SInsertParseBaseContext { - SParseContext *pComCxt; - char *pSql; - SMsgBuf msg; -} SInsertParseBaseContext; - -typedef struct SInsertParseSyntaxCxt { - SParseContext *pComCxt; - char *pSql; - SMsgBuf msg; - SParseMetaCache *pMetaCache; -} SInsertParseSyntaxCxt; - -typedef struct SMemParam { - SRowBuilder *rb; - SSchema *schema; - int32_t toffset; - col_id_t colIdx; -} SMemParam; - -typedef struct { - uint8_t rowType; // default is 0, that is SDataRow - int32_t rowSize; -} SMemRowBuilder; - -typedef struct STableDataBlocks { - int8_t tsSource; // where does the UNIX timestamp come from, server or client - bool ordered; // if current rows are ordered or not - int32_t vgId; // virtual group id - int64_t prevTS; // previous timestamp, recorded to decide if the records array is ts ascending - int32_t numOfTables; // number of tables in current submit block - int32_t rowSize; // row size for current table - uint32_t nAllocSize; - uint32_t headerSize; // header for table info (uid, tid, submit metadata) - uint32_t size; - STableMeta *pTableMeta; // the tableMeta of current table, the table meta will be used during submit, keep a ref to - // avoid to be removed from cache - char *pData; - bool cloned; - int32_t createTbReqLen; - SParsedDataColInfo boundColumnInfo; - SRowBuilder rowBuilder; -} STableDataBlocks; +typedef struct SVgroupDataCxt { + int32_t vgId; + SSubmitReq2 *pData; +} SVgroupDataCxt; -int32_t insGetExtendedRowSize(STableDataBlocks *pBlock); -void insGetSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo *spd, col_id_t idx, int32_t *toffset, col_id_t *colIdx); -int32_t insSetBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks *dataBuf, int32_t numOfRows, SMsgBuf *pMsg); -int32_t insSchemaIdxCompar(const void *lhs, const void *rhs); -int32_t insBoundIdxCompar(const void *lhs, const void *rhs); -void insSetBoundColumnInfo(SParsedDataColInfo *pColList, SSchema *pSchema, col_id_t numOfCols); -void insDestroyBlockArrayList(SArray *pDataBlockList); -void insDestroyBlockHashmap(SHashObj *pDataBlockHash); -int32_t insInitRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo *pColInfo); -int32_t insGetDataBlockFromList(SHashObj *pHashList, void *id, int32_t idLen, int32_t size, int32_t startOffset, - int32_t rowSize, STableMeta *pTableMeta, STableDataBlocks **dataBlocks, - SArray *pBlockList, SVCreateTbReq *pCreateTbReq); -int32_t insMergeTableDataBlocks(SHashObj *pHashObj, SArray **pVgDataBlocks); -int32_t insBuildCreateTbMsg(STableDataBlocks *pBlocks, SVCreateTbReq *pCreateTbReq); -int32_t insAllocateMemForSize(STableDataBlocks *pDataBlock, int32_t allSize); int32_t insCreateSName(SName *pName, struct SToken *pTableName, int32_t acctId, const char *dbName, SMsgBuf *pMsgBuf); -int32_t insFindCol(struct SToken *pColname, int32_t start, int32_t end, SSchema *pSchema); +int16_t insFindCol(struct SToken *pColname, int16_t start, int16_t end, SSchema *pSchema); void insBuildCreateTbReq(SVCreateTbReq *pTbReq, const char *tname, STag *pTag, int64_t suid, const char *sname, SArray *tagName, uint8_t tagNum, int32_t ttl); -int32_t insMemRowAppend(SMsgBuf *pMsgBuf, const void *value, int32_t len, void *param); -int32_t insCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start); -int32_t insBuildOutput(SHashObj *pVgroupsHashObj, SArray *pVgDataBlocks, SArray **pDataBlocks); -void insDestroyDataBlock(STableDataBlocks *pDataBlock); +int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo *pInfo); +void insCheckTableDataOrder(STableDataCxt *pTableCxt, TSKEY tsKey); +int32_t insGetTableDataCxt(SHashObj *pHash, void *id, int32_t idLen, STableMeta *pTableMeta, + SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode); +int32_t initTableColSubmitData(STableDataCxt *pTableCxt); +int32_t insMergeTableDataCxt(SHashObj *pTableHash, SArray **pVgDataBlocks); +int32_t insBuildVgDataBlocks(SHashObj *pVgroupsHashObj, SArray *pVgDataBlocks, SArray **pDataBlocks); +void insDestroyTableDataCxtHashMap(SHashObj *pTableCxtHash); +void insDestroyVgroupDataCxt(SVgroupDataCxt *pVgCxt); +void insDestroyVgroupDataCxtList(SArray *pVgCxtList); +void insDestroyVgroupDataCxtHashMap(SHashObj *pVgCxtHash); +void insDestroyTableDataCxt(STableDataCxt *pTableCxt); +void insDestroyBoundColInfo(SBoundColInfo *pInfo); #endif // TDENGINE_PAR_INSERT_UTIL_H diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index cd1260584e5ea6c4beea82088f0b28f6e7c40d33..3a7161be56ab1cca62fbaac4d587c667d7f387e2 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -458,14 +458,16 @@ tag_item(A) ::= column_name(B) column_alias(C). tag_item(A) ::= column_name(B) AS column_alias(C). { A = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &B), &C); } /************************************************ create index ********************************************************/ -cmd ::= CREATE SMA INDEX not_exists_opt(D) +cmd ::= CREATE SMA INDEX not_exists_opt(D) full_table_name(A) ON full_table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, D, A, B, NULL, C); } +cmd ::= CREATE INDEX not_exists_opt(D) + full_table_name(A) ON full_table_name(B) NK_LP col_name_list(C) NK_RP. { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_NORMAL, D, A, B, C, NULL); } cmd ::= DROP INDEX exists_opt(B) full_table_name(A). { pCxt->pRootNode = createDropIndexStmt(pCxt, B, A); } -index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL +index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL NK_LP duration_literal(C) NK_RP sliding_opt(D) sma_stream_opt(E). { A = createIndexOption(pCxt, B, releaseRawExprNode(pCxt, C), NULL, D, E); } -index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL - NK_LP duration_literal(C) NK_COMMA duration_literal(D) NK_RP sliding_opt(E) +index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL + NK_LP duration_literal(C) NK_COMMA duration_literal(D) NK_RP sliding_opt(E) sma_stream_opt(F). { A = createIndexOption(pCxt, B, releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, D), E, F); } %type func_list { SNodeList* } @@ -539,9 +541,15 @@ bufsize_opt(A) ::= BUFSIZE NK_INTEGER(B). /************************************************ create/drop stream **************************************************/ cmd ::= CREATE STREAM not_exists_opt(E) stream_name(A) stream_options(B) INTO - full_table_name(C) tags_def_opt(F) subtable_opt(G) AS query_or_subquery(D). { pCxt->pRootNode = createCreateStreamStmt(pCxt, E, &A, C, B, F, G, D); } + full_table_name(C) col_list_opt(H) tags_def_opt(F) subtable_opt(G) + AS query_or_subquery(D). { pCxt->pRootNode = createCreateStreamStmt(pCxt, E, &A, C, B, F, G, D, H); } cmd ::= DROP STREAM exists_opt(A) stream_name(B). { pCxt->pRootNode = createDropStreamStmt(pCxt, A, &B); } +%type col_list_opt { SNodeList* } +%destructor col_list_opt { nodesDestroyList($$); } +col_list_opt(A) ::= . { A = NULL; } +col_list_opt(A) ::= NK_LP col_name_list(B) NK_RP. { A = B; } + stream_options(A) ::= . { A = createStreamOptions(pCxt); } stream_options(A) ::= stream_options(B) TRIGGER AT_ONCE. { ((SStreamOptions*)B)->triggerType = STREAM_TRIGGER_AT_ONCE; A = B; } stream_options(A) ::= stream_options(B) TRIGGER WINDOW_CLOSE. { ((SStreamOptions*)B)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; A = B; } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index db43fa5ed94c3431dca308439250c9b664ff0cf0..98954250c8c262e38f48acf0e5e530068c06639a 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1593,8 +1593,11 @@ SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, } SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, - const SToken* pSubDbName, bool withMeta) { + SToken* pSubDbName, bool withMeta) { CHECK_PARSER_STATUS(pCxt); + if (!checkDbName(pCxt, pSubDbName, true)) { + return NULL; + } SCreateTopicStmt* pStmt = (SCreateTopicStmt*)nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT); CHECK_OUT_OF_MEM(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName); @@ -1742,21 +1745,20 @@ SNode* createStreamOptions(SAstCreateContext* pCxt) { } SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pStreamName, SNode* pRealTable, - SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery) { + SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery, SNodeList* pCols) { CHECK_PARSER_STATUS(pCxt); SCreateStreamStmt* pStmt = (SCreateStreamStmt*)nodesMakeNode(QUERY_NODE_CREATE_STREAM_STMT); CHECK_OUT_OF_MEM(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->streamName, pStreamName); - if (NULL != pRealTable) { - strcpy(pStmt->targetDbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->targetTabName, ((SRealTableNode*)pRealTable)->table.tableName); - nodesDestroyNode(pRealTable); - } + strcpy(pStmt->targetDbName, ((SRealTableNode*)pRealTable)->table.dbName); + strcpy(pStmt->targetTabName, ((SRealTableNode*)pRealTable)->table.tableName); + nodesDestroyNode(pRealTable); pStmt->ignoreExists = ignoreExists; pStmt->pOptions = (SStreamOptions*)pOptions; pStmt->pQuery = pQuery; pStmt->pTags = pTags; pStmt->pSubtable = pSubtable; + pStmt->pCols = pCols; return (SNode*)pStmt; } diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 954e6c26fd6850ea8d870b09d09040828dd856dc..fae62626fa3666909208ce9220215baea8be3810 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -355,7 +355,12 @@ static int32_t collectMetaKeyFromDescribe(SCollectMetaKeyCxt* pCxt, SDescribeStm } static int32_t collectMetaKeyFromCreateStream(SCollectMetaKeyCxt* pCxt, SCreateStreamStmt* pStmt) { - return collectMetaKeyFromQuery(pCxt, pStmt->pQuery); + int32_t code = + reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->targetDbName, pStmt->targetTabName, pCxt->pMetaCache); + if (TSDB_CODE_SUCCESS == code) { + code = collectMetaKeyFromQuery(pCxt, pStmt->pQuery); + } + return code; } static int32_t collectMetaKeyFromShowDnodes(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 358baa74cb3932887894bda79a6e26e9a888c797..b5baa54f77c13dd4403f9a43f86ab304b13d6813 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -45,95 +45,46 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* return TSDB_CODE_SUCCESS; } -typedef struct SmlExecTableHandle { - SParsedDataColInfo tags; // each table - SVCreateTbReq createTblReq; // each table -} SmlExecTableHandle; - -typedef struct SmlExecHandle { - SHashObj* pBlockHash; - SmlExecTableHandle tableExecHandle; - SQuery* pQuery; -} SSmlExecHandle; - -static void smlDestroyTableHandle(void* pHandle) { - SmlExecTableHandle* handle = (SmlExecTableHandle*)pHandle; - destroyBoundColumnInfo(&handle->tags); - tdDestroySVCreateTbReq(&handle->createTblReq); -} - -static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SSchema* pSchema, bool isTag) { - col_id_t nCols = pColList->numOfCols; - - pColList->numOfBound = 0; - pColList->boundNullLen = 0; - memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols); - for (col_id_t i = 0; i < nCols; ++i) { - pColList->cols[i].valStat = VAL_STAT_NONE; +static int32_t smlBoundColumnData(SArray* cols, SBoundColInfo* pBoundInfo, SSchema* pSchema, bool isTag) { + bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool)); + if (NULL == pUseCols) { + return TSDB_CODE_OUT_OF_MEMORY; } - bool isOrdered = true; - col_id_t lastColIdx = -1; // last column found + pBoundInfo->numOfBound = 0; + int16_t lastColIdx = -1; // last column found + int32_t code = TSDB_CODE_SUCCESS; + for (int i = 0; i < taosArrayGetSize(cols); ++i) { - SSmlKv* kv = taosArrayGetP(cols, i); + SSmlKv* kv = taosArrayGet(cols, i); SToken sToken = {.n = kv->keyLen, .z = (char*)kv->key}; col_id_t t = lastColIdx + 1; - col_id_t index = ((t == 0 && !isTag) ? 0 : insFindCol(&sToken, t, nCols, pSchema)); - uDebug("SML, index:%d, t:%d, ncols:%d", index, t, nCols); + col_id_t index = ((t == 0 && !isTag) ? 0 : insFindCol(&sToken, t, pBoundInfo->numOfCols, pSchema)); + uDebug("SML, index:%d, t:%d, ncols:%d", index, t, pBoundInfo->numOfCols); if (index < 0 && t > 0) { index = insFindCol(&sToken, 0, t, pSchema); - isOrdered = false; } + if (index < 0) { uError("smlBoundColumnData. index:%d", index); - return TSDB_CODE_SML_INVALID_DATA; + code = TSDB_CODE_SML_INVALID_DATA; + goto end; } - if (pColList->cols[index].valStat == VAL_STAT_HAS) { + if (pUseCols[index]) { uError("smlBoundColumnData. already set. index:%d", index); - return TSDB_CODE_SML_INVALID_DATA; + code = TSDB_CODE_SML_INVALID_DATA; + goto end; } lastColIdx = index; - pColList->cols[index].valStat = VAL_STAT_HAS; - pColList->boundColumns[pColList->numOfBound] = index; - ++pColList->numOfBound; - switch (pSchema[t].type) { - case TSDB_DATA_TYPE_BINARY: - pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + CHAR_BYTES); - break; - case TSDB_DATA_TYPE_NCHAR: - pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); - break; - default: - pColList->boundNullLen += TYPE_BYTES[pSchema[t].type]; - break; - } + pUseCols[index] = true; + pBoundInfo->pColIndex[pBoundInfo->numOfBound] = index; + ++pBoundInfo->numOfBound; } - pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; - - if (!isOrdered) { - pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); - if (NULL == pColList->colIdxInfo) { - return TSDB_CODE_OUT_OF_MEMORY; - } - SBoundIdxInfo* pColIdx = pColList->colIdxInfo; - for (col_id_t i = 0; i < pColList->numOfBound; ++i) { - pColIdx[i].schemaColIdx = pColList->boundColumns[i]; - pColIdx[i].boundIdx = i; - } - taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insSchemaIdxCompar); - for (col_id_t i = 0; i < pColList->numOfBound; ++i) { - pColIdx[i].finalIdx = i; - } - taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insBoundIdxCompar); - } - - if (pColList->numOfCols > pColList->numOfBound) { - memset(&pColList->boundColumns[pColList->numOfBound], 0, - sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); - } +end: + taosMemoryFree(pUseCols); - return TSDB_CODE_SUCCESS; + return code; } /** @@ -146,7 +97,7 @@ static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SS * @param msg * @return int32_t */ -static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* pSchema, STag** ppTag, SArray** tagName, +static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchema, STag** ppTag, SArray** tagName, SMsgBuf* msg) { SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal)); if (!pTagArray) { @@ -159,8 +110,8 @@ static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* p int32_t code = TSDB_CODE_SUCCESS; for (int i = 0; i < tags->numOfBound; ++i) { - SSchema* pTagSchema = &pSchema[tags->boundColumns[i]]; - SSmlKv* kv = taosArrayGetP(cols, i); + SSchema* pTagSchema = &pSchema[tags->pColIndex[i]]; + SSmlKv* kv = taosArrayGet(cols, i); taosArrayPush(*tagName, pTagSchema->name); STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; @@ -207,153 +158,237 @@ end: return code; } -int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta, - char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int16_t msgBufLen) { - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; +STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) { + STableDataCxt* pTableCxt = NULL; + SVCreateTbReq* pCreateTbReq = NULL; + int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, + sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false); + if (ret != TSDB_CODE_SUCCESS) { + return NULL; + } - SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle; - smlDestroyTableHandle(&smlHandle->tableExecHandle); // free for each table - SSchema* pTagsSchema = getTableTagSchema(pTableMeta); - insSetBoundColumnInfo(&smlHandle->tableExecHandle.tags, pTagsSchema, getNumOfTags(pTableMeta)); - int ret = smlBoundColumnData(tags, &smlHandle->tableExecHandle.tags, pTagsSchema, true); + ret = initTableColSubmitData(pTableCxt); if (ret != TSDB_CODE_SUCCESS) { - buildInvalidOperationMsg(&pBuf, "bound tags error"); + return NULL; + } + return pTableCxt; +} + +int32_t smlBuildRow(STableDataCxt* pTableCxt) { + SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + int ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); + if (TSDB_CODE_SUCCESS != ret) { return ret; } - STag* pTag = NULL; - SArray* tagName = NULL; - ret = smlBuildTagRow(tags, &smlHandle->tableExecHandle.tags, pTagsSchema, &pTag, &tagName, &pBuf); + insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow)); + return TSDB_CODE_SUCCESS; +} + +int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32_t index) { + int ret = TSDB_CODE_SUCCESS; + SSchema* pColSchema = schema + index; + SColVal* pVal = taosArrayGet(pTableCxt->pValues, index); + SSmlKv* kv = (SSmlKv*)data; + if (kv->type == TSDB_DATA_TYPE_NCHAR) { + int32_t len = 0; + char* pUcs4 = taosMemoryCalloc(1, pColSchema->bytes - VARSTR_HEADER_SIZE); + if (NULL == pUcs4) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len)) { + if (errno == E2BIG) { + ret = TSDB_CODE_PAR_VALUE_TOO_LONG; + goto end; + } + ret = TSDB_CODE_TSC_INVALID_VALUE; + goto end; + } + pVal->value.pData = pUcs4; + pVal->value.nData = len; + } else if (kv->type == TSDB_DATA_TYPE_BINARY) { + pVal->value.nData = kv->length; + pVal->value.pData = (uint8_t*)kv->value; + } else { + memcpy(&pVal->value.val, &(kv->value), kv->length); + } + pVal->flag = CV_FLAG_VALUE; + +end: + return ret; +} + +int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, + STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, + char* msgBuf, int16_t msgBufLen) { + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + + SSchema* pTagsSchema = getTableTagSchema(pTableMeta); + SBoundColInfo bindTags = {0}; + SVCreateTbReq* pCreateTblReq = NULL; + SArray* tagName = NULL; + + insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags); + int ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true); if (ret != TSDB_CODE_SUCCESS) { - taosArrayDestroy(tagName); - return ret; + buildInvalidOperationMsg(&pBuf, "bound tags error"); + goto end; } - insBuildCreateTbReq(&smlHandle->tableExecHandle.createTblReq, tableName, pTag, pTableMeta->suid, NULL, tagName, - pTableMeta->tableInfo.numOfTags, ttl); - taosArrayDestroy(tagName); + STag* pTag = NULL; - smlHandle->tableExecHandle.createTblReq.ctb.stbName = taosMemoryMalloc(sTableNameLen + 1); - memcpy(smlHandle->tableExecHandle.createTblReq.ctb.stbName, sTableName, sTableNameLen); - smlHandle->tableExecHandle.createTblReq.ctb.stbName[sTableNameLen] = 0; + ret = smlBuildTagRow(tags, &bindTags, pTagsSchema, &pTag, &tagName, &pBuf); + if (ret != TSDB_CODE_SUCCESS) { + goto end; + } - STableDataBlocks* pDataBlock = NULL; - ret = insGetDataBlockFromList(smlHandle->pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid), - TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize, - pTableMeta, &pDataBlock, NULL, &smlHandle->tableExecHandle.createTblReq); + pCreateTblReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == pCreateTblReq) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + insBuildCreateTbReq(pCreateTblReq, tableName, pTag, pTableMeta->suid, NULL, tagName, pTableMeta->tableInfo.numOfTags, + ttl); + + pCreateTblReq->ctb.stbName = taosMemoryCalloc(1, sTableNameLen + 1); + memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen); + + if (dataFormat) { + STableDataCxt** pTableCxt = (STableDataCxt**)taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, + &pTableMeta->uid, sizeof(pTableMeta->uid)); + if (NULL == pTableCxt) { + ret = buildInvalidOperationMsg(&pBuf, "dataformat true. get tableDataCtx error"); + goto end; + } + (*pTableCxt)->pData->flags |= SUBMIT_REQ_AUTO_CREATE_TABLE; + (*pTableCxt)->pData->pCreateTbReq = pCreateTblReq; + (*pTableCxt)->pMeta->uid = pTableMeta->uid; + (*pTableCxt)->pMeta->vgId = pTableMeta->vgId; + pCreateTblReq = NULL; + goto end; + } + + STableDataCxt* pTableCxt = NULL; + ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, + sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false); if (ret != TSDB_CODE_SUCCESS) { - buildInvalidOperationMsg(&pBuf, "create data block error"); - return ret; + buildInvalidOperationMsg(&pBuf, "insGetTableDataCxt error"); + goto end; } SSchema* pSchema = getTableColumnSchema(pTableMeta); - - ret = smlBoundColumnData(colsSchema, &pDataBlock->boundColumnInfo, pSchema, false); + ret = smlBoundColumnData(colsSchema, &pTableCxt->boundColsInfo, pSchema, false); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "bound cols error"); - return ret; + goto end; } - int32_t extendedRowSize = insGetExtendedRowSize(pDataBlock); - SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; - SRowBuilder* pBuilder = &pDataBlock->rowBuilder; - SMemParam param = {.rb = pBuilder}; - insInitRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo); + ret = initTableColSubmitData(pTableCxt); + if (ret != TSDB_CODE_SUCCESS) { + buildInvalidOperationMsg(&pBuf, "initTableColSubmitData error"); + goto end; + } int32_t rowNum = taosArrayGetSize(cols); if (rowNum <= 0) { - return buildInvalidOperationMsg(&pBuf, "cols size <= 0"); - } - ret = insAllocateMemForSize(pDataBlock, extendedRowSize * rowNum); - if (ret != TSDB_CODE_SUCCESS) { - buildInvalidOperationMsg(&pBuf, "allocate memory error"); - return ret; + ret = buildInvalidOperationMsg(&pBuf, "cols size <= 0"); + goto end; } + for (int32_t r = 0; r < rowNum; ++r) { - STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header - tdSRowResetBuf(pBuilder, row); - void* rowData = taosArrayGetP(cols, r); - size_t rowDataSize = 0; - if (format) { - rowDataSize = taosArrayGetSize(rowData); - } + void* rowData = taosArrayGetP(cols, r); // 1. set the parsed value from sql string - for (int c = 0, j = 0; c < spd->numOfBound; ++c) { - SSchema* pColSchema = &pSchema[spd->boundColumns[c]]; - - param.schema = pColSchema; - insGetSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); - - SSmlKv* kv = NULL; - if (format) { - if (j < rowDataSize) { - kv = taosArrayGetP(rowData, j); - if (rowDataSize != spd->numOfBound && j != 0 && - (kv->keyLen != strlen(pColSchema->name) || strncmp(kv->key, pColSchema->name, kv->keyLen) != 0)) { - kv = NULL; - } else { - j++; - } - } - } else { - void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); - if (p) kv = *p; + for (int c = 0; c < pTableCxt->boundColsInfo.numOfBound; ++c) { + SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]]; + SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]); + void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); + if (p == NULL) { + continue; } + SSmlKv* kv = *(SSmlKv**)p; - if (kv) { - int32_t colLen = kv->length; - if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { - uDebug("SML:data before:%" PRId64 ", precision:%d", kv->i, pTableMeta->tableInfo.precision); - kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision); - uDebug("SML:data after:%" PRId64 ", precision:%d", kv->i, pTableMeta->tableInfo.precision); + if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { + kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision); + } + if (kv->type == TSDB_DATA_TYPE_NCHAR) { + int32_t len = 0; + char* pUcs4 = taosMemoryCalloc(1, pColSchema->bytes - VARSTR_HEADER_SIZE); + if (NULL == pUcs4) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; } - - if (IS_VAR_DATA_TYPE(kv->type)) { - insMemRowAppend(&pBuf, kv->value, colLen, ¶m); - } else { - insMemRowAppend(&pBuf, &(kv->value), colLen, ¶m); + if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len)) { + if (errno == E2BIG) { + buildInvalidOperationMsg(&pBuf, "value too long"); + ret = TSDB_CODE_PAR_VALUE_TOO_LONG; + goto end; + } + ret = buildInvalidOperationMsg(&pBuf, strerror(errno)); + goto end; } + pVal->value.pData = pUcs4; + pVal->value.nData = len; + } else if (kv->type == TSDB_DATA_TYPE_BINARY) { + pVal->value.nData = kv->length; + pVal->value.pData = (uint8_t*)kv->value; } else { - pBuilder->hasNone = true; - } - - if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { - TSKEY tsKey = TD_ROW_KEY(row); - insCheckTimestamp(pDataBlock, (const char*)&tsKey); + memcpy(&pVal->value.val, &(kv->value), kv->length); } + pVal->flag = CV_FLAG_VALUE; } - // set the null value for the columns that do not assign values - if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - pBuilder->hasNone = true; + SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); + if (TSDB_CODE_SUCCESS != ret) { + buildInvalidOperationMsg(&pBuf, "tRowBuild error"); + goto end; } - - tdSRowEnd(pBuilder); - pDataBlock->size += extendedRowSize; + insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow)); } - SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); - return insSetBlockInfo(pBlocks, pDataBlock, rowNum, &pBuf); +end: + insDestroyBoundColInfo(&bindTags); + taosMemoryFree(pCreateTblReq); + taosArrayDestroy(tagName); + return ret; } -void* smlInitHandle(SQuery* pQuery) { - SSmlExecHandle* handle = taosMemoryCalloc(1, sizeof(SSmlExecHandle)); - if (!handle) return NULL; - handle->pBlockHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - handle->pQuery = pQuery; - - return handle; -} +SQuery* smlInitHandle() { + SQuery* pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); + if (NULL == pQuery) { + uError("create pQuery error"); + return NULL; + } + pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + pQuery->haveResultSet = false; + pQuery->msgType = TDMT_VND_SUBMIT; + SVnodeModifyOpStmt* stmt = (SVnodeModifyOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT); + if (NULL == stmt) { + uError("create SVnodeModifyOpStmt error"); + qDestroyQuery(pQuery); + return NULL; + } + stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + stmt->freeHashFunc = insDestroyTableDataCxtHashMap; + stmt->freeArrayFunc = insDestroyVgroupDataCxtList; -void smlDestroyHandle(void* pHandle) { - if (!pHandle) return; - SSmlExecHandle* handle = (SSmlExecHandle*)pHandle; - insDestroyBlockHashmap(handle->pBlockHash); - smlDestroyTableHandle(&handle->tableExecHandle); - taosMemoryFree(handle); + pQuery->pRoot = (SNode*)stmt; + return pQuery; } -int32_t smlBuildOutput(void* handle, SHashObj* pVgHash) { - SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle; - return qBuildStmtOutput(smlHandle->pQuery, pVgHash, smlHandle->pBlockHash); +int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) { + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)(handle)->pRoot; + // merge according to vgId + int32_t code = insMergeTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks); + if (code != TSDB_CODE_SUCCESS) { + uError("insMergeTableDataCxt failed"); + return code; + } + code = insBuildVgDataBlocks(pVgHash, pStmt->pVgDataBlocks, &pStmt->pDataBlocks); + if (code != TSDB_CODE_SUCCESS) { + uError("insBuildVgDataBlocks failed"); + return code; + } + return code; } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index f7415b9ba83a642f9fa579abdd406ade01c683f8..582bb466d2cd00cd3061572217fb0f285577fedf 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -38,13 +38,13 @@ } while (TK_NK_SPACE == (token).type) typedef struct SInsertParseContext { - SParseContext* pComCxt; - SMsgBuf msg; - char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW]; - SParsedDataColInfo tags; // for stmt - bool missCache; - bool usingDuplicateTable; - bool forceUpdate; + SParseContext* pComCxt; + SMsgBuf msg; + char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW]; + SBoundColInfo tags; // for stmt + bool missCache; + bool usingDuplicateTable; + bool forceUpdate; } SInsertParseContext; typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param); @@ -173,21 +173,19 @@ static int32_t parseDuplicateUsingClause(SInsertParseContext* pCxt, SVnodeModify } // pStmt->pSql -> field1_name, ...) -static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, bool isTags, - SParsedDataColInfo* pColList, SSchema* pSchema) { - col_id_t nCols = pColList->numOfCols; - - pColList->numOfBound = 0; - pColList->boundNullLen = 0; - memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols); - for (col_id_t i = 0; i < nCols; ++i) { - pColList->cols[i].valStat = VAL_STAT_NONE; +static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, bool isTags, SSchema* pSchema, + SBoundColInfo* pBoundInfo) { + bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool)); + if (NULL == pUseCols) { + return TSDB_CODE_OUT_OF_MEMORY; } - SToken token; - bool isOrdered = true; - col_id_t lastColIdx = -1; // last column found - while (1) { + pBoundInfo->numOfBound = 0; + + int16_t lastColIdx = -1; // last column found + int32_t code = TSDB_CODE_SUCCESS; + while (TSDB_CODE_SUCCESS == code) { + SToken token; NEXT_TOKEN(*pSql, token); if (TK_NK_RP == token.type) { @@ -199,64 +197,30 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, b token.z = tmpTokenBuf; token.n = strdequote(token.z); - col_id_t t = lastColIdx + 1; - col_id_t index = insFindCol(&token, t, nCols, pSchema); + int16_t t = lastColIdx + 1; + int16_t index = insFindCol(&token, t, pBoundInfo->numOfCols, pSchema); if (index < 0 && t > 0) { index = insFindCol(&token, 0, t, pSchema); - isOrdered = false; } if (index < 0) { - return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMN, token.z); - } - if (pColList->cols[index].valStat == VAL_STAT_HAS) { - return buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", token.z); - } - lastColIdx = index; - pColList->cols[index].valStat = VAL_STAT_HAS; - pColList->boundColumns[pColList->numOfBound] = index; - ++pColList->numOfBound; - switch (pSchema[t].type) { - case TSDB_DATA_TYPE_BINARY: - pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + CHAR_BYTES); - break; - case TSDB_DATA_TYPE_NCHAR: - pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); - break; - default: - pColList->boundNullLen += TYPE_BYTES[pSchema[t].type]; - break; + code = generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMN, token.z); + } else if (pUseCols[index]) { + code = buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", token.z); + } else { + lastColIdx = index; + pUseCols[index] = true; + pBoundInfo->pColIndex[pBoundInfo->numOfBound] = index; + ++pBoundInfo->numOfBound; } } - if (!isTags && pColList->cols[0].valStat == VAL_STAT_NONE) { - return buildInvalidOperationMsg(&pCxt->msg, "primary timestamp column can not be null"); + if (TSDB_CODE_SUCCESS == code && !isTags && !pUseCols[0]) { + code = buildInvalidOperationMsg(&pCxt->msg, "primary timestamp column can not be null"); } - pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; - - if (!isOrdered) { - pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); - if (NULL == pColList->colIdxInfo) { - return TSDB_CODE_OUT_OF_MEMORY; - } - SBoundIdxInfo* pColIdx = pColList->colIdxInfo; - for (col_id_t i = 0; i < pColList->numOfBound; ++i) { - pColIdx[i].schemaColIdx = pColList->boundColumns[i]; - pColIdx[i].boundIdx = i; - } - taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insSchemaIdxCompar); - for (col_id_t i = 0; i < pColList->numOfBound; ++i) { - pColIdx[i].finalIdx = i; - } - taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insBoundIdxCompar); - } - - if (pColList->numOfCols > pColList->numOfBound) { - memset(&pColList->boundColumns[pColList->numOfBound], 0, - sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); - } + taosMemoryFree(pUseCols); - return TSDB_CODE_SUCCESS; + return code; } static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t* time, SMsgBuf* pMsgBuf) { @@ -519,8 +483,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, // input pStmt->pSql: [(tag1_name, ...)] TAGS (tag1_value, ...) ... // output pStmt->pSql: TAGS (tag1_value, ...) ... static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - SSchema* pTagsSchema = getTableTagSchema(pStmt->pTableMeta); - insSetBoundColumnInfo(&pCxt->tags, pTagsSchema, getNumOfTags(pStmt->pTableMeta)); + insInitBoundColsInfo(getNumOfTags(pStmt->pTableMeta), &pCxt->tags); SToken token; int32_t index = 0; @@ -530,7 +493,7 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStm } pStmt->pSql += index; - return parseBoundColumns(pCxt, &pStmt->pSql, true, &pCxt->tags, pTagsSchema); + return parseBoundColumns(pCxt, &pStmt->pSql, true, getTableTagSchema(pStmt->pTableMeta), &pCxt->tags); } static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SSchema* pTagSchema, SToken* pToken, @@ -561,10 +524,15 @@ static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStm return code; } -static void buildCreateTbReq(SVnodeModifyOpStmt* pStmt, STag* pTag, SArray* pTagName) { - insBuildCreateTbReq(&pStmt->createTblReq, pStmt->targetTableName.tname, pTag, pStmt->pTableMeta->suid, +static int32_t buildCreateTbReq(SVnodeModifyOpStmt* pStmt, STag* pTag, SArray* pTagName) { + pStmt->pCreateTblReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == pStmt->pCreateTblReq) { + return TSDB_CODE_OUT_OF_MEMORY; + } + insBuildCreateTbReq(pStmt->pCreateTblReq, pStmt->targetTableName.tname, pTag, pStmt->pTableMeta->suid, pStmt->usingTableName.tname, pTagName, pStmt->pTableMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); + return TSDB_CODE_SUCCESS; } static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf) { @@ -618,7 +586,7 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt break; } - SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i]]; + SSchema* pTagSchema = &pSchema[pCxt->tags.pColIndex[i]]; isJson = pTagSchema->type == TSDB_DATA_TYPE_JSON; code = checkAndTrimValue(&token, pCxt->tmpTokenBuf, &pCxt->msg); if (TSDB_CODE_SUCCESS == code) { @@ -631,7 +599,7 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt } if (TSDB_CODE_SUCCESS == code && !isParseBindParam) { - buildCreateTbReq(pStmt, pTag, pTagName); + code = buildCreateTbReq(pStmt, pTag, pTagName); pTag = NULL; } @@ -699,8 +667,8 @@ static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifyOpStmt* if (TK_NK_INTEGER != token.type) { return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", token.z); } - pStmt->createTblReq.ttl = taosStr2Int32(token.z, NULL, 10); - if (pStmt->createTblReq.ttl < 0) { + pStmt->pCreateTblReq->ttl = taosStr2Int32(token.z, NULL, 10); + if (pStmt->pCreateTblReq->ttl < 0) { return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", token.z); } } else if (TK_COMMENT == token.type) { @@ -713,11 +681,11 @@ static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifyOpStmt* return buildSyntaxErrMsg(&pCxt->msg, "comment too long", token.z); } int32_t len = trimString(token.z, token.n, pCxt->tmpTokenBuf, TSDB_TB_COMMENT_LEN); - pStmt->createTblReq.comment = strndup(pCxt->tmpTokenBuf, len); - if (NULL == pStmt->createTblReq.comment) { + pStmt->pCreateTblReq->comment = strndup(pCxt->tmpTokenBuf, len); + if (NULL == pStmt->pCreateTblReq->comment) { return TSDB_CODE_OUT_OF_MEMORY; } - pStmt->createTblReq.commentLen = len; + pStmt->pCreateTblReq->commentLen = len; } else { break; } @@ -975,26 +943,22 @@ static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModif return skipParentheses(pCxt, &pStmt->pSql); } -static int32_t getTableDataBlocks(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataBlocks** pDataBuf) { +static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt** pTableCxt) { if (pCxt->pComCxt->async) { - uint64_t uid = pStmt->pTableMeta->uid; - if (pStmt->usingTableProcessing) { - pStmt->pTableMeta->uid = 0; - } - - return insGetDataBlockFromList( - pStmt->pTableBlockHashObj, &uid, sizeof(pStmt->pTableMeta->uid), TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), - getTableInfo(pStmt->pTableMeta).rowSize, pStmt->pTableMeta, pDataBuf, NULL, &pStmt->createTblReq); + return insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), + pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false); } + char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(&pStmt->targetTableName, tbFName); - return insGetDataBlockFromList(pStmt->pTableBlockHashObj, tbFName, strlen(tbFName), TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), getTableInfo(pStmt->pTableMeta).rowSize, pStmt->pTableMeta, - pDataBuf, NULL, &pStmt->createTblReq); + if (pStmt->usingTableProcessing) { + pStmt->pTableMeta->uid = 0; + } + return insGetTableDataCxt(pStmt->pTableBlockHashObj, tbFName, strlen(tbFName), pStmt->pTableMeta, + &pStmt->pCreateTblReq, pTableCxt, NULL != pCxt->pComCxt->pStmtCb); } -static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, - STableDataBlocks* pDataBuf) { +static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { SToken token; int32_t index = 0; NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index); @@ -1004,13 +968,30 @@ static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOp return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } // pStmt->pSql -> field1_name, ...) - return parseBoundColumns(pCxt, &pStmt->pSql, false, &pDataBuf->boundColumnInfo, - getTableColumnSchema(pStmt->pTableMeta)); + return parseBoundColumns(pCxt, &pStmt->pSql, false, getTableColumnSchema(pStmt->pTableMeta), + &pTableCxt->boundColsInfo); } if (NULL != pStmt->pBoundCols) { - return parseBoundColumns(pCxt, &pStmt->pBoundCols, false, &pDataBuf->boundColumnInfo, - getTableColumnSchema(pStmt->pTableMeta)); + return parseBoundColumns(pCxt, &pStmt->pBoundCols, false, getTableColumnSchema(pStmt->pTableMeta), + &pTableCxt->boundColsInfo); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t initTableColSubmitData(STableDataCxt* pTableCxt) { + if (0 == (pTableCxt->pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT)) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < pTableCxt->boundColsInfo.numOfBound; ++i) { + SSchema* pSchema = &pTableCxt->pMeta->schema[pTableCxt->boundColsInfo.pColIndex[i]]; + SColData* pCol = taosArrayReserve(pTableCxt->pData->aCol, 1); + if (NULL == pCol) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tColDataInit(pCol, pSchema->colId, pSchema->type, 0); } return TSDB_CODE_SUCCESS; @@ -1021,13 +1002,16 @@ static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOp // 2. VALUES ... | FILE ... // output pStmt->pSql: VALUES ... | FILE ... static int32_t parseSchemaClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, - STableDataBlocks** pDataBuf) { + STableDataCxt** pTableCxt) { int32_t code = parseUsingClauseBottom(pCxt, pStmt); if (TSDB_CODE_SUCCESS == code) { - code = getTableDataBlocks(pCxt, pStmt, pDataBuf); + code = getTableDataCxt(pCxt, pStmt, pTableCxt); } if (TSDB_CODE_SUCCESS == code) { - code = parseBoundColumnsClause(pCxt, pStmt, *pDataBuf); + code = parseBoundColumnsClause(pCxt, pStmt, *pTableCxt); + } + if (TSDB_CODE_SUCCESS == code) { + code = initTableColSubmitData(*pTableCxt); } return code; } @@ -1050,108 +1034,88 @@ static int32_t parseSchemaClauseTop(SInsertParseContext* pCxt, SVnodeModifyOpStm } static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, SToken* pToken, SSchema* pSchema, - int16_t timePrec, _row_append_fn_t func, void* param) { - int64_t iv; - uint64_t uv; - char* endptr = NULL; - + int16_t timePrec, SColVal* pVal) { switch (pSchema->type) { case TSDB_DATA_TYPE_BOOL: { if ((pToken->type == TK_NK_BOOL || pToken->type == TK_NK_STRING) && (pToken->n != 0)) { if (strncmp(pToken->z, "true", pToken->n) == 0) { - return func(&pCxt->msg, &TRUE_VALUE, pSchema->bytes, param); + pVal->value.val = TRUE_VALUE; } else if (strncmp(pToken->z, "false", pToken->n) == 0) { - return func(&pCxt->msg, &FALSE_VALUE, pSchema->bytes, param); + pVal->value.val = FALSE_VALUE; } else { return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); } } else if (pToken->type == TK_NK_INTEGER) { - return func(&pCxt->msg, ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), - pSchema->bytes, param); + pVal->value.val = ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? FALSE_VALUE : TRUE_VALUE); } else if (pToken->type == TK_NK_FLOAT) { - return func(&pCxt->msg, ((taosStr2Double(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, - param); + pVal->value.val = ((taosStr2Double(pToken->z, NULL) == 0) ? FALSE_VALUE : TRUE_VALUE); } else { return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); } + break; } - case TSDB_DATA_TYPE_TINYINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z); - } else if (!IS_VALID_TINYINT(iv)) { + } else if (!IS_VALID_TINYINT(pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "tinyint data overflow", pToken->z); } - - uint8_t tmpVal = (uint8_t)iv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_UTINYINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z); - } else if (uv > UINT8_MAX) { + } else if (pVal->value.val > UINT8_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned tinyint data overflow", pToken->z); } - uint8_t tmpVal = (uint8_t)uv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_SMALLINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z); - } else if (!IS_VALID_SMALLINT(iv)) { + } else if (!IS_VALID_SMALLINT(pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "smallint data overflow", pToken->z); } - int16_t tmpVal = (int16_t)iv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_USMALLINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z); - } else if (uv > UINT16_MAX) { + } else if (pVal->value.val > UINT16_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned smallint data overflow", pToken->z); } - uint16_t tmpVal = (uint16_t)uv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_INT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z); - } else if (!IS_VALID_INT(iv)) { + } else if (!IS_VALID_INT(pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "int data overflow", pToken->z); } - int32_t tmpVal = (int32_t)iv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_UINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); - } else if (uv > UINT32_MAX) { + } else if (pVal->value.val > UINT32_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned int data overflow", pToken->z); } - uint32_t tmpVal = (uint32_t)uv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_BIGINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z); } - return func(&pCxt->msg, &iv, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_UBIGINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z); } - return func(&pCxt->msg, &uv, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_FLOAT: { + char* endptr = NULL; double dv; if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); @@ -1160,11 +1124,12 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, isnan(dv)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } - float tmpVal = (float)dv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + float f = dv; + memcpy(&pVal->value.val, &f, sizeof(f)); + break; } - case TSDB_DATA_TYPE_DOUBLE: { + char* endptr = NULL; double dv; if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); @@ -1172,49 +1137,76 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); } - return func(&pCxt->msg, &dv, pSchema->bytes, param); + pVal->value.val = *(int64_t*)&dv; + break; } - case TSDB_DATA_TYPE_BINARY: { // Too long values will raise the invalid sql error message if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } - - return func(&pCxt->msg, pToken->z, pToken->n, param); + pVal->value.pData = taosMemoryMalloc(pToken->n); + if (NULL == pVal->value.pData) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(pVal->value.pData, pToken->z, pToken->n); + pVal->value.nData = pToken->n; + break; } - case TSDB_DATA_TYPE_NCHAR: { - return func(&pCxt->msg, pToken->z, pToken->n, param); + // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' + int32_t len = 0; + char* pUcs4 = taosMemoryCalloc(1, pSchema->bytes - VARSTR_HEADER_SIZE); + if (NULL == pUcs4) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)pUcs4, pSchema->bytes - VARSTR_HEADER_SIZE, &len)) { + if (errno == E2BIG) { + return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + } + char buf[512] = {0}; + snprintf(buf, tListLen(buf), "%s", strerror(errno)); + return buildSyntaxErrMsg(&pCxt->msg, buf, pToken->z); + } + pVal->value.pData = pUcs4; + pVal->value.nData = len; + break; } case TSDB_DATA_TYPE_JSON: { if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { return buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", pToken->z); } - return func(&pCxt->msg, pToken->z, pToken->n, param); + pVal->value.pData = taosMemoryMalloc(pToken->n); + if (NULL == pVal->value.pData) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(pVal->value.pData, pToken->z, pToken->n); + pVal->value.nData = pToken->n; + break; } case TSDB_DATA_TYPE_TIMESTAMP: { - int64_t tmpVal; - if (parseTime(pSql, pToken, timePrec, &tmpVal, &pCxt->msg) != TSDB_CODE_SUCCESS) { + if (parseTime(pSql, pToken, timePrec, &pVal->value.val, &pCxt->msg) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp", pToken->z); } - - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } + default: + return TSDB_CODE_FAILED; } - return TSDB_CODE_FAILED; + pVal->flag = CV_FLAG_VALUE; + return TSDB_CODE_SUCCESS; } static int32_t parseValueToken(SInsertParseContext* pCxt, const char** pSql, SToken* pToken, SSchema* pSchema, - int16_t timePrec, _row_append_fn_t func, void* param) { + int16_t timePrec, SColVal* pVal) { int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg); if (TSDB_CODE_SUCCESS == code && isNullValue(pSchema->type, pToken)) { if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { return buildSyntaxErrMsg(&pCxt->msg, "primary timestamp should not be null", pToken->z); } - - return func(&pCxt->msg, NULL, 0, param); + pVal->flag = CV_FLAG_NULL; + return TSDB_CODE_SUCCESS; } if (TSDB_CODE_SUCCESS == code && IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { @@ -1222,26 +1214,34 @@ static int32_t parseValueToken(SInsertParseContext* pCxt, const char** pSql, STo } if (TSDB_CODE_SUCCESS == code) { - code = parseValueTokenImpl(pCxt, pSql, pToken, pSchema, timePrec, func, param); + code = parseValueTokenImpl(pCxt, pSql, pToken, pSchema, timePrec, pVal); } return code; } -static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataBlocks* pDataBuf, bool* pGotRow, +static void clearColValArray(SArray* pCols) { + int32_t num = taosArrayGetSize(pCols); + for (int32_t i = 0; i < num; ++i) { + SColVal* pCol = taosArrayGet(pCols, i); + if (IS_VAR_DATA_TYPE(pCol->type)) { + taosMemoryFreeClear(pCol->value.pData); + } + } +} + +static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataCxt* pTableCxt, bool* pGotRow, SToken* pToken) { - SRowBuilder* pBuilder = &pDataBuf->rowBuilder; - STSRow* row = (STSRow*)(pDataBuf->pData + pDataBuf->size); // skip the SSubmitBlk header - SParsedDataColInfo* pCols = &pDataBuf->boundColumnInfo; - bool isParseBindParam = false; - SSchema* pSchemas = getTableColumnSchema(pDataBuf->pTableMeta); - SMemParam param = {.rb = pBuilder}; - - int32_t code = tdSRowResetBuf(pBuilder, row); + SBoundColInfo* pCols = &pTableCxt->boundColsInfo; + bool isParseBindParam = false; + SSchema* pSchemas = getTableColumnSchema(pTableCxt->pMeta); + + int32_t code = TSDB_CODE_SUCCESS; // 1. set the parsed value from sql string for (int i = 0; i < pCols->numOfBound && TSDB_CODE_SUCCESS == code; ++i) { NEXT_TOKEN_WITH_PREV(*pSql, *pToken); - SSchema* pSchema = &pSchemas[pCols->boundColumns[i]]; + SSchema* pSchema = &pSchemas[pCols->pColIndex[i]]; + SColVal* pVal = taosArrayGet(pTableCxt->pValues, pCols->pColIndex[i]); if (pToken->type == TK_NK_QUESTION) { isParseBindParam = true; @@ -1260,10 +1260,7 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataB } if (TSDB_CODE_SUCCESS == code) { - param.schema = pSchema; - insGetSTSRowAppendInfo(pBuilder->rowType, pCols, i, ¶m.toffset, ¶m.colIdx); - code = parseValueToken(pCxt, pSql, pToken, pSchema, getTableInfo(pDataBuf->pTableMeta).precision, insMemRowAppend, - ¶m); + code = parseValueToken(pCxt, pSql, pToken, pSchema, getTableInfo(pTableCxt->pMeta).precision, pVal); } if (TSDB_CODE_SUCCESS == code && i < pCols->numOfBound - 1) { @@ -1274,65 +1271,28 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataB } } - if (TSDB_CODE_SUCCESS == code) { - TSKEY tsKey = TD_ROW_KEY(row); - code = insCheckTimestamp(pDataBuf, (const char*)&tsKey); - } - if (TSDB_CODE_SUCCESS == code && !isParseBindParam) { - // set the null value for the columns that do not assign values - if ((pCols->numOfBound < pCols->numOfCols) && TD_IS_TP_ROW(row)) { - pBuilder->hasNone = true; + SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + code = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); + if (TSDB_CODE_SUCCESS == code) { + insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow)); } + } - tdSRowEnd(pBuilder); - + if (TSDB_CODE_SUCCESS == code && !isParseBindParam) { *pGotRow = true; - -#ifdef TD_DEBUG_PRINT_ROW - STSchema* pSTSchema = tdGetSTSChemaFromSSChema(schema, spd->numOfCols, 1); - tdSRowPrint(row, pSTSchema, __func__); - taosMemoryFree(pSTSchema); -#endif } - return code; -} + clearColValArray(pTableCxt->pValues); -static int32_t allocateMemIfNeed(STableDataBlocks* pDataBlock, int32_t rowSize, int32_t* numOfRows) { - size_t remain = pDataBlock->nAllocSize - pDataBlock->size; - const int factor = 5; - uint32_t nAllocSizeOld = pDataBlock->nAllocSize; - - // expand the allocated size - if (remain < rowSize * factor) { - while (remain < rowSize * factor) { - pDataBlock->nAllocSize = (uint32_t)(pDataBlock->nAllocSize * 1.5); - remain = pDataBlock->nAllocSize - pDataBlock->size; - } - - char* tmp = taosMemoryRealloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize); - if (tmp != NULL) { - pDataBlock->pData = tmp; - memset(pDataBlock->pData + pDataBlock->size, 0, pDataBlock->nAllocSize - pDataBlock->size); - } else { - // do nothing, if allocate more memory failed - pDataBlock->nAllocSize = nAllocSizeOld; - *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize; - return TSDB_CODE_OUT_OF_MEMORY; - } - } - - *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize; - return TSDB_CODE_SUCCESS; + return code; } // pSql -> (field1_value, ...) [(field1_value2, ...) ...] -static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataBlocks* pDataBuf, - int32_t maxRows, int32_t* pNumOfRows, SToken* pToken) { - int32_t code = insInitRowBuilder(&pDataBuf->rowBuilder, pDataBuf->pTableMeta->sversion, &pDataBuf->boundColumnInfo); +static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, + int32_t* pNumOfRows, SToken* pToken) { + int32_t code = TSDB_CODE_SUCCESS; - int32_t extendedRowSize = insGetExtendedRowSize(pDataBuf); (*pNumOfRows) = 0; while (TSDB_CODE_SUCCESS == code) { int32_t index = 0; @@ -1342,13 +1302,9 @@ static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, } pStmt->pSql += index; - if ((*pNumOfRows) >= maxRows || pDataBuf->size + extendedRowSize >= pDataBuf->nAllocSize) { - code = allocateMemIfNeed(pDataBuf, extendedRowSize, &maxRows); - } - bool gotRow = false; if (TSDB_CODE_SUCCESS == code) { - code = parseOneRow(pCxt, &pStmt->pSql, pDataBuf, &gotRow, pToken); + code = parseOneRow(pCxt, &pStmt->pSql, pTableCxt, &gotRow, pToken); } if (TSDB_CODE_SUCCESS == code) { @@ -1361,7 +1317,6 @@ static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, } if (TSDB_CODE_SUCCESS == code && gotRow) { - pDataBuf->size += extendedRowSize; (*pNumOfRows)++; } } @@ -1374,19 +1329,11 @@ static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, } // VALUES (field1_value, ...) [(field1_value2, ...) ...] -static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataBlocks* pDataBuf, +static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, SToken* pToken) { - int32_t maxNumOfRows = 0; int32_t numOfRows = 0; - int32_t code = allocateMemIfNeed(pDataBuf, insGetExtendedRowSize(pDataBuf), &maxNumOfRows); + int32_t code = parseValues(pCxt, pStmt, pTableCxt, &numOfRows, pToken); if (TSDB_CODE_SUCCESS == code) { - code = parseValues(pCxt, pStmt, pDataBuf, maxNumOfRows, &numOfRows, pToken); - } - if (TSDB_CODE_SUCCESS == code) { - code = insSetBlockInfo((SSubmitBlk*)(pDataBuf->pData), pDataBuf, numOfRows, &pCxt->msg); - } - if (TSDB_CODE_SUCCESS == code) { - pDataBuf->numOfTables = 1; pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_INSERT); @@ -1394,11 +1341,9 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* return code; } -static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataBlocks* pDataBuf, - int maxRows, int32_t* pNumOfRows) { - int32_t code = insInitRowBuilder(&pDataBuf->rowBuilder, pDataBuf->pTableMeta->sversion, &pDataBuf->boundColumnInfo); - - int32_t extendedRowSize = insGetExtendedRowSize(pDataBuf); +static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, + int32_t* pNumOfRows) { + int32_t code = TSDB_CODE_SUCCESS; (*pNumOfRows) = 0; char* pLine = NULL; int64_t readLen = 0; @@ -1414,16 +1359,13 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt continue; } - if ((*pNumOfRows) >= maxRows || pDataBuf->size + extendedRowSize >= pDataBuf->nAllocSize) { - code = allocateMemIfNeed(pDataBuf, extendedRowSize, &maxRows); - } - bool gotRow = false; if (TSDB_CODE_SUCCESS == code) { SToken token; strtolower(pLine, pLine); const char* pRow = pLine; - code = parseOneRow(pCxt, (const char**)&pRow, pDataBuf, &gotRow, &token); + + code = parseOneRow(pCxt, (const char**)&pRow, pTableCxt, &gotRow, &token); if (code && firstLine) { firstLine = false; code = 0; @@ -1432,11 +1374,10 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt } if (TSDB_CODE_SUCCESS == code && gotRow) { - pDataBuf->size += extendedRowSize; (*pNumOfRows)++; } - if (TSDB_CODE_SUCCESS == code && pDataBuf->nAllocSize > tsMaxMemUsedByInsert * 1024 * 1024) { + if (TSDB_CODE_SUCCESS == code && (*pNumOfRows) > tsMaxMemUsedByInsert * 1024 * 1024) { pStmt->fileProcessing = true; break; } @@ -1452,18 +1393,10 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt return code; } -static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataBlocks* pDataBuf) { - int32_t maxNumOfRows = 0; +static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { int32_t numOfRows = 0; - int32_t code = allocateMemIfNeed(pDataBuf, insGetExtendedRowSize(pDataBuf), &maxNumOfRows); + int32_t code = parseCsvFile(pCxt, pStmt, pTableCxt, &numOfRows); if (TSDB_CODE_SUCCESS == code) { - code = parseCsvFile(pCxt, pStmt, pDataBuf, maxNumOfRows, &numOfRows); - } - if (TSDB_CODE_SUCCESS == code) { - code = insSetBlockInfo((SSubmitBlk*)(pDataBuf->pData), pDataBuf, numOfRows, &pCxt->msg); - } - if (TSDB_CODE_SUCCESS == code) { - pDataBuf->numOfTables = 1; pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_FILE_INSERT); @@ -1477,7 +1410,7 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpSt } static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pFilePath, - STableDataBlocks* pDataBuf) { + STableDataCxt* pTableCxt) { char filePathStr[TSDB_FILENAME_LEN] = {0}; if (TK_NK_STRING == pFilePath->type) { trimString(pFilePath->z, pFilePath->n, filePathStr, sizeof(filePathStr)); @@ -1489,10 +1422,10 @@ static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* return TAOS_SYSTEM_ERROR(errno); } - return parseDataFromFileImpl(pCxt, pStmt, pDataBuf); + return parseDataFromFileImpl(pCxt, pStmt, pTableCxt); } -static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataBlocks* pDataBuf, +static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, SToken* pToken) { if (tsUseAdapter) { return buildInvalidOperationMsg(&pCxt->msg, "proxy mode does not support csv loading"); @@ -1502,18 +1435,18 @@ static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS if (0 == pToken->n || (TK_NK_STRING != pToken->type && TK_NK_ID != pToken->type)) { return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", pToken->z); } - return parseDataFromFile(pCxt, pStmt, pToken, pDataBuf); + return parseDataFromFile(pCxt, pStmt, pToken, pTableCxt); } // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path -static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataBlocks* pDataBuf) { +static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { SToken token; NEXT_TOKEN(pStmt->pSql, token); switch (token.type) { case TK_VALUES: - return parseValuesClause(pCxt, pStmt, pDataBuf, &token); + return parseValuesClause(pCxt, pStmt, pTableCxt, &token); case TK_FILE: - return parseFileClause(pCxt, pStmt, pDataBuf, &token); + return parseFileClause(pCxt, pStmt, pTableCxt, &token); default: break; } @@ -1524,18 +1457,19 @@ static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS // 1. [(tag1_name, ...)] ... // 2. VALUES ... | FILE ... static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - STableDataBlocks* pDataBuf = NULL; - int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pDataBuf); + STableDataCxt* pTableCxt = NULL; + int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pTableCxt); if (TSDB_CODE_SUCCESS == code) { - code = parseDataClause(pCxt, pStmt, pDataBuf); + code = parseDataClause(pCxt, pStmt, pTableCxt); } return code; } static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - destroyBoundColumnInfo(&pCxt->tags); + insDestroyBoundColInfo(&pCxt->tags); taosMemoryFreeClear(pStmt->pTableMeta); - tdDestroySVCreateTbReq(&pStmt->createTblReq); + tdDestroySVCreateTbReq(pStmt->pCreateTblReq); + taosMemoryFreeClear(pStmt->pCreateTblReq); pCxt->missCache = false; pCxt->usingDuplicateTable = false; pStmt->pBoundCols = NULL; @@ -1593,7 +1527,7 @@ static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModif } static int32_t setStmtInfo(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); + SBoundColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); if (NULL == tags) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1616,13 +1550,11 @@ static int32_t parseInsertBodyBottom(SInsertParseContext* pCxt, SVnodeModifyOpSt } // merge according to vgId - int32_t code = TSDB_CODE_SUCCESS; - if (taosHashGetSize(pStmt->pTableBlockHashObj) > 0) { - code = insMergeTableDataBlocks(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks); - } + int32_t code = insMergeTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks); if (TSDB_CODE_SUCCESS == code) { - code = insBuildOutput(pStmt->pVgroupsHashObj, pStmt->pVgDataBlocks, &pStmt->pDataBlocks); + code = insBuildVgDataBlocks(pStmt->pVgroupsHashObj, pStmt->pVgDataBlocks, &pStmt->pDataBlocks); } + return code; } @@ -1663,8 +1595,8 @@ static int32_t createVnodeModifOpStmt(SInsertParseContext* pCxt, bool reentry, S TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT); } pStmt->pSql = pCxt->pComCxt->pSql; - pStmt->freeHashFunc = insDestroyBlockHashmap; - pStmt->freeArrayFunc = insDestroyBlockArrayList; + pStmt->freeHashFunc = insDestroyTableDataCxtHashMap; + pStmt->freeArrayFunc = insDestroyVgroupDataCxtList; if (!reentry) { pStmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); @@ -1878,10 +1810,10 @@ static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifyOp } static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - STableDataBlocks* pDataBuf = NULL; - int32_t code = getTableDataBlocks(pCxt, pStmt, &pDataBuf); + STableDataCxt* pTableCxt = NULL; + int32_t code = getTableDataCxt(pCxt, pStmt, &pTableCxt); if (TSDB_CODE_SUCCESS == code) { - code = parseDataFromFileImpl(pCxt, pStmt, pDataBuf); + code = parseDataFromFileImpl(pCxt, pStmt, pTableCxt); } if (TSDB_CODE_SUCCESS == code) { @@ -2005,6 +1937,6 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal QUERY_EXEC_STAGE_SCHEDULE == (*pQuery)->execStage) { code = setRefreshMate(*pQuery); } - destroyBoundColumnInfo(&context.tags); + insDestroyBoundColInfo(&context.tags); return code; } diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index e10b195d34dd3b1be5774294703a2155fc93aa81..01a635e4b2d34bbb7982d45400429ac323e9547e 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -29,26 +29,53 @@ typedef struct SKvParam { char buf[TSDB_MAX_TAGS_LEN]; } SKvParam; +int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData** pData) { + *pData = taosMemoryCalloc(1, sizeof(SSubmitTbData)); + if (NULL == *pData) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SSubmitTbData* pNew = *pData; + + *pNew = *pDataBlock->pData; + + cloneSVreateTbReq(pDataBlock->pData->pCreateTbReq, &pNew->pCreateTbReq); + pNew->aCol = taosArrayDup(pDataBlock->pData->aCol, NULL); + + int32_t colNum = taosArrayGetSize(pNew->aCol); + for (int32_t i = 0; i < colNum; ++i) { + SColData* pCol = (SColData*)taosArrayGet(pNew->aCol, i); + tColDataDeepClear(pCol); + } + + return TSDB_CODE_SUCCESS; +} + int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash) { - int32_t code = TSDB_CODE_SUCCESS; - SArray* pVgDataBlocks = NULL; + int32_t code = TSDB_CODE_SUCCESS; + SArray* pVgDataBlocks = NULL; + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; + // merge according to vgId if (taosHashGetSize(pBlockHash) > 0) { - code = insMergeTableDataBlocks(pBlockHash, &pVgDataBlocks); + code = insMergeTableDataCxt(pBlockHash, &pVgDataBlocks); } if (TSDB_CODE_SUCCESS == code) { - code = insBuildOutput(pVgHash, pVgDataBlocks, &((SVnodeModifyOpStmt*)pQuery->pRoot)->pDataBlocks); + code = insBuildVgDataBlocks(pVgHash, pVgDataBlocks, &pStmt->pDataBlocks); + } + + if (pStmt->freeArrayFunc) { + pStmt->freeArrayFunc(pVgDataBlocks); } - insDestroyBlockArrayList(pVgDataBlocks); return code; } int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; - int32_t code = TSDB_CODE_SUCCESS; - SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + int32_t code = TSDB_CODE_SUCCESS; + SBoundColInfo* tags = (SBoundColInfo*)boundTags; if (NULL == tags) { return TSDB_CODE_APP_ERROR; } @@ -64,7 +91,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch goto end; } - SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); + SSchema* pSchema = getTableTagSchema(pDataBlock->pMeta); bool isJson = false; STag* pTag = NULL; @@ -74,7 +101,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch continue; } - SSchema* pTagSchema = &pSchema[tags->boundColumns[c]]; + SSchema* pTagSchema = &pSchema[tags->pColIndex[c]]; int32_t colLen = pTagSchema->bytes; if (IS_VAR_DATA_TYPE(pTagSchema->type)) { colLen = bind[c].length[0]; @@ -136,11 +163,16 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch goto end; } - SVCreateTbReq tbReq = {0}; - insBuildCreateTbReq(&tbReq, tName, pTag, suid, sTableName, tagName, pDataBlock->pTableMeta->tableInfo.numOfTags, - TSDB_DEFAULT_TABLE_TTL); - code = insBuildCreateTbMsg(pDataBlock, &tbReq); - tdDestroySVCreateTbReq(&tbReq); + if (NULL == pDataBlock->pData->pCreateTbReq) { + pDataBlock->pData->pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == pDataBlock->pData->pCreateTbReq) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + } + + insBuildCreateTbReq(pDataBlock->pData->pCreateTbReq, tName, pTag, suid, sTableName, tagName, + pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); end: for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) { @@ -155,199 +187,178 @@ end: return code; } -int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - int32_t extendedRowSize = insGetExtendedRowSize(pDataBlock); - SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; - SRowBuilder* pBuilder = &pDataBlock->rowBuilder; - SMemParam param = {.rb = pBuilder}; - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; - int32_t rowNum = bind->num; - - CHECK_CODE( - insInitRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo)); +int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND* src, TAOS_MULTI_BIND* dst) { + int32_t output = 0; + int32_t newBuflen = (pSchema->bytes - VARSTR_HEADER_SIZE) * src->num; + if (dst->buffer_length < newBuflen) { + dst->buffer = taosMemoryRealloc(dst->buffer, newBuflen); + if (NULL == dst->buffer) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } - CHECK_CODE(insAllocateMemForSize(pDataBlock, extendedRowSize * bind->num)); + if (NULL == dst->length) { + dst->length = taosMemoryRealloc(dst->length, sizeof(int32_t) * src->num); + if (NULL == dst->buffer) { + taosMemoryFreeClear(dst->buffer); + return TSDB_CODE_OUT_OF_MEMORY; + } + } - for (int32_t r = 0; r < bind->num; ++r) { - STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header - tdSRowResetBuf(pBuilder, row); + dst->buffer_length = pSchema->bytes - VARSTR_HEADER_SIZE; - for (int c = 0; c < spd->numOfBound; ++c) { - SSchema* pColSchema = &pSchema[spd->boundColumns[c]]; + for (int32_t i = 0; i < src->num; ++i) { + if (src->is_null && src->is_null[i]) { + continue; + } - if (bind[c].num != rowNum) { - return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); + if (!taosMbsToUcs4(((char*)src->buffer) + src->buffer_length * i, src->length[i], + (TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output)) { + if (errno == E2BIG) { + return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } + char buf[512] = {0}; + snprintf(buf, tListLen(buf), "%s", strerror(errno)); + return buildSyntaxErrMsg(pMsgBuf, buf, NULL); + } - param.schema = pColSchema; - insGetSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); - - if (bind[c].is_null && bind[c].is_null[r]) { - if (pColSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - return buildInvalidOperationMsg(&pBuf, "primary timestamp should not be NULL"); - } - - CHECK_CODE(insMemRowAppend(&pBuf, NULL, 0, ¶m)); - } else { - if (bind[c].buffer_type != pColSchema->type) { - return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); - } + dst->length[i] = output; + } - int32_t colLen = pColSchema->bytes; - if (IS_VAR_DATA_TYPE(pColSchema->type)) { - colLen = bind[c].length[r]; - } + dst->buffer_type = src->buffer_type; + dst->is_null = src->is_null; + dst->num = src->num; - CHECK_CODE(insMemRowAppend(&pBuf, (char*)bind[c].buffer + bind[c].buffer_length * r, colLen, ¶m)); - } + return TSDB_CODE_SUCCESS; +} - if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { - TSKEY tsKey = TD_ROW_KEY(row); - insCheckTimestamp(pDataBlock, (const char*)&tsKey); - } - } - // set the null value for the columns that do not assign values - if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - pBuilder->hasNone = true; +int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); + SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + int32_t rowNum = bind->num; + TAOS_MULTI_BIND ncharBind = {0}; + TAOS_MULTI_BIND* pBind = NULL; + int32_t code = 0; + + for (int c = 0; c < boundInfo->numOfBound; ++c) { + SSchema* pColSchema = &pSchema[boundInfo->pColIndex[c]]; + SColData* pCol = taosArrayGet(pDataBlock->pData->aCol, c); + + if (bind[c].num != rowNum) { + code = buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); + goto _return; } - tdSRowEnd(pBuilder); -#ifdef TD_DEBUG_PRINT_ROW - STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1); - tdSRowPrint(row, pSTSchema, __func__); - taosMemoryFree(pSTSchema); -#endif - pDataBlock->size += extendedRowSize; - } - SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); - return insSetBlockInfo(pBlocks, pDataBlock, bind->num, &pBuf); -} + if (bind[c].buffer_type != pColSchema->type) { + code = buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); + goto _return; + } -int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, - int32_t rowNum) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - int32_t extendedRowSize = insGetExtendedRowSize(pDataBlock); - SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; - SRowBuilder* pBuilder = &pDataBlock->rowBuilder; - SMemParam param = {.rb = pBuilder}; - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; - bool rowStart = (0 == colIdx); - bool rowEnd = ((colIdx + 1) == spd->numOfBound); - - if (rowStart) { - CHECK_CODE( - insInitRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo)); - CHECK_CODE(insAllocateMemForSize(pDataBlock, extendedRowSize * bind->num)); - } - - for (int32_t r = 0; r < bind->num; ++r) { - STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size + extendedRowSize * r); // skip the SSubmitBlk header - if (rowStart) { - tdSRowResetBuf(pBuilder, row); + if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { + code = convertStmtNcharCol(&pBuf, pColSchema, bind + c, &ncharBind); + if (code) { + goto _return; + } + pBind = &ncharBind; } else { - tdSRowGetBuf(pBuilder, row); + pBind = bind + c; } - SSchema* pColSchema = &pSchema[spd->boundColumns[colIdx]]; + tColDataAddValueByBind(pCol, pBind); + } - if (bind->num != rowNum) { - return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); - } + qDebug("stmt all %d columns bind %d rows data", boundInfo->numOfBound, rowNum); - param.schema = pColSchema; - insGetSTSRowAppendInfo(pBuilder->rowType, spd, colIdx, ¶m.toffset, ¶m.colIdx); +_return: - if (bind->is_null && bind->is_null[r]) { - if (pColSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - return buildInvalidOperationMsg(&pBuf, "primary timestamp should not be NULL"); - } + taosMemoryFree(ncharBind.buffer); + taosMemoryFree(ncharBind.length); - CHECK_CODE(insMemRowAppend(&pBuf, NULL, 0, ¶m)); - } else { - if (bind->buffer_type != pColSchema->type) { - return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); - } - - int32_t colLen = pColSchema->bytes; - if (IS_VAR_DATA_TYPE(pColSchema->type)) { - colLen = bind->length[r]; - } + return code; +} - CHECK_CODE(insMemRowAppend(&pBuf, (char*)bind->buffer + bind->buffer_length * r, colLen, ¶m)); - } +int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, + int32_t rowNum) { + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); + SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + SSchema* pColSchema = &pSchema[boundInfo->pColIndex[colIdx]]; + SColData* pCol = taosArrayGet(pDataBlock->pData->aCol, colIdx); + TAOS_MULTI_BIND ncharBind = {0}; + TAOS_MULTI_BIND* pBind = NULL; + int32_t code = 0; + + if (bind->num != rowNum) { + return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); + } - if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { - TSKEY tsKey = TD_ROW_KEY(row); - insCheckTimestamp(pDataBlock, (const char*)&tsKey); - } + if (bind->buffer_type != pColSchema->type) { + return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); + } - // set the null value for the columns that do not assign values - if (rowEnd && (spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - pBuilder->hasNone = true; - } - if (rowEnd) { - tdSRowEnd(pBuilder); - } -#ifdef TD_DEBUG_PRINT_ROW - if (rowEnd) { - STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1); - tdSRowPrint(row, pSTSchema, __func__); - taosMemoryFree(pSTSchema); + if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { + code = convertStmtNcharCol(&pBuf, pColSchema, bind, &ncharBind); + if (code) { + goto _return; } -#endif + pBind = &ncharBind; + } else { + pBind = bind; } - if (rowEnd) { - pDataBlock->size += extendedRowSize * bind->num; + tColDataAddValueByBind(pCol, pBind); - SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); - CHECK_CODE(insSetBlockInfo(pBlocks, pDataBlock, bind->num, &pBuf)); - } + qDebug("stmt col %d bind %d rows data", colIdx, rowNum); - return TSDB_CODE_SUCCESS; +_return: + + taosMemoryFree(ncharBind.buffer); + taosMemoryFree(ncharBind.length); + + return code; } -int32_t buildBoundFields(SParsedDataColInfo* boundInfo, SSchema* pSchema, int32_t* fieldNum, TAOS_FIELD_E** fields, - uint8_t timePrec) { +int32_t buildBoundFields(int32_t numOfBound, int16_t* boundColumns, SSchema* pSchema, int32_t* fieldNum, + TAOS_FIELD_E** fields, uint8_t timePrec) { if (fields) { - *fields = taosMemoryCalloc(boundInfo->numOfBound, sizeof(TAOS_FIELD)); + *fields = taosMemoryCalloc(numOfBound, sizeof(TAOS_FIELD_E)); if (NULL == *fields) { return TSDB_CODE_OUT_OF_MEMORY; } - SSchema* schema = &pSchema[boundInfo->boundColumns[0]]; + SSchema* schema = &pSchema[boundColumns[0]]; if (TSDB_DATA_TYPE_TIMESTAMP == schema->type) { (*fields)[0].precision = timePrec; } - for (int32_t i = 0; i < boundInfo->numOfBound; ++i) { - schema = &pSchema[boundInfo->boundColumns[i]]; + for (int32_t i = 0; i < numOfBound; ++i) { + schema = &pSchema[boundColumns[i]]; strcpy((*fields)[i].name, schema->name); (*fields)[i].type = schema->type; (*fields)[i].bytes = schema->bytes; } } - *fieldNum = boundInfo->numOfBound; + *fieldNum = numOfBound; return TSDB_CODE_SUCCESS; } int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + SBoundColInfo* tags = (SBoundColInfo*)boundTags; if (NULL == tags) { return TSDB_CODE_APP_ERROR; } - if (pDataBlock->pTableMeta->tableType != TSDB_SUPER_TABLE && pDataBlock->pTableMeta->tableType != TSDB_CHILD_TABLE) { + if (pDataBlock->pMeta->tableType != TSDB_SUPER_TABLE && pDataBlock->pMeta->tableType != TSDB_CHILD_TABLE) { return TSDB_CODE_TSC_STMT_API_ERROR; } - SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); + SSchema* pSchema = getTableTagSchema(pDataBlock->pMeta); if (tags->numOfBound <= 0) { *fieldNum = 0; *fields = NULL; @@ -355,15 +366,15 @@ int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TA return TSDB_CODE_SUCCESS; } - CHECK_CODE(buildBoundFields(tags, pSchema, fieldNum, fields, 0)); + CHECK_CODE(buildBoundFields(tags->numOfBound, tags->pColIndex, pSchema, fieldNum, fields, 0)); return TSDB_CODE_SUCCESS; } int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fields) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - if (pDataBlock->boundColumnInfo.numOfBound <= 0) { + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); + if (pDataBlock->boundColsInfo.numOfBound <= 0) { *fieldNum = 0; if (fields) { *fields = NULL; @@ -372,107 +383,125 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fiel return TSDB_CODE_SUCCESS; } - CHECK_CODE(buildBoundFields(&pDataBlock->boundColumnInfo, pSchema, fieldNum, fields, - pDataBlock->pTableMeta->tableInfo.precision)); + CHECK_CODE(buildBoundFields(pDataBlock->boundColsInfo.numOfBound, pDataBlock->boundColsInfo.pColIndex, pSchema, + fieldNum, fields, pDataBlock->pMeta->tableInfo.precision)); return TSDB_CODE_SUCCESS; } -int32_t qResetStmtDataBlock(void* block, bool keepBuf) { - STableDataBlocks* pBlock = (STableDataBlocks*)block; +int32_t qResetStmtDataBlock(STableDataCxt* block, bool deepClear) { + STableDataCxt* pBlock = (STableDataCxt*)block; + int32_t colNum = taosArrayGetSize(pBlock->pData->aCol); - if (keepBuf) { - taosMemoryFreeClear(pBlock->pData); - pBlock->pData = taosMemoryMalloc(TSDB_PAYLOAD_SIZE); - if (NULL == pBlock->pData) { - return TSDB_CODE_OUT_OF_MEMORY; + for (int32_t i = 0; i < colNum; ++i) { + SColData* pCol = (SColData*)taosArrayGet(pBlock->pData->aCol, i); + if (deepClear) { + tColDataDeepClear(pCol); + } else { + tColDataClear(pCol); } - memset(pBlock->pData, 0, sizeof(SSubmitBlk)); - } else { - pBlock->pData = NULL; } - pBlock->ordered = true; - pBlock->prevTS = INT64_MIN; - pBlock->size = sizeof(SSubmitBlk); - pBlock->tsSource = -1; - pBlock->numOfTables = 1; - pBlock->nAllocSize = TSDB_PAYLOAD_SIZE; - pBlock->headerSize = pBlock->size; - pBlock->createTbReqLen = 0; - - memset(&pBlock->rowBuilder, 0, sizeof(pBlock->rowBuilder)); - return TSDB_CODE_SUCCESS; } -int32_t qCloneStmtDataBlock(void** pDst, void* pSrc) { - *pDst = taosMemoryMalloc(sizeof(STableDataBlocks)); +int32_t qCloneStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, bool reset) { + int32_t code = 0; + + *pDst = taosMemoryCalloc(1, sizeof(STableDataCxt)); if (NULL == *pDst) { return TSDB_CODE_OUT_OF_MEMORY; } - memcpy(*pDst, pSrc, sizeof(STableDataBlocks)); - ((STableDataBlocks*)(*pDst))->cloned = true; + STableDataCxt* pNewCxt = (STableDataCxt*)*pDst; + STableDataCxt* pCxt = (STableDataCxt*)pSrc; + pNewCxt->pSchema = NULL; + pNewCxt->pValues = NULL; - STableDataBlocks* pBlock = (STableDataBlocks*)(*pDst); - if (pBlock->pTableMeta) { - void* pNewMeta = taosMemoryMalloc(TABLE_META_SIZE(pBlock->pTableMeta)); + if (pCxt->pMeta) { + void* pNewMeta = taosMemoryMalloc(TABLE_META_SIZE(pCxt->pMeta)); if (NULL == pNewMeta) { - taosMemoryFreeClear(*pDst); + insDestroyTableDataCxt(*pDst); + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(pNewMeta, pCxt->pMeta, TABLE_META_SIZE(pCxt->pMeta)); + pNewCxt->pMeta = pNewMeta; + } + + memcpy(&pNewCxt->boundColsInfo, &pCxt->boundColsInfo, sizeof(pCxt->boundColsInfo)); + pNewCxt->boundColsInfo.pColIndex = NULL; + + if (pCxt->boundColsInfo.pColIndex) { + void* pNewColIdx = taosMemoryMalloc(pCxt->boundColsInfo.numOfBound * sizeof(*pCxt->boundColsInfo.pColIndex)); + if (NULL == pNewColIdx) { + insDestroyTableDataCxt(*pDst); return TSDB_CODE_OUT_OF_MEMORY; } - memcpy(pNewMeta, pBlock->pTableMeta, TABLE_META_SIZE(pBlock->pTableMeta)); - pBlock->pTableMeta = pNewMeta; + memcpy(pNewColIdx, pCxt->boundColsInfo.pColIndex, + pCxt->boundColsInfo.numOfBound * sizeof(*pCxt->boundColsInfo.pColIndex)); + pNewCxt->boundColsInfo.pColIndex = pNewColIdx; + } + + if (pCxt->pData) { + SSubmitTbData* pNewTb = (SSubmitTbData*)taosMemoryMalloc(sizeof(SSubmitTbData)); + if (NULL == pNewTb) { + insDestroyTableDataCxt(*pDst); + return TSDB_CODE_OUT_OF_MEMORY; + } + + memcpy(pNewTb, pCxt->pData, sizeof(*pCxt->pData)); + pNewTb->pCreateTbReq = NULL; + + pNewTb->aCol = taosArrayDup(pCxt->pData->aCol, NULL); + if (NULL == pNewTb) { + insDestroyTableDataCxt(*pDst); + return TSDB_CODE_OUT_OF_MEMORY; + } + + pNewCxt->pData = pNewTb; + + if (reset) { + code = qResetStmtDataBlock(*pDst, true); + } } - return qResetStmtDataBlock(*pDst, false); + return code; } -int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgId) { - int32_t code = qCloneStmtDataBlock(pDst, pSrc); +int32_t qRebuildStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_t uid, uint64_t suid, int32_t vgId, + bool rebuildCreateTb) { + int32_t code = qCloneStmtDataBlock(pDst, pSrc, false); if (code) { return code; } - STableDataBlocks* pBlock = (STableDataBlocks*)*pDst; - pBlock->pData = taosMemoryMalloc(pBlock->nAllocSize); - if (NULL == pBlock->pData) { - qFreeStmtDataBlock(pBlock); - return TSDB_CODE_OUT_OF_MEMORY; + STableDataCxt* pBlock = (STableDataCxt*)*pDst; + if (pBlock->pMeta) { + pBlock->pMeta->uid = uid; + pBlock->pMeta->vgId = vgId; + pBlock->pMeta->suid = suid; } - pBlock->vgId = vgId; + pBlock->pData->suid = suid; + pBlock->pData->uid = uid; - if (pBlock->pTableMeta) { - pBlock->pTableMeta->uid = uid; - pBlock->pTableMeta->vgId = vgId; + if (rebuildCreateTb && NULL == pBlock->pData->pCreateTbReq) { + pBlock->pData->pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == pBlock->pData->pCreateTbReq) { + return TSDB_CODE_OUT_OF_MEMORY; + } } - memset(pBlock->pData, 0, sizeof(SSubmitBlk)); - return TSDB_CODE_SUCCESS; } -STableMeta* qGetTableMetaInDataBlock(void* pDataBlock) { return ((STableDataBlocks*)pDataBlock)->pTableMeta; } +STableMeta* qGetTableMetaInDataBlock(STableDataCxt* pDataBlock) { return ((STableDataCxt*)pDataBlock)->pMeta; } -void qFreeStmtDataBlock(void* pDataBlock) { - if (pDataBlock == NULL) { - return; - } - - taosMemoryFreeClear(((STableDataBlocks*)pDataBlock)->pTableMeta); - taosMemoryFreeClear(((STableDataBlocks*)pDataBlock)->pData); - taosMemoryFreeClear(pDataBlock); -} - -void qDestroyStmtDataBlock(void* pBlock) { +void qDestroyStmtDataBlock(STableDataCxt* pBlock) { if (pBlock == NULL) { return; } - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - - pDataBlock->cloned = false; - insDestroyDataBlock(pDataBlock); + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + insDestroyTableDataCxt(pDataBlock); } diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 73cedfeb3d4109622062477214cd148bdd441391..b3018d63b4cf7606aa429dd8a71f07025cf4186f 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -20,938 +20,675 @@ #include "parUtil.h" #include "querynodes.h" #include "tRealloc.h" +#include "tdatablock.h" -typedef struct SBlockKeyTuple { - TSKEY skey; - void* payloadAddr; - int16_t index; -} SBlockKeyTuple; - -typedef struct SBlockKeyInfo { - int32_t maxBytesAlloc; - SBlockKeyTuple* pKeyTuple; -} SBlockKeyInfo; +void qDestroyBoundColInfo(void* pInfo) { + if (NULL == pInfo) { + return; + } -typedef struct { - int32_t index; - SArray* rowArray; // array of merged rows(mem allocated by tRealloc/free by tFree) - STSchema* pSchema; - int64_t tbUid; // suid for child table, uid for normal table -} SBlockRowMerger; + SBoundColInfo* pBoundInfo = (SBoundColInfo*)pInfo; -static FORCE_INLINE void tdResetSBlockRowMerger(SBlockRowMerger* pMerger) { - if (pMerger) { - pMerger->index = -1; - } + taosMemoryFreeClear(pBoundInfo->pColIndex); } -static void tdFreeSBlockRowMerger(SBlockRowMerger* pMerger) { - if (pMerger) { - int32_t size = taosArrayGetSize(pMerger->rowArray); - for (int32_t i = 0; i < size; ++i) { - tFree(*(void**)taosArrayGet(pMerger->rowArray, i)); +static char* tableNameGetPosition(SToken* pToken, char target) { + bool inEscape = false; + bool inQuote = false; + char quotaStr = 0; + + for (uint32_t i = 0; i < pToken->n; ++i) { + if (*(pToken->z + i) == target && (!inEscape) && (!inQuote)) { + return pToken->z + i; } - taosArrayDestroy(pMerger->rowArray); - taosMemoryFreeClear(pMerger->pSchema); - taosMemoryFree(pMerger); - } -} + if (*(pToken->z + i) == TS_ESCAPE_CHAR) { + if (!inQuote) { + inEscape = !inEscape; + } + } -static int32_t rowDataCompar(const void* lhs, const void* rhs) { - TSKEY left = *(TSKEY*)lhs; - TSKEY right = *(TSKEY*)rhs; - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; + if (*(pToken->z + i) == '\'' || *(pToken->z + i) == '"') { + if (!inEscape) { + if (!inQuote) { + quotaStr = *(pToken->z + i); + inQuote = !inQuote; + } else if (quotaStr == *(pToken->z + i)) { + inQuote = !inQuote; + } + } + } } -} -static int32_t rowDataComparStable(const void* lhs, const void* rhs) { - TSKEY left = *(TSKEY*)lhs; - TSKEY right = *(TSKEY*)rhs; - if (left == right) { - return ((SBlockKeyTuple*)lhs)->index - ((SBlockKeyTuple*)rhs)->index; - } else { - return left > right ? 1 : -1; - } + return NULL; } -int32_t insGetExtendedRowSize(STableDataBlocks* pBlock) { - STableComInfo* pTableInfo = &pBlock->pTableMeta->tableInfo; - ASSERT(pBlock->rowSize == pTableInfo->rowSize); - return pBlock->rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + pBlock->boundColumnInfo.extendedVarLen + - (int32_t)TD_BITMAP_BYTES(pTableInfo->numOfColumns - 1); -} +int32_t insCreateSName(SName* pName, SToken* pTableName, int32_t acctId, const char* dbName, SMsgBuf* pMsgBuf) { + const char* msg1 = "name too long"; + const char* msg2 = "invalid database name"; + const char* msg3 = "db is not specified"; + const char* msg4 = "invalid table name"; -void insGetSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo* spd, col_id_t idx, int32_t* toffset, - col_id_t* colIdx) { - col_id_t schemaIdx = 0; - if (IS_DATA_COL_ORDERED(spd)) { - schemaIdx = spd->boundColumns[idx]; - if (TD_IS_TP_ROW_T(rowType)) { - *toffset = (spd->cols + schemaIdx)->toffset; // the offset of firstPart - *colIdx = schemaIdx; - } else { - *toffset = idx * sizeof(SKvRowIdx); // the offset of SKvRowIdx - *colIdx = idx; + int32_t code = TSDB_CODE_SUCCESS; + char* p = tableNameGetPosition(pTableName, TS_PATH_DELIMITER[0]); + + if (p != NULL) { // db has been specified in sql string so we ignore current db path + assert(*p == TS_PATH_DELIMITER[0]); + + int32_t dbLen = p - pTableName->z; + if (dbLen <= 0) { + return buildInvalidOperationMsg(pMsgBuf, msg2); } - } else { - ASSERT(idx == (spd->colIdxInfo + idx)->boundIdx); - schemaIdx = (spd->colIdxInfo + idx)->schemaColIdx; - if (TD_IS_TP_ROW_T(rowType)) { - *toffset = (spd->cols + schemaIdx)->toffset; - *colIdx = schemaIdx; - } else { - *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SKvRowIdx); - *colIdx = (spd->colIdxInfo + idx)->finalIdx; + char name[TSDB_DB_FNAME_LEN] = {0}; + strncpy(name, pTableName->z, dbLen); + int32_t actualDbLen = strdequote(name); + + code = tNameSetDbName(pName, acctId, name, actualDbLen); + if (code != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + int32_t tbLen = pTableName->n - dbLen - 1; + if (tbLen <= 0) { + return buildInvalidOperationMsg(pMsgBuf, msg4); } - } -} -int32_t insSetBlockInfo(SSubmitBlk* pBlocks, STableDataBlocks* dataBuf, int32_t numOfRows, SMsgBuf* pMsg) { - pBlocks->suid = (TSDB_NORMAL_TABLE == dataBuf->pTableMeta->tableType ? 0 : dataBuf->pTableMeta->suid); - pBlocks->uid = dataBuf->pTableMeta->uid; - pBlocks->sversion = dataBuf->pTableMeta->sversion; - pBlocks->schemaLen = dataBuf->createTbReqLen; + char tbname[TSDB_TABLE_FNAME_LEN] = {0}; + strncpy(tbname, p + 1, tbLen); + /*tbLen = */ strdequote(tbname); - if (pBlocks->numOfRows + numOfRows >= INT32_MAX) { - return buildInvalidOperationMsg(pMsg, "too many rows in sql, total number of rows should be less than INT32_MAX"); - } - pBlocks->numOfRows += numOfRows; - return TSDB_CODE_SUCCESS; -} + code = tNameFromString(pName, tbname, T_NAME_TABLE); + if (code != 0) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + } else { // get current DB name first, and then set it into path + if (pTableName->n >= TSDB_TABLE_NAME_LEN) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + assert(pTableName->n < TSDB_TABLE_FNAME_LEN); -void insSetBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, col_id_t numOfCols) { - pColList->numOfCols = numOfCols; - pColList->numOfBound = numOfCols; - pColList->orderStatus = ORDER_STATUS_ORDERED; // default is ORDERED for non-bound mode - pColList->boundColumns = taosMemoryCalloc(pColList->numOfCols, sizeof(col_id_t)); - pColList->cols = taosMemoryCalloc(pColList->numOfCols, sizeof(SBoundColumn)); - pColList->colIdxInfo = NULL; - pColList->flen = 0; - pColList->allNullLen = 0; - - int32_t nVar = 0; - for (int32_t i = 0; i < pColList->numOfCols; ++i) { - uint8_t type = pSchema[i].type; - if (i > 0) { - pColList->cols[i].offset = pColList->cols[i - 1].offset + pSchema[i - 1].bytes; - pColList->cols[i].toffset = pColList->flen; - pColList->flen += TYPE_BYTES[type]; + char name[TSDB_TABLE_FNAME_LEN] = {0}; + strncpy(name, pTableName->z, pTableName->n); + strdequote(name); + + if (dbName == NULL) { + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + + code = tNameSetDbName(pName, acctId, dbName, strlen(dbName)); + if (code != TSDB_CODE_SUCCESS) { + code = buildInvalidOperationMsg(pMsgBuf, msg2); + return code; } - switch (type) { - case TSDB_DATA_TYPE_BINARY: - pColList->allNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES); - ++nVar; - break; - case TSDB_DATA_TYPE_NCHAR: - pColList->allNullLen += (VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); - ++nVar; - break; - default: - break; + + code = tNameFromString(pName, name, T_NAME_TABLE); + if (code != 0) { + code = buildInvalidOperationMsg(pMsgBuf, msg1); } - pColList->boundColumns[i] = i; } - pColList->allNullLen += pColList->flen; - pColList->boundNullLen = pColList->allNullLen; // default set allNullLen - pColList->extendedVarLen = (uint16_t)(nVar * sizeof(VarDataOffsetT)); -} -int32_t insSchemaIdxCompar(const void* lhs, const void* rhs) { - uint16_t left = *(uint16_t*)lhs; - uint16_t right = *(uint16_t*)rhs; + if (NULL != strchr(pName->tname, '.')) { + code = generateSyntaxErrMsgExt(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, "The table name cannot contain '.'"); + } - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; + return code; +} + +int16_t insFindCol(SToken* pColname, int16_t start, int16_t end, SSchema* pSchema) { + while (start < end) { + if (strlen(pSchema[start].name) == pColname->n && strncmp(pColname->z, pSchema[start].name, pColname->n) == 0) { + return start; + } + ++start; } + return -1; } -int32_t insBoundIdxCompar(const void* lhs, const void* rhs) { - uint16_t left = *(uint16_t*)POINTER_SHIFT(lhs, sizeof(uint16_t)); - uint16_t right = *(uint16_t*)POINTER_SHIFT(rhs, sizeof(uint16_t)); +void insBuildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, int64_t suid, const char* sname, + SArray* tagName, uint8_t tagNum, int32_t ttl) { + pTbReq->type = TD_CHILD_TABLE; + pTbReq->name = strdup(tname); + pTbReq->ctb.suid = suid; + pTbReq->ctb.tagNum = tagNum; + if (sname) pTbReq->ctb.stbName = strdup(sname); + pTbReq->ctb.pTag = (uint8_t*)pTag; + pTbReq->ctb.tagName = taosArrayDup(tagName, NULL); + pTbReq->ttl = ttl; + pTbReq->commentLen = -1; + + return; +} - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; +static void initBoundCols(int32_t ncols, int16_t* pBoundCols) { + for (int32_t i = 0; i < ncols; ++i) { + pBoundCols[i] = i; } } -void destroyBoundColumnInfo(void* pBoundInfo) { - if (NULL == pBoundInfo) { - return; +static void initColValues(STableMeta* pTableMeta, SArray* pValues) { + SSchema* pSchemas = getTableColumnSchema(pTableMeta); + for (int32_t i = 0; i < pTableMeta->tableInfo.numOfColumns; ++i) { + SColVal val = COL_VAL_NONE(pSchemas[i].colId, pSchemas[i].type); + taosArrayPush(pValues, &val); } - - SParsedDataColInfo* pColList = (SParsedDataColInfo*)pBoundInfo; - - taosMemoryFreeClear(pColList->boundColumns); - taosMemoryFreeClear(pColList->cols); - taosMemoryFreeClear(pColList->colIdxInfo); } -static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, STableMeta* pTableMeta, - STableDataBlocks** dataBlocks) { - STableDataBlocks* dataBuf = (STableDataBlocks*)taosMemoryCalloc(1, sizeof(STableDataBlocks)); - if (dataBuf == NULL) { +int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) { + pInfo->numOfCols = numOfBound; + pInfo->numOfBound = numOfBound; + pInfo->pColIndex = taosMemoryCalloc(numOfBound, sizeof(int16_t)); + if (NULL == pInfo->pColIndex) { return TSDB_CODE_OUT_OF_MEMORY; } + initBoundCols(numOfBound, pInfo->pColIndex); + return TSDB_CODE_SUCCESS; +} - dataBuf->nAllocSize = (uint32_t)defaultSize; - dataBuf->headerSize = startOffset; +void insCheckTableDataOrder(STableDataCxt* pTableCxt, TSKEY tsKey) { + // once the data block is disordered, we do NOT keep last timestamp any more + if (!pTableCxt->ordered) { + return; + } - // the header size will always be the startOffset value, reserved for the subumit block header - if (dataBuf->nAllocSize <= dataBuf->headerSize) { - dataBuf->nAllocSize = dataBuf->headerSize * 2; + if (tsKey < pTableCxt->lastTs) { + pTableCxt->ordered = false; } - dataBuf->pData = taosMemoryMalloc(dataBuf->nAllocSize); - if (dataBuf->pData == NULL) { - taosMemoryFreeClear(dataBuf); - return TSDB_CODE_OUT_OF_MEMORY; + if (tsKey == pTableCxt->lastTs) { + pTableCxt->duplicateTs = true; } - memset(dataBuf->pData, 0, sizeof(SSubmitBlk)); - dataBuf->pTableMeta = tableMetaDup(pTableMeta); + pTableCxt->lastTs = tsKey; + return; +} - SParsedDataColInfo* pColInfo = &dataBuf->boundColumnInfo; - SSchema* pSchema = getTableColumnSchema(dataBuf->pTableMeta); - insSetBoundColumnInfo(pColInfo, pSchema, dataBuf->pTableMeta->tableInfo.numOfColumns); +void insDestroyBoundColInfo(SBoundColInfo* pInfo) { taosMemoryFreeClear(pInfo->pColIndex); } - dataBuf->ordered = true; - dataBuf->prevTS = INT64_MIN; - dataBuf->rowSize = rowSize; - dataBuf->size = startOffset; - dataBuf->vgId = dataBuf->pTableMeta->vgId; +static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreateTbReq, STableDataCxt** pOutput, + bool colMode) { + STableDataCxt* pTableCxt = taosMemoryCalloc(1, sizeof(STableDataCxt)); + if (NULL == pTableCxt) { + return TSDB_CODE_OUT_OF_MEMORY; + } - assert(defaultSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); + int32_t code = TSDB_CODE_SUCCESS; - *dataBlocks = dataBuf; - return TSDB_CODE_SUCCESS; -} + pTableCxt->lastTs = 0; + pTableCxt->ordered = true; + pTableCxt->duplicateTs = false; -int32_t insBuildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) { - SEncoder coder = {0}; - char* pBuf; - int32_t len; - - int32_t ret = 0; - tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len, ret); - if (pBlocks->nAllocSize - pBlocks->size < len) { - pBlocks->nAllocSize += len + pBlocks->rowSize; - char* pTmp = taosMemoryRealloc(pBlocks->pData, pBlocks->nAllocSize); - if (pTmp != NULL) { - pBlocks->pData = pTmp; - memset(pBlocks->pData + pBlocks->size, 0, pBlocks->nAllocSize - pBlocks->size); + pTableCxt->pMeta = tableMetaDup(pTableMeta); + if (NULL == pTableCxt->pMeta) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + if (TSDB_CODE_SUCCESS == code) { + pTableCxt->pSchema = + tBuildTSchema(getTableColumnSchema(pTableMeta), pTableMeta->tableInfo.numOfColumns, pTableMeta->sversion); + if (NULL == pTableCxt->pSchema) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + if (TSDB_CODE_SUCCESS == code) { + code = insInitBoundColsInfo(pTableMeta->tableInfo.numOfColumns, &pTableCxt->boundColsInfo); + } + if (TSDB_CODE_SUCCESS == code) { + pTableCxt->pValues = taosArrayInit(pTableMeta->tableInfo.numOfColumns, sizeof(SColVal)); + if (NULL == pTableCxt->pValues) { + code = TSDB_CODE_OUT_OF_MEMORY; } else { - pBlocks->nAllocSize -= len + pBlocks->rowSize; - return TSDB_CODE_OUT_OF_MEMORY; + initColValues(pTableMeta, pTableCxt->pValues); + } + } + if (TSDB_CODE_SUCCESS == code) { + pTableCxt->pData = taosMemoryCalloc(1, sizeof(SSubmitTbData)); + if (NULL == pTableCxt->pData) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pTableCxt->pData->flags = NULL != *pCreateTbReq ? SUBMIT_REQ_AUTO_CREATE_TABLE : 0; + pTableCxt->pData->flags |= colMode ? SUBMIT_REQ_COLUMN_DATA_FORMAT : 0; + pTableCxt->pData->suid = pTableMeta->suid; + pTableCxt->pData->uid = pTableMeta->uid; + pTableCxt->pData->sver = pTableMeta->sversion; + pTableCxt->pData->pCreateTbReq = *pCreateTbReq; + *pCreateTbReq = NULL; + if (pTableCxt->pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + pTableCxt->pData->aCol = taosArrayInit(128, sizeof(SColData)); + if (NULL == pTableCxt->pData->aCol) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } else { + pTableCxt->pData->aRowP = taosArrayInit(128, POINTER_BYTES); + if (NULL == pTableCxt->pData->aRowP) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } } } - pBuf = pBlocks->pData + pBlocks->size; - - tEncoderInit(&coder, pBuf, len); - int32_t code = tEncodeSVCreateTbReq(&coder, pCreateTbReq); - tEncoderClear(&coder); - pBlocks->size += len; - pBlocks->createTbReqLen = len; + if (TSDB_CODE_SUCCESS == code) { + *pOutput = pTableCxt; + qDebug("tableDataCxt created, uid:%" PRId64 ", vgId:%d", pTableMeta->uid, pTableMeta->vgId); + } else { + taosMemoryFree(pTableCxt); + } return code; } -void insDestroyDataBlock(STableDataBlocks* pDataBlock) { - if (pDataBlock == NULL) { - return; +static void resetColValues(SArray* pValues) { + int32_t num = taosArrayGetSize(pValues); + for (int32_t i = 0; i < num; ++i) { + SColVal* pVal = taosArrayGet(pValues, i); + pVal->flag = CV_FLAG_NONE; } +} - taosMemoryFreeClear(pDataBlock->pData); - taosMemoryFreeClear(pDataBlock->pTableMeta); - destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); - taosMemoryFreeClear(pDataBlock); +int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta* pTableMeta, + SVCreateTbReq** pCreateTbReq, STableDataCxt** pTableCxt, bool colMode) { + STableDataCxt** tmp = (STableDataCxt**)taosHashGet(pHash, id, idLen); + if (NULL != tmp) { + *pTableCxt = *tmp; + resetColValues((*pTableCxt)->pValues); + return TSDB_CODE_SUCCESS; + } + int32_t code = createTableDataCxt(pTableMeta, pCreateTbReq, pTableCxt, colMode); + if (TSDB_CODE_SUCCESS == code) { + code = taosHashPut(pHash, id, idLen, pTableCxt, POINTER_BYTES); + } + return code; } -int32_t insGetDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, int32_t size, int32_t startOffset, - int32_t rowSize, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, - SArray* pBlockList, SVCreateTbReq* pCreateTbReq) { - *dataBlocks = NULL; - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)id, idLen); - if (t1 != NULL) { - *dataBlocks = *t1; +static void destroyColVal(void* p) { + SColVal* pVal = p; + if (TSDB_DATA_TYPE_NCHAR == pVal->type) { + taosMemoryFree(pVal->value.pData); } +} - if (*dataBlocks == NULL) { - int32_t ret = createDataBlock((size_t)size, rowSize, startOffset, pTableMeta, dataBlocks); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } +void insDestroyTableDataCxt(STableDataCxt* pTableCxt) { + if (NULL == pTableCxt) { + return; + } - if (NULL != pCreateTbReq && NULL != pCreateTbReq->ctb.pTag) { - ret = insBuildCreateTbMsg(*dataBlocks, pCreateTbReq); - if (ret != TSDB_CODE_SUCCESS) { - insDestroyDataBlock(*dataBlocks); - return ret; - } - } + taosMemoryFreeClear(pTableCxt->pMeta); + tDestroyTSchema(pTableCxt->pSchema); + insDestroyBoundColInfo(&pTableCxt->boundColsInfo); + taosArrayDestroyEx(pTableCxt->pValues, destroyColVal); + if (pTableCxt->pData) { + tDestroySSubmitTbData(pTableCxt->pData, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pTableCxt->pData); + } + taosMemoryFree(pTableCxt); +} - // converting to 'const char*' is to handle coverity scan errors - taosHashPut(pHashList, (const char*)id, idLen, (const char*)dataBlocks, POINTER_BYTES); - if (pBlockList) { - taosArrayPush(pBlockList, dataBlocks); - } +void insDestroyVgroupDataCxt(SVgroupDataCxt* pVgCxt) { + if (NULL == pVgCxt) { + return; } - return TSDB_CODE_SUCCESS; + tDestroySSubmitReq2(pVgCxt->pData, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pVgCxt->pData); + taosMemoryFree(pVgCxt); } -void insDestroyBlockArrayList(SArray* pDataBlockList) { - if (pDataBlockList == NULL) { +void insDestroyVgroupDataCxtList(SArray* pVgCxtList) { + if (NULL == pVgCxtList) { return; } - size_t size = taosArrayGetSize(pDataBlockList); + size_t size = taosArrayGetSize(pVgCxtList); for (int32_t i = 0; i < size; i++) { - void* p = taosArrayGetP(pDataBlockList, i); - insDestroyDataBlock(p); + void* p = taosArrayGetP(pVgCxtList, i); + insDestroyVgroupDataCxt(p); } - taosArrayDestroy(pDataBlockList); + taosArrayDestroy(pVgCxtList); } -void insDestroyBlockHashmap(SHashObj* pDataBlockHash) { - if (pDataBlockHash == NULL) { +void insDestroyVgroupDataCxtHashMap(SHashObj* pVgCxtHash) { + if (NULL == pVgCxtHash) { return; } - void** p1 = taosHashIterate(pDataBlockHash, NULL); - while (p1) { - STableDataBlocks* pBlocks = *p1; - insDestroyDataBlock(pBlocks); + void** p = taosHashIterate(pVgCxtHash, NULL); + while (p) { + insDestroyVgroupDataCxt(*(SVgroupDataCxt**)p); - p1 = taosHashIterate(pDataBlockHash, p1); + p = taosHashIterate(pVgCxtHash, p); } - taosHashCleanup(pDataBlockHash); + taosHashCleanup(pVgCxtHash); } -// data block is disordered, sort it in ascending order -static int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo) { - SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData; - int16_t nRows = pBlocks->numOfRows; - - // size is less than the total size, since duplicated rows may be removed yet. - - // allocate memory - size_t nAlloc = nRows * sizeof(SBlockKeyTuple); - if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { - char* tmp = taosMemoryRealloc(pBlkKeyInfo->pKeyTuple, nAlloc); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple*)tmp; - pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc; +void insDestroyTableDataCxtHashMap(SHashObj* pTableCxtHash) { + if (NULL == pTableCxtHash) { + return; } - memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc); - int32_t extendedRowSize = insGetExtendedRowSize(dataBuf); - SBlockKeyTuple* pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - char* pBlockData = pBlocks->data + pBlocks->schemaLen; - int n = 0; - while (n < nRows) { - pBlkKeyTuple->skey = TD_ROW_KEY((STSRow*)pBlockData); - pBlkKeyTuple->payloadAddr = pBlockData; - pBlkKeyTuple->index = n; + void** p = taosHashIterate(pTableCxtHash, NULL); + while (p) { + insDestroyTableDataCxt(*(STableDataCxt**)p); - // next loop - pBlockData += extendedRowSize; - ++pBlkKeyTuple; - ++n; + p = taosHashIterate(pTableCxtHash, p); } - if (!dataBuf->ordered) { - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - - // todo. qsort is unstable, if timestamp is same, should get the last one - taosSort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataComparStable); - - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - int32_t i = 0; - int32_t j = 1; - while (j < nRows) { - TSKEY ti = (pBlkKeyTuple + i)->skey; - TSKEY tj = (pBlkKeyTuple + j)->skey; - - if (ti == tj) { - ++j; - continue; - } + taosHashCleanup(pTableCxtHash); +} - int32_t nextPos = (++i); - if (nextPos != j) { - memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + j, sizeof(SBlockKeyTuple)); - } - ++j; +static int32_t fillVgroupDataCxt(STableDataCxt* pTableCxt, SVgroupDataCxt* pVgCxt) { + if (NULL == pVgCxt->pData->aSubmitTbData) { + pVgCxt->pData->aSubmitTbData = taosArrayInit(128, sizeof(SSubmitTbData)); + if (NULL == pVgCxt->pData->aSubmitTbData) { + return TSDB_CODE_OUT_OF_MEMORY; } - - dataBuf->ordered = true; - pBlocks->numOfRows = i + 1; } + taosArrayPush(pVgCxt->pData->aSubmitTbData, pTableCxt->pData); + taosMemoryFreeClear(pTableCxt->pData); - dataBuf->size = sizeof(SSubmitBlk) + pBlocks->numOfRows * extendedRowSize; - dataBuf->prevTS = INT64_MIN; + qDebug("add tableDataCxt uid:%" PRId64 " to vgId:%d", pTableCxt->pMeta->uid, pVgCxt->vgId); - return 0; -} - -static void* tdGetCurRowFromBlockMerger(SBlockRowMerger* pBlkRowMerger) { - if (pBlkRowMerger && (pBlkRowMerger->index >= 0)) { - ASSERT(pBlkRowMerger->index < taosArrayGetSize(pBlkRowMerger->rowArray)); - return *(void**)taosArrayGet(pBlkRowMerger->rowArray, pBlkRowMerger->index); - } - return NULL; + return TSDB_CODE_SUCCESS; } -static int32_t tdBlockRowMerge(STableMeta* pTableMeta, SBlockKeyTuple* pEndKeyTp, int32_t nDupRows, - SBlockRowMerger** pBlkRowMerger, int32_t rowSize) { - ASSERT(nDupRows > 1); - SBlockKeyTuple* pStartKeyTp = pEndKeyTp - (nDupRows - 1); - ASSERT(pStartKeyTp->skey == pEndKeyTp->skey); - - // TODO: optimization if end row is all normal -#if 0 - STSRow* pEndRow = (STSRow*)pEndKeyTp->payloadAddr; - if(isNormal(pEndRow)) { // set the end row if it is normal and return directly - pStartKeyTp->payloadAddr = pEndKeyTp->payloadAddr; - return TSDB_CODE_SUCCESS; - } -#endif - - if (!(*pBlkRowMerger)) { - (*pBlkRowMerger) = taosMemoryCalloc(1, sizeof(**pBlkRowMerger)); - if (!(*pBlkRowMerger)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - (*pBlkRowMerger)->index = -1; - if (!(*pBlkRowMerger)->rowArray) { - (*pBlkRowMerger)->rowArray = taosArrayInit(1, sizeof(void*)); - if (!(*pBlkRowMerger)->rowArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - } - } - - if ((*pBlkRowMerger)->pSchema) { - if ((*pBlkRowMerger)->pSchema->version != pTableMeta->sversion) { - taosMemoryFreeClear((*pBlkRowMerger)->pSchema); - } else { - if ((*pBlkRowMerger)->tbUid != (pTableMeta->suid > 0 ? pTableMeta->suid : pTableMeta->uid)) { - taosMemoryFreeClear((*pBlkRowMerger)->pSchema); - } - } +static int32_t createVgroupDataCxt(STableDataCxt* pTableCxt, SHashObj* pVgroupHash, SArray* pVgroupList, + SVgroupDataCxt** pOutput) { + SVgroupDataCxt* pVgCxt = taosMemoryCalloc(1, sizeof(SVgroupDataCxt)); + if (NULL == pVgCxt) { + return TSDB_CODE_OUT_OF_MEMORY; } - - if (!(*pBlkRowMerger)->pSchema) { - (*pBlkRowMerger)->pSchema = - tdGetSTSChemaFromSSChema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->sversion); - - if (!(*pBlkRowMerger)->pSchema) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - (*pBlkRowMerger)->tbUid = pTableMeta->suid > 0 ? pTableMeta->suid : pTableMeta->uid; + pVgCxt->pData = taosMemoryCalloc(1, sizeof(SSubmitReq2)); + if (NULL == pVgCxt->pData) { + insDestroyVgroupDataCxt(pVgCxt); + return TSDB_CODE_OUT_OF_MEMORY; } - void* pDestRow = NULL; - ++((*pBlkRowMerger)->index); - if ((*pBlkRowMerger)->index < taosArrayGetSize((*pBlkRowMerger)->rowArray)) { - void** pAlloc = (void**)taosArrayGet((*pBlkRowMerger)->rowArray, (*pBlkRowMerger)->index); - if (tRealloc((uint8_t**)pAlloc, rowSize) != 0) { - return TSDB_CODE_FAILED; - } - pDestRow = *pAlloc; + pVgCxt->vgId = pTableCxt->pMeta->vgId; + int32_t code = taosHashPut(pVgroupHash, &pVgCxt->vgId, sizeof(pVgCxt->vgId), &pVgCxt, POINTER_BYTES); + if (TSDB_CODE_SUCCESS == code) { + taosArrayPush(pVgroupList, &pVgCxt); + *pOutput = pVgCxt; } else { - if (tRealloc((uint8_t**)&pDestRow, rowSize) != 0) { - return TSDB_CODE_FAILED; - } - taosArrayPush((*pBlkRowMerger)->rowArray, &pDestRow); - } - - // merge rows to pDestRow - STSchema* pSchema = (*pBlkRowMerger)->pSchema; - SArray* pArray = taosArrayInit(pSchema->numOfCols, sizeof(SColVal)); - for (int32_t i = 0; i < pSchema->numOfCols; ++i) { - SColVal colVal = {0}; - for (int32_t j = 0; j < nDupRows; ++j) { - tTSRowGetVal((pEndKeyTp - j)->payloadAddr, pSchema, i, &colVal); - if (!COL_VAL_IS_NONE(&colVal)) { - break; - } - } - taosArrayPush(pArray, &colVal); - } - if (tdSTSRowNew(pArray, pSchema, (STSRow**)&pDestRow) < 0) { - taosArrayDestroy(pArray); - return TSDB_CODE_FAILED; + insDestroyVgroupDataCxt(pVgCxt); } - - taosArrayDestroy(pArray); - return TSDB_CODE_SUCCESS; + return code; } -// data block is disordered, sort it in ascending order, and merge dup rows if exists -static int sortMergeDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo, - SBlockRowMerger** ppBlkRowMerger) { - SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData; - STableMeta* pTableMeta = dataBuf->pTableMeta; - int32_t nRows = pBlocks->numOfRows; - - // size is less than the total size, since duplicated rows may be removed. - - // allocate memory - size_t nAlloc = nRows * sizeof(SBlockKeyTuple); - if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { - char* tmp = taosMemoryRealloc(pBlkKeyInfo->pKeyTuple, nAlloc); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple*)tmp; - pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc; +int insColDataComp(const void* lp, const void* rp) { + SColData* pLeft = (SColData*)lp; + SColData* pRight = (SColData*)rp; + if (pLeft->cid < pRight->cid) { + return -1; + } else if (pLeft->cid > pRight->cid) { + return 1; } - memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc); - - tdResetSBlockRowMerger(*ppBlkRowMerger); - int32_t extendedRowSize = insGetExtendedRowSize(dataBuf); - SBlockKeyTuple* pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - char* pBlockData = pBlocks->data + pBlocks->schemaLen; - int32_t n = 0; - while (n < nRows) { - pBlkKeyTuple->skey = TD_ROW_KEY((STSRow*)pBlockData); - pBlkKeyTuple->payloadAddr = pBlockData; - pBlkKeyTuple->index = n; + return 0; +} - // next loop - pBlockData += extendedRowSize; - ++pBlkKeyTuple; - ++n; +int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks) { + SHashObj* pVgroupHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); + SArray* pVgroupList = taosArrayInit(8, POINTER_BYTES); + if (NULL == pVgroupHash || NULL == pVgroupList) { + taosHashCleanup(pVgroupHash); + taosArrayDestroy(pVgroupList); + return TSDB_CODE_OUT_OF_MEMORY; } - if (!dataBuf->ordered) { - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - - taosSort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataComparStable); - - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - bool hasDup = false; - int32_t nextPos = 0; - int32_t i = 0; - int32_t j = 1; + int32_t code = TSDB_CODE_SUCCESS; + bool colFormat = false; - while (j < nRows) { - TSKEY ti = (pBlkKeyTuple + i)->skey; - TSKEY tj = (pBlkKeyTuple + j)->skey; + void* p = taosHashIterate(pTableHash, NULL); + if (p) { + STableDataCxt* pTableCxt = *(STableDataCxt**)p; + colFormat = (0 != (pTableCxt->pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT)); + } - if (ti == tj) { - ++j; + while (TSDB_CODE_SUCCESS == code && NULL != p) { + STableDataCxt* pTableCxt = *(STableDataCxt**)p; + if (colFormat) { + SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, 0); + if (pCol->nVal <= 0) { + p = taosHashIterate(pTableHash, p); continue; } - if ((j - i) > 1) { - if (tdBlockRowMerge(pTableMeta, (pBlkKeyTuple + j - 1), j - i, ppBlkRowMerger, extendedRowSize) < 0) { - return TSDB_CODE_FAILED; - } - (pBlkKeyTuple + nextPos)->payloadAddr = tdGetCurRowFromBlockMerger(*ppBlkRowMerger); - if (!hasDup) { - hasDup = true; - } - i = j; - } else { - if (hasDup) { - memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + i, sizeof(SBlockKeyTuple)); - } - ++i; + if (pTableCxt->pData->pCreateTbReq) { + pTableCxt->pData->flags |= SUBMIT_REQ_AUTO_CREATE_TABLE; } - ++nextPos; - ++j; - } + taosArraySort(pTableCxt->pData->aCol, insColDataComp); - if ((j - i) > 1) { - ASSERT((pBlkKeyTuple + i)->skey == (pBlkKeyTuple + j - 1)->skey); - if (tdBlockRowMerge(pTableMeta, (pBlkKeyTuple + j - 1), j - i, ppBlkRowMerger, extendedRowSize) < 0) { - return TSDB_CODE_FAILED; + tColDataSortMerge(pTableCxt->pData->aCol); + } else { + if (!pTableCxt->ordered) { + tRowSort(pTableCxt->pData->aRowP); + } + if (!pTableCxt->ordered || pTableCxt->duplicateTs) { + code = tRowMerge(pTableCxt->pData->aRowP, pTableCxt->pSchema, 0); } - (pBlkKeyTuple + nextPos)->payloadAddr = tdGetCurRowFromBlockMerger(*ppBlkRowMerger); - } else if (hasDup) { - memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + i, sizeof(SBlockKeyTuple)); } - dataBuf->ordered = true; - pBlocks->numOfRows = nextPos + 1; - } - - dataBuf->size = sizeof(SSubmitBlk) + pBlocks->numOfRows * extendedRowSize; - dataBuf->prevTS = INT64_MIN; - - return TSDB_CODE_SUCCESS; -} - -// Erase the empty space reserved for binary data -static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SBlockKeyTuple* blkKeyTuple) { - // TODO: optimize this function, handle the case while binary is not presented - int32_t nonDataLen = sizeof(SSubmitBlk) + pTableDataBlock->createTbReqLen; - SSubmitBlk* pBlock = pDataBlock; - memcpy(pDataBlock, pTableDataBlock->pData, nonDataLen); - pDataBlock = (char*)pDataBlock + nonDataLen; - - pBlock->schemaLen = pTableDataBlock->createTbReqLen; - pBlock->dataLen = 0; - - int32_t numOfRows = pBlock->numOfRows; - for (int32_t i = 0; i < numOfRows; ++i) { - void* payload = (blkKeyTuple + i)->payloadAddr; - TDRowLenT rowTLen = TD_ROW_LEN((STSRow*)payload); - memcpy(pDataBlock, payload, rowTLen); - pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen); - pBlock->dataLen += rowTLen; - } - - return pBlock->dataLen + pBlock->schemaLen; -} - -int32_t insMergeTableDataBlocks(SHashObj* pHashObj, SArray** pVgDataBlocks) { - const int INSERT_HEAD_SIZE = sizeof(SSubmitReq); - int code = 0; - SHashObj* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); - SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); - - STableDataBlocks** p = taosHashIterate(pHashObj, NULL); - STableDataBlocks* pOneTableBlock = *p; - SBlockKeyInfo blkKeyInfo = {0}; // share by pOneTableBlock - SBlockRowMerger* pBlkRowMerger = NULL; - - while (pOneTableBlock) { - SSubmitBlk* pBlocks = (SSubmitBlk*)pOneTableBlock->pData; - if (pBlocks->numOfRows > 0) { - STableDataBlocks* dataBuf = NULL; - pOneTableBlock->pTableMeta->vgId = pOneTableBlock->vgId; // for schemaless, restore origin vgId - int32_t ret = insGetDataBlockFromList(pVnodeDataBlockHashList, &pOneTableBlock->vgId, - sizeof(pOneTableBlock->vgId), TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, - pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList, NULL); - if (ret != TSDB_CODE_SUCCESS) { - tdFreeSBlockRowMerger(pBlkRowMerger); - taosHashCleanup(pVnodeDataBlockHashList); - insDestroyBlockArrayList(pVnodeDataBlockList); - taosMemoryFreeClear(blkKeyInfo.pKeyTuple); - return ret; + if (TSDB_CODE_SUCCESS == code) { + SVgroupDataCxt* pVgCxt = NULL; + int32_t vgId = pTableCxt->pMeta->vgId; + void** p = taosHashGet(pVgroupHash, &vgId, sizeof(vgId)); + if (NULL == p) { + code = createVgroupDataCxt(pTableCxt, pVgroupHash, pVgroupList, &pVgCxt); + } else { + pVgCxt = *(SVgroupDataCxt**)p; } - ASSERT(pOneTableBlock->pTableMeta->tableInfo.rowSize > 0); - // the maximum expanded size in byte when a row-wise data is converted to SDataRow format - int64_t destSize = dataBuf->size + pOneTableBlock->size + - sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta) + - pOneTableBlock->createTbReqLen; - - if (dataBuf->nAllocSize < destSize) { - dataBuf->nAllocSize = (uint32_t)(destSize * 1.5); - char* tmp = taosMemoryRealloc(dataBuf->pData, dataBuf->nAllocSize); - if (tmp != NULL) { - dataBuf->pData = tmp; - } else { // failed to allocate memory, free already allocated memory and return error code - tdFreeSBlockRowMerger(pBlkRowMerger); - taosHashCleanup(pVnodeDataBlockHashList); - insDestroyBlockArrayList(pVnodeDataBlockList); - taosMemoryFreeClear(dataBuf->pData); - taosMemoryFreeClear(blkKeyInfo.pKeyTuple); - return TSDB_CODE_OUT_OF_MEMORY; - } + if (TSDB_CODE_SUCCESS == code) { + code = fillVgroupDataCxt(pTableCxt, pVgCxt); } - - if ((code = sortMergeDataBlockDupRows(pOneTableBlock, &blkKeyInfo, &pBlkRowMerger)) != 0) { - tdFreeSBlockRowMerger(pBlkRowMerger); - taosHashCleanup(pVnodeDataBlockHashList); - insDestroyBlockArrayList(pVnodeDataBlockList); - taosMemoryFreeClear(dataBuf->pData); - taosMemoryFreeClear(blkKeyInfo.pKeyTuple); - return code; - } - ASSERT(blkKeyInfo.pKeyTuple != NULL && pBlocks->numOfRows > 0); - - // erase the empty space reserved for binary data - int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, blkKeyInfo.pKeyTuple); - - dataBuf->size += (finalLen + sizeof(SSubmitBlk)); - assert(dataBuf->size <= dataBuf->nAllocSize); - dataBuf->numOfTables += 1; } - - p = taosHashIterate(pHashObj, p); - if (p == NULL) { - break; + if (TSDB_CODE_SUCCESS == code) { + p = taosHashIterate(pTableHash, p); } + } - pOneTableBlock = *p; + taosHashCleanup(pVgroupHash); + if (TSDB_CODE_SUCCESS == code) { + *pVgDataBlocks = pVgroupList; + } else { + insDestroyVgroupDataCxtList(pVgroupList); } - // free the table data blocks; - tdFreeSBlockRowMerger(pBlkRowMerger); - taosHashCleanup(pVnodeDataBlockHashList); - taosMemoryFreeClear(blkKeyInfo.pKeyTuple); - *pVgDataBlocks = pVnodeDataBlockList; - return TSDB_CODE_SUCCESS; + return code; } -int32_t insAllocateMemForSize(STableDataBlocks* pDataBlock, int32_t allSize) { - size_t remain = pDataBlock->nAllocSize - pDataBlock->size; - uint32_t nAllocSizeOld = pDataBlock->nAllocSize; - - // expand the allocated size - if (remain < allSize) { - pDataBlock->nAllocSize = (pDataBlock->size + allSize) * 1.5; - - char* tmp = taosMemoryRealloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize); - if (tmp != NULL) { - pDataBlock->pData = tmp; - memset(pDataBlock->pData + pDataBlock->size, 0, pDataBlock->nAllocSize - pDataBlock->size); - } else { - // do nothing, if allocate more memory failed - pDataBlock->nAllocSize = nAllocSizeOld; +static int32_t buildSubmitReq(int32_t vgId, SSubmitReq2* pReq, void** pData, uint32_t* pLen) { + int32_t code = TSDB_CODE_SUCCESS; + uint32_t len = 0; + void* pBuf = NULL; + tEncodeSize(tEncodeSSubmitReq2, pReq, len, code); + if (TSDB_CODE_SUCCESS == code) { + SEncoder encoder; + len += sizeof(SMsgHead); + pBuf = taosMemoryMalloc(len); + if (NULL == pBuf) { return TSDB_CODE_OUT_OF_MEMORY; } + ((SMsgHead*)pBuf)->vgId = htonl(vgId); + ((SMsgHead*)pBuf)->contLen = htonl(len); + tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SMsgHead)), len - sizeof(SMsgHead)); + code = tEncodeSSubmitReq2(&encoder, pReq); + tEncoderClear(&encoder); } - return TSDB_CODE_SUCCESS; + if (TSDB_CODE_SUCCESS == code) { + *pData = pBuf; + *pLen = len; + } else { + taosMemoryFree(pBuf); + } + return code; } -int32_t insInitRowBuilder(SRowBuilder* pBuilder, int16_t schemaVer, SParsedDataColInfo* pColInfo) { - ASSERT(pColInfo->numOfCols > 0 && (pColInfo->numOfBound <= pColInfo->numOfCols)); - tdSRowInit(pBuilder, schemaVer); - tdSRowSetExtendedInfo(pBuilder, pColInfo->numOfCols, pColInfo->numOfBound, pColInfo->flen, pColInfo->allNullLen, - pColInfo->boundNullLen); - return TSDB_CODE_SUCCESS; +static void destroyVgDataBlocks(void* p) { + SVgDataBlocks* pVg = p; + taosMemoryFree(pVg->pData); + taosMemoryFree(pVg); } -static char* tableNameGetPosition(SToken* pToken, char target) { - bool inEscape = false; - bool inQuote = false; - char quotaStr = 0; - - for (uint32_t i = 0; i < pToken->n; ++i) { - if (*(pToken->z + i) == target && (!inEscape) && (!inQuote)) { - return pToken->z + i; - } - - if (*(pToken->z + i) == TS_ESCAPE_CHAR) { - if (!inQuote) { - inEscape = !inEscape; - } - } - - if (*(pToken->z + i) == '\'' || *(pToken->z + i) == '"') { - if (!inEscape) { - if (!inQuote) { - quotaStr = *(pToken->z + i); - inQuote = !inQuote; - } else if (quotaStr == *(pToken->z + i)) { - inQuote = !inQuote; - } - } - } +int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList, SArray** pVgDataBlocks) { + size_t numOfVg = taosArrayGetSize(pVgDataCxtList); + SArray* pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); + if (NULL == pDataBlocks) { + return TSDB_CODE_OUT_OF_MEMORY; } - return NULL; -} - -int32_t insCreateSName(SName* pName, SToken* pTableName, int32_t acctId, const char* dbName, SMsgBuf* pMsgBuf) { - const char* msg1 = "name too long"; - const char* msg2 = "invalid database name"; - const char* msg3 = "db is not specified"; - const char* msg4 = "invalid table name"; - int32_t code = TSDB_CODE_SUCCESS; - char* p = tableNameGetPosition(pTableName, TS_PATH_DELIMITER[0]); - - if (p != NULL) { // db has been specified in sql string so we ignore current db path - assert(*p == TS_PATH_DELIMITER[0]); - - int32_t dbLen = p - pTableName->z; - if (dbLen <= 0) { - return buildInvalidOperationMsg(pMsgBuf, msg2); - } - char name[TSDB_DB_FNAME_LEN] = {0}; - strncpy(name, pTableName->z, dbLen); - int32_t actualDbLen = strdequote(name); - - code = tNameSetDbName(pName, acctId, name, actualDbLen); - if (code != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg1); + for (size_t i = 0; TSDB_CODE_SUCCESS == code && i < numOfVg; ++i) { + SVgroupDataCxt* src = taosArrayGetP(pVgDataCxtList, i); + SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); + if (NULL == dst) { + code = TSDB_CODE_OUT_OF_MEMORY; } - - int32_t tbLen = pTableName->n - dbLen - 1; - if (tbLen <= 0) { - return buildInvalidOperationMsg(pMsgBuf, msg4); + if (TSDB_CODE_SUCCESS == code) { + dst->numOfTables = taosArrayGetSize(src->pData->aSubmitTbData); + code = taosHashGetDup(pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); } - - char tbname[TSDB_TABLE_FNAME_LEN] = {0}; - strncpy(tbname, p + 1, tbLen); - /*tbLen = */ strdequote(tbname); - - code = tNameFromString(pName, tbname, T_NAME_TABLE); - if (code != 0) { - return buildInvalidOperationMsg(pMsgBuf, msg1); + if (TSDB_CODE_SUCCESS == code) { + code = buildSubmitReq(src->vgId, src->pData, &dst->pData, &dst->size); } - } else { // get current DB name first, and then set it into path - if (pTableName->n >= TSDB_TABLE_NAME_LEN) { - return buildInvalidOperationMsg(pMsgBuf, msg1); + if (TSDB_CODE_SUCCESS == code) { + code = (NULL == taosArrayPush(pDataBlocks, &dst) ? TSDB_CODE_OUT_OF_MEMORY : TSDB_CODE_SUCCESS); } + } - assert(pTableName->n < TSDB_TABLE_FNAME_LEN); + if (TSDB_CODE_SUCCESS == code) { + *pVgDataBlocks = pDataBlocks; + } else { + taosArrayDestroyP(pDataBlocks, destroyVgDataBlocks); + } - char name[TSDB_TABLE_FNAME_LEN] = {0}; - strncpy(name, pTableName->z, pTableName->n); - strdequote(name); + return code; +} - if (dbName == NULL) { - return buildInvalidOperationMsg(pMsgBuf, msg3); - } +static int bindFileds(SBoundColInfo* pBoundInfo, SSchema* pSchema, TAOS_FIELD* fields, int numFields) { + bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool)); + if (NULL == pUseCols) { + return TSDB_CODE_OUT_OF_MEMORY; + } - code = tNameSetDbName(pName, acctId, dbName, strlen(dbName)); - if (code != TSDB_CODE_SUCCESS) { - code = buildInvalidOperationMsg(pMsgBuf, msg2); - return code; - } + pBoundInfo->numOfBound = 0; - code = tNameFromString(pName, name, T_NAME_TABLE); - if (code != 0) { - code = buildInvalidOperationMsg(pMsgBuf, msg1); + int16_t lastColIdx = -1; // last column found + int32_t code = TSDB_CODE_SUCCESS; + for (int i = 0; i < numFields; i++) { + SToken token; + token.z = fields[i].name; + token.n = strlen(fields[i].name); + + int16_t t = lastColIdx + 1; + int16_t index = insFindCol(&token, t, pBoundInfo->numOfCols, pSchema); + if (index < 0 && t > 0) { + index = insFindCol(&token, 0, t, pSchema); + } + if (index < 0) { + uError("can not find column name:%s", token.z); + code = TSDB_CODE_PAR_INVALID_COLUMN; + break; + } else if (pUseCols[index]) { + code = TSDB_CODE_PAR_INVALID_COLUMN; + uError("duplicated column name:%s", token.z); + break; + } else { + lastColIdx = index; + pUseCols[index] = true; + pBoundInfo->pColIndex[pBoundInfo->numOfBound] = index; + ++pBoundInfo->numOfBound; } } - if (NULL != strchr(pName->tname, '.')) { - code = generateSyntaxErrMsgExt(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, "The table name cannot contain '.'"); + if (TSDB_CODE_SUCCESS == code && !pUseCols[0]) { + uError("primary timestamp column can not be null:"); + code = TSDB_CODE_PAR_INVALID_COLUMN; } + taosMemoryFree(pUseCols); return code; } -int32_t insFindCol(SToken* pColname, int32_t start, int32_t end, SSchema* pSchema) { - while (start < end) { - if (strlen(pSchema[start].name) == pColname->n && strncmp(pColname->z, pSchema[start].name, pColname->n) == 0) { - return start; +int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD* tFields, + int numFields) { + STableDataCxt* pTableCxt = NULL; + int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, + sizeof(pTableMeta->uid), pTableMeta, &pCreateTb, &pTableCxt, true); + if (ret != TSDB_CODE_SUCCESS) { + uError("insGetTableDataCxt error"); + goto end; + } + if (tFields != NULL) { + ret = bindFileds(&pTableCxt->boundColsInfo, getTableColumnSchema(pTableMeta), tFields, numFields); + if (ret != TSDB_CODE_SUCCESS) { + uError("bindFileds error"); + goto end; } - ++start; } - return -1; -} + // no need to bind, because select * get all fields + ret = initTableColSubmitData(pTableCxt); + if (ret != TSDB_CODE_SUCCESS) { + uError("initTableColSubmitData error"); + goto end; + } -void insBuildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, int64_t suid, const char* sname, - SArray* tagName, uint8_t tagNum, int32_t ttl) { - pTbReq->type = TD_CHILD_TABLE; - pTbReq->name = strdup(tname); - pTbReq->ctb.suid = suid; - pTbReq->ctb.tagNum = tagNum; - if (sname) pTbReq->ctb.stbName = strdup(sname); - pTbReq->ctb.pTag = (uint8_t*)pTag; - pTbReq->ctb.tagName = taosArrayDup(tagName, NULL); - pTbReq->ttl = ttl; - pTbReq->commentLen = -1; + char* p = (char*)data; + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column + // length | + p += sizeof(int32_t); + p += sizeof(int32_t); - return; -} + int32_t numOfRows = *(int32_t*)p; + p += sizeof(int32_t); -int32_t insMemRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param) { - SMemParam* pa = (SMemParam*)param; - SRowBuilder* rb = pa->rb; + int32_t numOfCols = *(int32_t*)p; + p += sizeof(int32_t); - if (value == NULL) { // it is a null data - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, pa->colIdx); - return TSDB_CODE_SUCCESS; - } + p += sizeof(int32_t); + p += sizeof(uint64_t); - if (TSDB_DATA_TYPE_BINARY == pa->schema->type) { - const char* rowEnd = tdRowEnd(rb->pBuf); - STR_WITH_SIZE_TO_VARSTR(rowEnd, value, len); - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, false, pa->toffset, pa->colIdx); - } else if (TSDB_DATA_TYPE_NCHAR == pa->schema->type) { - // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' - int32_t output = 0; - const char* rowEnd = tdRowEnd(rb->pBuf); - if (!taosMbsToUcs4(value, len, (TdUcs4*)varDataVal(rowEnd), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { - if (errno == E2BIG) { - return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pa->schema->name); - } - char buf[512] = {0}; - snprintf(buf, tListLen(buf), "%s", strerror(errno)); - return buildSyntaxErrMsg(pMsgBuf, buf, value); - } - varDataSetLen(rowEnd, output); - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, false, pa->toffset, pa->colIdx); - } else { - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, value, false, pa->toffset, pa->colIdx); - } + int8_t* fields = p; + p += numOfCols * (sizeof(int8_t) + sizeof(int32_t)); - return TSDB_CODE_SUCCESS; -} + int32_t* colLength = (int32_t*)p; + p += sizeof(int32_t) * numOfCols; -int32_t insCheckTimestamp(STableDataBlocks* pDataBlocks, const char* start) { - // once the data block is disordered, we do NOT keep previous timestamp any more - if (!pDataBlocks->ordered) { - return TSDB_CODE_SUCCESS; - } - - TSKEY k = *(TSKEY*)start; - if (k <= pDataBlocks->prevTS) { - pDataBlocks->ordered = false; - } + char* pStart = p; - pDataBlocks->prevTS = k; - return TSDB_CODE_SUCCESS; -} + SSchema* pSchema = getTableColumnSchema(pTableCxt->pMeta); + SBoundColInfo* boundInfo = &pTableCxt->boundColsInfo; -static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { - SSubmitReq* submit = (SSubmitReq*)blocks->pData; - submit->header.vgId = htonl(blocks->vg.vgId); - submit->header.contLen = htonl(blocks->size); - submit->length = submit->header.contLen; - submit->numOfBlocks = htonl(blocks->numOfTables); - SSubmitBlk* blk = (SSubmitBlk*)(submit + 1); - int32_t numOfBlocks = blocks->numOfTables; - while (numOfBlocks--) { - int32_t dataLen = blk->dataLen; - int32_t schemaLen = blk->schemaLen; - blk->uid = htobe64(blk->uid); - blk->suid = htobe64(blk->suid); - blk->sversion = htonl(blk->sversion); - blk->dataLen = htonl(blk->dataLen); - blk->schemaLen = htonl(blk->schemaLen); - blk->numOfRows = htonl(blk->numOfRows); - blk = (SSubmitBlk*)(blk->data + schemaLen + dataLen); + if (boundInfo->numOfBound != numOfCols) { + uError("boundInfo->numOfBound:%d != numOfCols:%d", boundInfo->numOfBound, numOfCols); + ret = TSDB_CODE_INVALID_PARA; + goto end; } -} + for (int c = 0; c < boundInfo->numOfBound; ++c) { + SSchema* pColSchema = &pSchema[boundInfo->pColIndex[c]]; + SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, c); -int32_t insBuildOutput(SHashObj* pVgroupsHashObj, SArray* pVgDataBlocks, SArray** pDataBlocks) { - size_t numOfVg = taosArrayGetSize(pVgDataBlocks); - *pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); - if (NULL == *pDataBlocks) { - return TSDB_CODE_OUT_OF_MEMORY; - } - for (size_t i = 0; i < numOfVg; ++i) { - STableDataBlocks* src = taosArrayGetP(pVgDataBlocks, i); - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - return TSDB_CODE_OUT_OF_MEMORY; + if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { + uError("type or bytes not equal"); + ret = TSDB_CODE_INVALID_PARA; + goto end; + } + + colLength[c] = htonl(colLength[c]); + int8_t* offset = pStart; + if (IS_VAR_DATA_TYPE(pColSchema->type)) { + pStart += numOfRows * sizeof(int32_t); + } else { + pStart += BitmapLen(numOfRows); } - taosHashGetDup(pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); - dst->numOfTables = src->numOfTables; - dst->size = src->size; - TSWAP(dst->pData, src->pData); - buildMsgHeader(src, dst); - taosArrayPush(*pDataBlocks, &dst); + char* pData = pStart; + + tColDataAddValueByDataBlock(pCol, pColSchema->type, pColSchema->bytes, numOfRows, offset, pData); + fields += sizeof(int8_t) + sizeof(int32_t); + pStart += colLength[c]; } - return TSDB_CODE_SUCCESS; -} + +end: + return ret; +} \ No newline at end of file diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 45eacad8557477f8c3dcc6654f8134ee2a3118b8..eb1083801cb2b41a5f223fd9d8888cdf6e70fd98 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -352,7 +352,7 @@ static int32_t getTableMetaImpl(STranslateContext* pCxt, const SName* pName, STa code = catalogGetTableMeta(pParCxt->pCatalog, &conn, pName, pMeta); } } - if (TSDB_CODE_SUCCESS != code && TSDB_CODE_TSC_INVALID_TABLE_NAME != code) { + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_PAR_TABLE_NOT_EXIST != code) { parserError("0x%" PRIx64 " catalogGetTableMeta error, code:%s, dbName:%s, tbName:%s", pCxt->pParseCxt->requestId, tstrerror(code), pName->dbname, pName->tname); } @@ -4569,58 +4569,109 @@ typedef struct SSampleAstInfo { SNode* pSliding; SNodeList* pPartitionByList; STableMeta* pRollupTableMeta; + bool createSmaIndex; } SSampleAstInfo; -static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, char** pAst, int32_t* pLen, char** pExpr, - int32_t* pExprLen) { - SSelectStmt* pSelect = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); - if (NULL == pSelect) { - return TSDB_CODE_OUT_OF_MEMORY; - } - sprintf(pSelect->stmtName, "%p", pSelect); - +static int32_t buildTableForSampleAst(SSampleAstInfo* pInfo, SNode** pOutput) { SRealTableNode* pTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); if (NULL == pTable) { - nodesDestroyNode((SNode*)pSelect); return TSDB_CODE_OUT_OF_MEMORY; } snprintf(pTable->table.dbName, sizeof(pTable->table.dbName), "%s", pInfo->pDbName); snprintf(pTable->table.tableName, sizeof(pTable->table.tableName), "%s", pInfo->pTableName); TSWAP(pTable->pMeta, pInfo->pRollupTableMeta); - pSelect->pFromTable = (SNode*)pTable; + *pOutput = (SNode*)pTable; + return TSDB_CODE_SUCCESS; +} - TSWAP(pSelect->pProjectionList, pInfo->pFuncs); +static int32_t addWstartToSampleProjects(SNodeList* pProjectionList) { SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); - if (NULL == pSelect->pProjectionList || NULL == pFunc) { - nodesDestroyNode((SNode*)pSelect); + if (NULL == pFunc) { return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pFunc->functionName, "_wstart"); - nodesListPushFront(pSelect->pProjectionList, (SNode*)pFunc); - SNode* pProject = NULL; - FOREACH(pProject, pSelect->pProjectionList) { sprintf(((SExprNode*)pProject)->aliasName, "#%p", pProject); } + return nodesListPushFront(pProjectionList, (SNode*)pFunc); +} + +static int32_t addWendToSampleProjects(SNodeList* pProjectionList) { + SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(pFunc->functionName, "_wend"); + return nodesListAppend(pProjectionList, (SNode*)pFunc); +} + +static int32_t addWdurationToSampleProjects(SNodeList* pProjectionList) { + SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(pFunc->functionName, "_wduration"); + return nodesListAppend(pProjectionList, (SNode*)pFunc); +} - TSWAP(pSelect->pPartitionByList, pInfo->pPartitionByList); +static int32_t buildProjectsForSampleAst(SSampleAstInfo* pInfo, SNodeList** pList) { + SNodeList* pProjectionList = pInfo->pFuncs; + pInfo->pFuncs = NULL; + + int32_t code = addWstartToSampleProjects(pProjectionList); + if (TSDB_CODE_SUCCESS == code && pInfo->createSmaIndex) { + code = addWendToSampleProjects(pProjectionList); + if (TSDB_CODE_SUCCESS == code) { + code = addWdurationToSampleProjects(pProjectionList); + } + } + if (TSDB_CODE_SUCCESS == code) { + SNode* pProject = NULL; + FOREACH(pProject, pProjectionList) { sprintf(((SExprNode*)pProject)->aliasName, "#%p", pProject); } + *pList = pProjectionList; + } else { + nodesDestroyList(pProjectionList); + } + return code; +} + +static int32_t buildIntervalForSampleAst(SSampleAstInfo* pInfo, SNode** pOutput) { SIntervalWindowNode* pInterval = (SIntervalWindowNode*)nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW); if (NULL == pInterval) { - nodesDestroyNode((SNode*)pSelect); return TSDB_CODE_OUT_OF_MEMORY; } - pSelect->pWindow = (SNode*)pInterval; TSWAP(pInterval->pInterval, pInfo->pInterval); TSWAP(pInterval->pOffset, pInfo->pOffset); TSWAP(pInterval->pSliding, pInfo->pSliding); pInterval->pCol = nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pInterval->pCol) { - nodesDestroyNode((SNode*)pSelect); + nodesDestroyNode((SNode*)pInterval); return TSDB_CODE_OUT_OF_MEMORY; } ((SColumnNode*)pInterval->pCol)->colId = PRIMARYKEY_TIMESTAMP_COL_ID; strcpy(((SColumnNode*)pInterval->pCol)->colName, ROWTS_PSEUDO_COLUMN_NAME); + *pOutput = (SNode*)pInterval; + return TSDB_CODE_SUCCESS; +} - pCxt->createStream = true; - int32_t code = translateQuery(pCxt, (SNode*)pSelect); +static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, char** pAst, int32_t* pLen, char** pExpr, + int32_t* pExprLen) { + SSelectStmt* pSelect = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); + if (NULL == pSelect) { + return TSDB_CODE_OUT_OF_MEMORY; + } + sprintf(pSelect->stmtName, "%p", pSelect); + + int32_t code = buildTableForSampleAst(pInfo, &pSelect->pFromTable); + if (TSDB_CODE_SUCCESS == code) { + code = buildProjectsForSampleAst(pInfo, &pSelect->pProjectionList); + } + if (TSDB_CODE_SUCCESS == code) { + TSWAP(pSelect->pPartitionByList, pInfo->pPartitionByList); + code = buildIntervalForSampleAst(pInfo, &pSelect->pWindow); + } + if (TSDB_CODE_SUCCESS == code) { + pCxt->createStream = true; + code = translateQuery(pCxt, (SNode*)pSelect); + } if (TSDB_CODE_SUCCESS == code) { code = nodesNodeToString((SNode*)pSelect, false, pAst, pLen); } @@ -4928,10 +4979,10 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* return TSDB_CODE_SUCCESS; } -static SSchema* getColSchema(STableMeta* pTableMeta, const char* pColName) { +static const SSchema* getColSchema(const STableMeta* pTableMeta, const char* pColName) { int32_t numOfFields = getNumOfTags(pTableMeta) + getNumOfColumns(pTableMeta); for (int32_t i = 0; i < numOfFields; ++i) { - SSchema* pSchema = pTableMeta->schema + i; + const SSchema* pSchema = pTableMeta->schema + i; if (0 == strcmp(pColName, pSchema->name)) { return pSchema; } @@ -4951,7 +5002,8 @@ static SSchema* getTagSchema(STableMeta* pTableMeta, const char* pTagName) { return NULL; } -static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta) { +static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTableStmt* pStmt, + const STableMeta* pTableMeta) { SSchema* pTagsSchema = getTableTagSchema(pTableMeta); if (getNumOfTags(pTableMeta) == 1 && pTagsSchema->type == TSDB_DATA_TYPE_JSON && (pStmt->alterType == TSDB_ALTER_TABLE_ADD_TAG || pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG || @@ -4970,7 +5022,7 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "Table is not super table"); } - SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); + const SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); if (NULL == pSchema) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); } else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type || @@ -5162,6 +5214,7 @@ static int32_t getSmaIndexSql(STranslateContext* pCxt, char** pSql, int32_t* pLe } static int32_t buildSampleAstInfoByIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SSampleAstInfo* pInfo) { + pInfo->createSmaIndex = true; pInfo->pDbName = pStmt->dbName; pInfo->pTableName = pStmt->tableName; pInfo->pFuncs = nodesCloneList(pStmt->pOptions->pFuncs); @@ -5651,7 +5704,8 @@ static int32_t addSubtableInfoToCreateStreamQuery(STranslateContext* pCxt, SCrea return code; } -static int32_t checkStreamQuery(STranslateContext* pCxt, SSelectStmt* pSelect) { +static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt) { + SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; if (TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type || !pSelect->isTimeLineResult || crossTableWithoutAggOper(pSelect) || NULL != pSelect->pOrderByList || crossTableWithUdaf(pSelect)) { @@ -5661,9 +5715,146 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SSelectStmt* pSelect) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "SUBTABLE expression must be of VARCHAR type"); } + if (NULL == pSelect->pWindow && STREAM_TRIGGER_AT_ONCE != pStmt->pOptions->triggerType) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "The trigger mode of non window query can only be AT_ONCE"); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t adjustDataTypeOfProjections(STranslateContext* pCxt, const STableMeta* pMeta, SNodeList* pProjections) { + if (getNumOfColumns(pMeta) != LIST_LENGTH(pProjections)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM, "Illegal number of columns"); + } + + SSchema* pSchemas = getTableColumnSchema(pMeta); + int32_t index = 0; + SNode* pProj = NULL; + FOREACH(pProj, pProjections) { + SSchema* pSchema = pSchemas + index; + SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes}; + if (!dataTypeEqual(&dt, &((SExprNode*)pProj)->resType)) { + SNode* pFunc = NULL; + int32_t code = createCastFunc(pCxt, pProj, dt, &pFunc); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + REPLACE_NODE(pFunc); + } + } + return TSDB_CODE_SUCCESS; } +typedef struct SProjColPos { + int32_t colId; + SNode* pProj; +} SProjColPos; + +static int32_t projColPosCompar(const void* l, const void* r) { + return ((SProjColPos*)l)->colId < ((SProjColPos*)r)->colId; +} + +static void projColPosDelete(void* p) { taosMemoryFree(((SProjColPos*)p)->pProj); } + +static int32_t addProjToProjColPos(STranslateContext* pCxt, const SSchema* pSchema, SNode* pProj, SArray* pProjColPos) { + SNode* pNewProj = nodesCloneNode(pProj); + if (NULL == pNewProj) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes}; + if (!dataTypeEqual(&dt, &((SExprNode*)pNewProj)->resType)) { + SNode* pFunc = NULL; + code = createCastFunc(pCxt, pNewProj, dt, &pFunc); + pNewProj = pFunc; + } + if (TSDB_CODE_SUCCESS == code) { + SProjColPos pos = {.colId = pSchema->colId, .pProj = pNewProj}; + code = (NULL == taosArrayPush(pProjColPos, &pos) ? TSDB_CODE_OUT_OF_MEMORY : TSDB_CODE_SUCCESS); + } + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pNewProj); + } + return code; +} + +static int32_t adjustOrderOfProjection(STranslateContext* pCxt, SNodeList* pCols, const STableMeta* pMeta, + SNodeList** pProjections) { + if (LIST_LENGTH(pCols) != LIST_LENGTH(*pProjections)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM, "Illegal number of columns"); + } + + SArray* pProjColPos = taosArrayInit(LIST_LENGTH(pCols), sizeof(SProjColPos)); + if (NULL == pProjColPos) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + SNode* pCol = NULL; + SNode* pProj = NULL; + FORBOTH(pCol, pCols, pProj, *pProjections) { + const SSchema* pSchema = getColSchema(pMeta, ((SColumnNode*)pCol)->colName); + code = addProjToProjColPos(pCxt, pSchema, pProj, pProjColPos); + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + + SNodeList* pNewProjections = NULL; + if (TSDB_CODE_SUCCESS == code) { + taosArraySort(pProjColPos, projColPosCompar); + int32_t num = taosArrayGetSize(pProjColPos); + pNewProjections = nodesMakeList(); + if (NULL == pNewProjections) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + for (int32_t i = 0; TSDB_CODE_SUCCESS == code && i < num; ++i) { + SProjColPos* pPos = taosArrayGet(pProjColPos, i); + code = nodesListStrictAppend(pNewProjections, pPos->pProj); + pPos->pProj = NULL; + } + } + + if (TSDB_CODE_SUCCESS == code) { + taosArrayDestroy(pProjColPos); + nodesDestroyList(*pProjections); + *pProjections = pNewProjections; + } else { + taosArrayDestroyEx(pProjColPos, projColPosDelete); + nodesDestroyList(pNewProjections); + } + + return code; +} + +static int32_t adjustStreamQueryForExistTableImpl(STranslateContext* pCxt, SCreateStreamStmt* pStmt, + const STableMeta* pMeta) { + SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; + if (NULL == pStmt->pCols) { + return adjustDataTypeOfProjections(pCxt, pMeta, pSelect->pProjectionList); + } + return adjustOrderOfProjection(pCxt, pStmt->pCols, pMeta, &pSelect->pProjectionList); +} + +static int32_t adjustStreamQueryForExistTable(STranslateContext* pCxt, SCreateStreamStmt* pStmt, + SCMCreateStreamReq* pReq) { + STableMeta* pMeta = NULL; + int32_t code = getTableMeta(pCxt, pStmt->targetDbName, pStmt->targetTabName, &pMeta); + if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { + if (NULL != pStmt->pCols) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TABLE_NOT_EXIST, pStmt->targetTabName); + } + pReq->createStb = STREAM_CREATE_STABLE_TRUE; + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SUCCESS == code) { + code = adjustStreamQueryForExistTableImpl(pCxt, pStmt, pMeta); + } + return code; +} + static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SCMCreateStreamReq* pReq) { pCxt->createStream = true; int32_t code = addWstartTsToCreateStreamQuery(pCxt, pStmt->pQuery); @@ -5674,7 +5865,10 @@ static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt code = translateQuery(pCxt, pStmt->pQuery); } if (TSDB_CODE_SUCCESS == code) { - code = checkStreamQuery(pCxt, (SSelectStmt*)pStmt->pQuery); + code = checkStreamQuery(pCxt, pStmt); + } + if (TSDB_CODE_SUCCESS == code) { + code = adjustStreamQueryForExistTable(pCxt, pStmt, pReq); } if (TSDB_CODE_SUCCESS == code) { getSourceDatabase(pStmt->pQuery, pCxt->pParseCxt->acctId, pReq->sourceDB); @@ -7299,12 +7493,12 @@ static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, S return TSDB_CODE_SUCCESS; } -static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, +static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, const STableMeta* pTableMeta, SVAlterTbReq* pReq) { if (2 == getNumOfColumns(pTableMeta)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DROP_COL); } - SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); + const SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); if (NULL == pSchema) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); } else if (PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { @@ -7320,11 +7514,11 @@ static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, return TSDB_CODE_SUCCESS; } -static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, +static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, const STableMeta* pTableMeta, SVAlterTbReq* pReq) { pReq->colModBytes = calcTypeBytes(pStmt->dataType); pReq->colModType = pStmt->dataType.type; - SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); + const SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); if (NULL == pSchema) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); } else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type || diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 1021aab6f099e7f65201684ab07ef5412d47ab60..7292870c3047f618ce06b373b67a7c39c5cdd21d 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -104,26 +104,26 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 461 +#define YYNOCODE 462 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SToken typedef union { int yyinit; ParseTOKENTYPE yy0; - EOrder yy32; - SToken yy77; - int32_t yy248; - int8_t yy287; - ENullOrder yy385; - EJoinType yy560; - SNode* yy600; - SNodeList* yy601; - SAlterOption yy661; - EOperatorType yy666; - int64_t yy717; - EFillMode yy798; - bool yy841; - SDataType yy888; + int8_t yy47; + EOperatorType yy128; + EJoinType yy288; + SNodeList* yy376; + SNode* yy476; + int32_t yy508; + SDataType yy532; + EOrder yy554; + EFillMode yy690; + ENullOrder yy697; + SToken yy701; + bool yy845; + SAlterOption yy893; + int64_t yy921; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -139,18 +139,17 @@ typedef union { #define ParseCTX_FETCH #define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 714 -#define YYNRULE 545 -#define YYNRULE_WITH_ACTION 545 +#define YYNSTATE 725 +#define YYNRULE 548 #define YYNTOKEN 325 -#define YY_MAX_SHIFT 713 -#define YY_MIN_SHIFTREDUCE 1061 -#define YY_MAX_SHIFTREDUCE 1605 -#define YY_ERROR_ACTION 1606 -#define YY_ACCEPT_ACTION 1607 -#define YY_NO_ACTION 1608 -#define YY_MIN_REDUCE 1609 -#define YY_MAX_REDUCE 2153 +#define YY_MAX_SHIFT 724 +#define YY_MIN_SHIFTREDUCE 1073 +#define YY_MAX_SHIFTREDUCE 1620 +#define YY_ERROR_ACTION 1621 +#define YY_ACCEPT_ACTION 1622 +#define YY_NO_ACTION 1623 +#define YY_MIN_REDUCE 1624 +#define YY_MAX_REDUCE 2171 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -217,843 +216,788 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (3069) +#define YY_ACTTAB_COUNT (2947) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 35, 276, 460, 1876, 461, 1645, 469, 367, 461, 1645, - /* 10 */ 1808, 1810, 45, 43, 1535, 1954, 1874, 590, 1751, 466, - /* 20 */ 362, 407, 1385, 38, 37, 462, 1950, 44, 42, 41, - /* 30 */ 40, 39, 169, 1465, 459, 1383, 229, 464, 1651, 1792, - /* 40 */ 571, 602, 38, 37, 2124, 602, 44, 42, 41, 40, - /* 50 */ 39, 8, 602, 334, 1862, 1946, 1952, 345, 1460, 570, - /* 60 */ 175, 320, 187, 18, 2125, 572, 613, 38, 37, 1967, - /* 70 */ 1391, 44, 42, 41, 40, 39, 468, 1411, 583, 464, - /* 80 */ 1651, 38, 37, 45, 43, 44, 42, 41, 40, 39, - /* 90 */ 2129, 362, 171, 1385, 1609, 14, 136, 327, 81, 2026, - /* 100 */ 1985, 80, 60, 27, 1465, 1802, 1383, 1412, 586, 134, - /* 110 */ 159, 1572, 1621, 1936, 603, 619, 48, 710, 125, 124, - /* 120 */ 123, 122, 121, 120, 119, 118, 117, 48, 126, 1460, - /* 130 */ 100, 160, 1467, 1468, 18, 499, 1716, 443, 1494, 1966, - /* 140 */ 64, 1391, 478, 2002, 135, 1762, 103, 1968, 623, 1970, - /* 150 */ 1971, 618, 1754, 613, 1410, 2129, 1815, 146, 172, 2124, - /* 160 */ 2055, 1440, 1450, 355, 356, 2051, 14, 1466, 1469, 257, - /* 170 */ 2063, 582, 1813, 127, 581, 2128, 566, 2124, 177, 2125, - /* 180 */ 2127, 2129, 1386, 1595, 1384, 2124, 2081, 528, 710, 1263, - /* 190 */ 1264, 260, 570, 175, 1495, 191, 190, 2125, 572, 49, - /* 200 */ 526, 2128, 524, 1467, 1468, 2125, 2126, 1389, 1390, 53, - /* 210 */ 1439, 1442, 1443, 1444, 1445, 1446, 1447, 1448, 1449, 615, - /* 220 */ 611, 1458, 1459, 1461, 1462, 1463, 1464, 2, 60, 60, - /* 230 */ 89, 60, 1440, 1450, 2070, 1410, 156, 116, 1466, 1469, - /* 240 */ 115, 114, 113, 112, 111, 110, 109, 108, 107, 648, - /* 250 */ 259, 227, 178, 1386, 2070, 1384, 1528, 38, 37, 1632, - /* 260 */ 2067, 44, 42, 41, 40, 39, 34, 360, 1489, 1490, - /* 270 */ 1491, 1492, 1493, 1497, 1498, 1499, 1500, 178, 1389, 1390, - /* 280 */ 2066, 1439, 1442, 1443, 1444, 1445, 1446, 1447, 1448, 1449, - /* 290 */ 615, 611, 1458, 1459, 1461, 1462, 1463, 1464, 2, 396, - /* 300 */ 11, 45, 43, 1936, 1954, 1385, 1539, 1631, 1985, 362, - /* 310 */ 408, 1385, 1410, 86, 322, 1950, 565, 532, 1383, 530, - /* 320 */ 398, 394, 1465, 409, 1383, 1220, 645, 644, 643, 1224, - /* 330 */ 642, 1226, 1227, 641, 1229, 638, 1678, 1235, 635, 1237, - /* 340 */ 1238, 632, 629, 213, 1946, 1952, 357, 1460, 1180, 178, - /* 350 */ 401, 1936, 18, 1391, 1815, 613, 564, 1532, 164, 1391, - /* 360 */ 1113, 366, 1112, 1410, 495, 491, 487, 483, 210, 1630, - /* 370 */ 1813, 1629, 45, 43, 1470, 212, 1316, 1317, 178, 178, - /* 380 */ 362, 178, 1385, 1182, 14, 44, 42, 41, 40, 39, - /* 390 */ 657, 1114, 547, 1465, 1409, 1383, 2124, 1113, 1628, 1112, - /* 400 */ 710, 514, 513, 512, 85, 1094, 710, 208, 1610, 131, - /* 410 */ 508, 2130, 175, 1936, 507, 1936, 2125, 572, 1460, 506, - /* 420 */ 511, 1467, 1468, 1602, 1627, 505, 267, 268, 1114, 116, - /* 430 */ 1391, 266, 115, 114, 113, 112, 111, 110, 109, 108, - /* 440 */ 107, 583, 1936, 1411, 1096, 1626, 1099, 1100, 1391, 1876, - /* 450 */ 1440, 1450, 603, 1441, 1815, 46, 1466, 1469, 11, 353, - /* 460 */ 9, 331, 1873, 590, 603, 1386, 54, 1384, 1936, 575, - /* 470 */ 1813, 1386, 134, 1384, 207, 201, 1625, 710, 180, 206, - /* 480 */ 38, 37, 474, 1762, 44, 42, 41, 40, 39, 1936, - /* 490 */ 1389, 1390, 1467, 1468, 84, 1762, 1389, 1390, 199, 1439, - /* 500 */ 1442, 1443, 1444, 1445, 1446, 1447, 1448, 1449, 615, 611, - /* 510 */ 1458, 1459, 1461, 1462, 1463, 1464, 2, 1758, 1601, 1412, - /* 520 */ 1936, 1440, 1450, 583, 1142, 1624, 571, 1466, 1469, 603, - /* 530 */ 2124, 585, 173, 2063, 2064, 603, 132, 2068, 1413, 514, - /* 540 */ 513, 512, 1386, 405, 1384, 570, 175, 131, 508, 406, - /* 550 */ 2125, 572, 507, 1623, 134, 1330, 1331, 506, 511, 1143, - /* 560 */ 1762, 1967, 1620, 505, 1809, 1810, 1762, 1389, 1390, 1936, - /* 570 */ 1439, 1442, 1443, 1444, 1445, 1446, 1447, 1448, 1449, 615, - /* 580 */ 611, 1458, 1459, 1461, 1462, 1463, 1464, 2, 45, 43, - /* 590 */ 1329, 1332, 1985, 1740, 84, 354, 362, 1936, 1385, 1531, - /* 600 */ 620, 1607, 605, 157, 2027, 1936, 1936, 619, 130, 1465, - /* 610 */ 228, 1383, 1764, 11, 174, 2063, 2064, 1757, 132, 2068, - /* 620 */ 38, 37, 1967, 2070, 44, 42, 41, 40, 39, 519, - /* 630 */ 561, 1966, 1413, 603, 1460, 2002, 669, 1562, 103, 1968, - /* 640 */ 623, 1970, 1971, 618, 529, 613, 1391, 126, 137, 2065, - /* 650 */ 143, 2026, 2055, 1985, 504, 478, 356, 2051, 226, 45, - /* 660 */ 43, 620, 259, 603, 1762, 377, 1936, 362, 619, 1385, - /* 670 */ 607, 46, 2027, 522, 1619, 1362, 1363, 415, 516, 1618, - /* 680 */ 1465, 184, 1383, 225, 1753, 60, 558, 1560, 1561, 1563, - /* 690 */ 1564, 576, 1966, 710, 1762, 1950, 2002, 510, 509, 161, - /* 700 */ 1968, 623, 1970, 1971, 618, 1460, 613, 547, 1467, 1468, - /* 710 */ 1738, 2124, 542, 368, 567, 562, 556, 1391, 1936, 67, - /* 720 */ 1739, 157, 66, 1936, 1946, 1952, 2130, 175, 1475, 1747, - /* 730 */ 1764, 2125, 572, 1617, 1410, 613, 649, 1440, 1450, 1806, - /* 740 */ 548, 2092, 14, 1466, 1469, 603, 38, 37, 1496, 1749, - /* 750 */ 44, 42, 41, 40, 39, 31, 169, 2128, 1386, 429, - /* 760 */ 1384, 38, 37, 503, 710, 44, 42, 41, 40, 39, - /* 770 */ 1745, 400, 657, 399, 1616, 237, 1762, 1936, 1863, 1467, - /* 780 */ 1468, 1737, 1615, 1389, 1390, 502, 1439, 1442, 1443, 1444, - /* 790 */ 1445, 1446, 1447, 1448, 1449, 615, 611, 1458, 1459, 1461, - /* 800 */ 1462, 1463, 1464, 2, 41, 40, 39, 603, 1440, 1450, - /* 810 */ 319, 50, 1408, 3, 1466, 1469, 365, 94, 1936, 437, - /* 820 */ 32, 430, 450, 232, 157, 449, 1936, 655, 614, 1386, - /* 830 */ 1501, 1384, 1614, 1764, 1551, 178, 681, 679, 1762, 1755, - /* 840 */ 421, 1955, 451, 647, 157, 423, 148, 147, 652, 651, - /* 850 */ 650, 145, 1950, 1765, 1389, 1390, 1410, 1439, 1442, 1443, - /* 860 */ 1444, 1445, 1446, 1447, 1448, 1449, 615, 611, 1458, 1459, - /* 870 */ 1461, 1462, 1463, 1464, 2, 1441, 1936, 670, 1413, 1732, - /* 880 */ 158, 1946, 1952, 33, 1845, 296, 335, 1675, 655, 38, - /* 890 */ 37, 1858, 613, 44, 42, 41, 40, 39, 411, 294, - /* 900 */ 70, 1815, 183, 69, 1858, 1858, 1622, 148, 147, 652, - /* 910 */ 651, 650, 145, 13, 12, 185, 189, 1814, 1099, 1100, - /* 920 */ 99, 195, 456, 454, 653, 2095, 654, 1806, 447, 1806, - /* 930 */ 96, 442, 441, 440, 439, 436, 435, 434, 433, 432, - /* 940 */ 428, 427, 426, 425, 336, 418, 417, 416, 589, 413, - /* 950 */ 412, 333, 687, 686, 685, 684, 372, 60, 683, 682, - /* 960 */ 138, 677, 676, 675, 674, 673, 672, 671, 150, 667, - /* 970 */ 666, 665, 371, 370, 662, 661, 660, 659, 658, 1613, - /* 980 */ 254, 603, 655, 1612, 290, 603, 375, 1792, 374, 578, - /* 990 */ 547, 2075, 1528, 603, 2124, 476, 102, 1441, 1717, 477, - /* 1000 */ 1967, 148, 147, 652, 651, 650, 145, 1759, 1508, 2130, - /* 1010 */ 175, 140, 1762, 128, 2125, 572, 1762, 218, 574, 71, - /* 1020 */ 216, 236, 220, 1936, 1762, 219, 52, 1936, 547, 603, - /* 1030 */ 547, 1985, 2124, 546, 2124, 424, 78, 77, 404, 586, - /* 1040 */ 583, 182, 603, 142, 1936, 1394, 619, 2130, 175, 2130, - /* 1050 */ 175, 559, 2125, 572, 2125, 572, 543, 1923, 535, 318, - /* 1060 */ 1762, 603, 392, 235, 390, 386, 382, 379, 376, 79, - /* 1070 */ 1966, 134, 1967, 1762, 2002, 587, 1393, 103, 1968, 623, - /* 1080 */ 1970, 1971, 618, 603, 613, 222, 62, 224, 221, 172, - /* 1090 */ 223, 2055, 1762, 1665, 211, 356, 2051, 271, 603, 603, - /* 1100 */ 547, 1658, 87, 1985, 2124, 384, 1656, 178, 603, 610, - /* 1110 */ 339, 620, 598, 600, 1762, 515, 1936, 2082, 619, 2130, - /* 1120 */ 175, 1957, 601, 517, 2125, 572, 1967, 603, 520, 1762, - /* 1130 */ 1762, 176, 2063, 2064, 241, 132, 2068, 248, 1559, 1762, - /* 1140 */ 47, 277, 1966, 1604, 1605, 1986, 2002, 264, 373, 103, - /* 1150 */ 1968, 623, 1970, 1971, 618, 663, 613, 1985, 1762, 68, - /* 1160 */ 144, 2144, 1867, 2055, 146, 620, 1646, 356, 2051, 1959, - /* 1170 */ 1936, 340, 619, 338, 337, 62, 501, 1161, 2089, 664, - /* 1180 */ 503, 1652, 1803, 603, 47, 47, 243, 1967, 13, 12, - /* 1190 */ 2085, 627, 1327, 584, 1397, 144, 1966, 369, 256, 269, - /* 1200 */ 2002, 1159, 502, 103, 1968, 623, 1970, 1971, 618, 253, - /* 1210 */ 613, 595, 273, 579, 1762, 2144, 1213, 2055, 1985, 1, - /* 1220 */ 146, 356, 2051, 129, 4, 1396, 620, 1502, 359, 358, - /* 1230 */ 705, 1936, 2102, 619, 378, 1349, 1451, 289, 1399, 144, - /* 1240 */ 383, 332, 284, 1241, 188, 410, 1413, 1245, 414, 1465, - /* 1250 */ 1967, 1392, 1868, 1486, 445, 419, 1408, 1966, 438, 444, - /* 1260 */ 431, 2002, 1860, 446, 103, 1968, 623, 1970, 1971, 618, - /* 1270 */ 1967, 613, 1252, 452, 1460, 1250, 2144, 453, 2055, 192, - /* 1280 */ 455, 1985, 356, 2051, 457, 1414, 1391, 458, 467, 620, - /* 1290 */ 1416, 149, 470, 554, 1936, 198, 619, 200, 1411, 471, - /* 1300 */ 1415, 1985, 472, 1417, 473, 203, 205, 475, 82, 620, - /* 1310 */ 83, 479, 1116, 209, 1936, 496, 619, 497, 500, 498, - /* 1320 */ 1966, 1752, 321, 106, 2002, 534, 215, 103, 1968, 623, - /* 1330 */ 1970, 1971, 618, 609, 613, 536, 285, 230, 1748, 2144, - /* 1340 */ 1966, 2055, 217, 151, 2002, 356, 2051, 103, 1968, 623, - /* 1350 */ 1970, 1971, 618, 152, 613, 1750, 2118, 1746, 153, 2144, - /* 1360 */ 154, 2055, 1913, 1912, 537, 356, 2051, 538, 233, 541, - /* 1370 */ 713, 544, 560, 1967, 551, 2086, 2074, 2101, 593, 557, - /* 1380 */ 2100, 346, 2096, 7, 283, 563, 569, 249, 239, 242, - /* 1390 */ 2077, 165, 247, 552, 550, 250, 549, 347, 1400, 168, - /* 1400 */ 1395, 580, 577, 1528, 1985, 703, 699, 695, 691, 281, - /* 1410 */ 251, 2147, 620, 255, 252, 2123, 133, 1936, 1412, 619, - /* 1420 */ 2071, 588, 261, 1403, 1405, 350, 591, 596, 286, 287, - /* 1430 */ 592, 1884, 1967, 1883, 1882, 352, 611, 1458, 1459, 1461, - /* 1440 */ 1462, 1463, 1464, 1966, 288, 101, 1763, 2002, 274, 597, - /* 1450 */ 103, 1968, 623, 1970, 1971, 618, 59, 613, 91, 93, - /* 1460 */ 2036, 95, 2030, 1985, 2055, 625, 280, 1807, 356, 2051, - /* 1470 */ 1733, 620, 291, 706, 707, 709, 1936, 315, 619, 51, - /* 1480 */ 295, 599, 1930, 323, 324, 293, 1967, 1929, 300, 75, - /* 1490 */ 1928, 314, 304, 1927, 76, 1924, 380, 381, 1377, 1378, - /* 1500 */ 181, 1922, 1966, 385, 387, 388, 2002, 389, 1921, 103, - /* 1510 */ 1968, 623, 1970, 1971, 618, 391, 613, 1985, 1920, 393, - /* 1520 */ 262, 2028, 1919, 2055, 395, 620, 397, 356, 2051, 1918, - /* 1530 */ 1936, 1352, 619, 1895, 1351, 1894, 402, 1356, 1893, 231, - /* 1540 */ 403, 1892, 1853, 1307, 1852, 1850, 139, 1967, 1849, 1848, - /* 1550 */ 1851, 1847, 1846, 1844, 1843, 1842, 1966, 186, 420, 1841, - /* 1560 */ 2002, 422, 1840, 103, 1968, 623, 1970, 1971, 618, 1839, - /* 1570 */ 613, 1838, 1837, 1836, 1835, 606, 1834, 2055, 1985, 1833, - /* 1580 */ 1832, 356, 2051, 1831, 1830, 1829, 620, 1828, 1827, 1826, - /* 1590 */ 1825, 1936, 1824, 619, 448, 1817, 1816, 1309, 141, 1823, - /* 1600 */ 1822, 1821, 1820, 1819, 1818, 1967, 1680, 1679, 1677, 1641, - /* 1610 */ 196, 170, 1188, 1956, 1640, 193, 194, 1966, 1908, 1102, - /* 1620 */ 73, 2002, 197, 1902, 104, 1968, 623, 1970, 1971, 618, - /* 1630 */ 1101, 613, 1891, 463, 1890, 204, 1985, 74, 2055, 1870, - /* 1640 */ 465, 1741, 2054, 2051, 620, 1676, 1674, 480, 1672, 1936, - /* 1650 */ 202, 619, 482, 1670, 484, 1135, 486, 488, 481, 1967, - /* 1660 */ 490, 1668, 492, 485, 494, 1655, 1654, 1637, 489, 1743, - /* 1670 */ 1257, 1256, 493, 1742, 61, 1966, 1171, 1179, 1178, 2002, - /* 1680 */ 678, 680, 104, 1968, 623, 1970, 1971, 618, 1967, 613, - /* 1690 */ 1985, 1177, 1176, 1173, 1172, 1170, 2055, 1666, 620, 214, - /* 1700 */ 608, 2051, 341, 1936, 1659, 619, 342, 1657, 343, 521, - /* 1710 */ 1636, 523, 518, 1635, 525, 1634, 527, 1367, 105, 1985, - /* 1720 */ 1366, 531, 1369, 26, 1907, 1358, 55, 617, 1901, 621, - /* 1730 */ 539, 155, 1936, 2002, 619, 1889, 104, 1968, 623, 1970, - /* 1740 */ 1971, 618, 1887, 613, 2129, 28, 1967, 19, 16, 553, - /* 1750 */ 2055, 1574, 246, 555, 326, 2051, 58, 63, 1966, 238, - /* 1760 */ 245, 1957, 2002, 30, 240, 312, 1968, 623, 1970, 1971, - /* 1770 */ 618, 616, 613, 604, 2020, 21, 163, 1985, 1589, 20, - /* 1780 */ 17, 1588, 348, 1558, 244, 620, 29, 1593, 1550, 1594, - /* 1790 */ 1936, 88, 619, 540, 234, 1595, 344, 1592, 349, 1888, - /* 1800 */ 1967, 57, 545, 258, 56, 1525, 166, 1524, 5, 1886, - /* 1810 */ 6, 1885, 22, 594, 263, 265, 1966, 1556, 270, 1869, - /* 1820 */ 2002, 65, 90, 162, 1968, 623, 1970, 1971, 618, 92, - /* 1830 */ 613, 1985, 96, 275, 272, 23, 12, 1401, 1432, 620, - /* 1840 */ 2005, 167, 179, 612, 1936, 1455, 619, 1453, 36, 626, - /* 1850 */ 624, 1234, 364, 15, 1967, 630, 1452, 1424, 1487, 24, - /* 1860 */ 25, 622, 1242, 628, 633, 1239, 631, 636, 639, 1236, - /* 1870 */ 1966, 634, 1477, 1230, 2002, 573, 2145, 104, 1968, 623, - /* 1880 */ 1970, 1971, 618, 637, 613, 1985, 1228, 640, 1219, 1233, - /* 1890 */ 278, 2055, 646, 620, 1232, 97, 2052, 1231, 1936, 98, - /* 1900 */ 619, 10, 1251, 1476, 1247, 72, 1967, 1133, 656, 1167, - /* 1910 */ 1166, 1165, 1164, 1163, 1162, 1160, 1158, 1186, 1157, 1156, - /* 1920 */ 668, 1154, 1967, 1153, 1966, 1152, 1151, 1183, 2002, 279, - /* 1930 */ 1150, 161, 1968, 623, 1970, 1971, 618, 1985, 613, 1139, - /* 1940 */ 1149, 1148, 1181, 1145, 1144, 620, 1141, 1140, 1138, 1673, - /* 1950 */ 1936, 688, 619, 1985, 690, 1671, 692, 694, 689, 1669, - /* 1960 */ 696, 620, 693, 698, 1667, 700, 1936, 697, 619, 1653, - /* 1970 */ 701, 702, 704, 2093, 1091, 1633, 1966, 712, 282, 708, - /* 1980 */ 2002, 1608, 1387, 306, 1968, 623, 1970, 1971, 618, 711, - /* 1990 */ 613, 1608, 1966, 292, 1608, 1608, 2002, 1608, 1608, 162, - /* 2000 */ 1968, 623, 1970, 1971, 618, 1608, 613, 1608, 1608, 1608, - /* 2010 */ 1608, 1967, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 2020 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 568, 1608, 1967, - /* 2030 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 2040 */ 1608, 1608, 1985, 1608, 1608, 1608, 1608, 351, 1608, 1608, - /* 2050 */ 620, 1608, 2146, 1608, 1608, 1936, 1608, 619, 1608, 1608, - /* 2060 */ 1985, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 617, 1608, - /* 2070 */ 1608, 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, 1608, - /* 2080 */ 1608, 1966, 1608, 1967, 1608, 2002, 1608, 1608, 313, 1968, - /* 2090 */ 623, 1970, 1971, 618, 1608, 613, 1608, 1608, 1608, 1966, - /* 2100 */ 1967, 1608, 1608, 2002, 1608, 1608, 312, 1968, 623, 1970, - /* 2110 */ 1971, 618, 1608, 613, 1985, 2021, 1608, 1608, 1608, 361, - /* 2120 */ 1608, 1608, 620, 1608, 1608, 1608, 1608, 1936, 1608, 619, - /* 2130 */ 1608, 1985, 1608, 1608, 1608, 1608, 363, 1608, 1608, 620, - /* 2140 */ 1608, 1608, 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, - /* 2150 */ 1608, 1608, 1967, 1966, 1608, 1608, 1608, 2002, 1608, 1608, - /* 2160 */ 313, 1968, 623, 1970, 1971, 618, 1608, 613, 1967, 1608, - /* 2170 */ 1966, 1608, 1608, 1608, 2002, 1608, 1608, 313, 1968, 623, - /* 2180 */ 1970, 1971, 618, 1985, 613, 1608, 1608, 1608, 1608, 1608, - /* 2190 */ 1608, 620, 1608, 1608, 1608, 1608, 1936, 1608, 619, 1985, - /* 2200 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 620, 1608, 1608, - /* 2210 */ 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, 1608, 1608, - /* 2220 */ 1967, 1608, 533, 1608, 1608, 1608, 2002, 1608, 1608, 308, - /* 2230 */ 1968, 623, 1970, 1971, 618, 1608, 613, 1608, 1966, 1608, - /* 2240 */ 1608, 1608, 2002, 1608, 1608, 297, 1968, 623, 1970, 1971, - /* 2250 */ 618, 1985, 613, 1608, 1608, 1608, 1608, 1608, 1608, 620, - /* 2260 */ 1608, 1608, 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, - /* 2270 */ 1608, 1608, 1608, 1608, 1967, 1608, 1608, 1608, 1608, 1608, - /* 2280 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 2290 */ 1966, 1608, 1608, 1608, 2002, 1967, 1608, 298, 1968, 623, - /* 2300 */ 1970, 1971, 618, 1608, 613, 1985, 1608, 1608, 1608, 1608, - /* 2310 */ 1608, 1608, 1608, 620, 1608, 1608, 1608, 1608, 1936, 1608, - /* 2320 */ 619, 1608, 1608, 1608, 1608, 1608, 1985, 1608, 1608, 1608, - /* 2330 */ 1608, 1608, 1608, 1608, 620, 1608, 1608, 1608, 1608, 1936, - /* 2340 */ 1608, 619, 1608, 1608, 1966, 1608, 1608, 1608, 2002, 1608, - /* 2350 */ 1608, 299, 1968, 623, 1970, 1971, 618, 1608, 613, 1608, - /* 2360 */ 1608, 1608, 1608, 1608, 1608, 1966, 1608, 1608, 1608, 2002, - /* 2370 */ 1608, 1967, 305, 1968, 623, 1970, 1971, 618, 1608, 613, - /* 2380 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1967, - /* 2390 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 2400 */ 1608, 1608, 1985, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 2410 */ 620, 1608, 1608, 1608, 1608, 1936, 1608, 619, 1608, 1608, - /* 2420 */ 1985, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 620, 1608, - /* 2430 */ 1608, 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, 1608, - /* 2440 */ 1608, 1966, 1608, 1967, 1608, 2002, 1608, 1608, 309, 1968, - /* 2450 */ 623, 1970, 1971, 618, 1608, 613, 1608, 1608, 1608, 1966, - /* 2460 */ 1967, 1608, 1608, 2002, 1608, 1608, 301, 1968, 623, 1970, - /* 2470 */ 1971, 618, 1608, 613, 1985, 1608, 1608, 1608, 1608, 1608, - /* 2480 */ 1608, 1608, 620, 1608, 1608, 1608, 1608, 1936, 1608, 619, - /* 2490 */ 1608, 1985, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 620, - /* 2500 */ 1608, 1608, 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, - /* 2510 */ 1608, 1608, 1967, 1966, 1608, 1608, 1608, 2002, 1608, 1608, - /* 2520 */ 310, 1968, 623, 1970, 1971, 618, 1608, 613, 1967, 1608, - /* 2530 */ 1966, 1608, 1608, 1608, 2002, 1608, 1608, 302, 1968, 623, - /* 2540 */ 1970, 1971, 618, 1985, 613, 1608, 1608, 1608, 1608, 1608, - /* 2550 */ 1608, 620, 1608, 1608, 1608, 1608, 1936, 1608, 619, 1985, - /* 2560 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 620, 1608, 1608, - /* 2570 */ 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, 1608, 1608, - /* 2580 */ 1967, 1608, 1966, 1608, 1608, 1608, 2002, 1608, 1608, 311, - /* 2590 */ 1968, 623, 1970, 1971, 618, 1608, 613, 1608, 1966, 1608, - /* 2600 */ 1608, 1608, 2002, 1967, 1608, 303, 1968, 623, 1970, 1971, - /* 2610 */ 618, 1985, 613, 1608, 1608, 1608, 1608, 1608, 1608, 620, - /* 2620 */ 1608, 1608, 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, - /* 2630 */ 1608, 1608, 1608, 1608, 1985, 1608, 1608, 1608, 1608, 1608, - /* 2640 */ 1608, 1608, 620, 1608, 1608, 1608, 1608, 1936, 1608, 619, - /* 2650 */ 1966, 1608, 1608, 1608, 2002, 1967, 1608, 316, 1968, 623, - /* 2660 */ 1970, 1971, 618, 1608, 613, 1608, 1608, 1608, 1608, 1608, - /* 2670 */ 1608, 1608, 1608, 1966, 1608, 1608, 1608, 2002, 1608, 1608, - /* 2680 */ 317, 1968, 623, 1970, 1971, 618, 1985, 613, 1608, 1608, - /* 2690 */ 1608, 1608, 1608, 1608, 620, 1608, 1608, 1608, 1608, 1936, - /* 2700 */ 1608, 619, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 2710 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1967, 1608, - /* 2720 */ 1608, 1608, 1608, 1608, 1608, 1966, 1608, 1608, 1608, 2002, - /* 2730 */ 1608, 1608, 1979, 1968, 623, 1970, 1971, 618, 1608, 613, - /* 2740 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1985, - /* 2750 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 620, 1608, 1608, - /* 2760 */ 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, 1608, 1608, - /* 2770 */ 1967, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 2780 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1966, 1967, - /* 2790 */ 1608, 1608, 2002, 1608, 1608, 1978, 1968, 623, 1970, 1971, - /* 2800 */ 618, 1985, 613, 1608, 1608, 1608, 1608, 1608, 1608, 620, - /* 2810 */ 1608, 1608, 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, - /* 2820 */ 1985, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 620, 1608, - /* 2830 */ 1608, 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, 1608, - /* 2840 */ 1966, 1967, 1608, 1608, 2002, 1608, 1608, 1977, 1968, 623, - /* 2850 */ 1970, 1971, 618, 1608, 613, 1608, 1608, 1967, 1608, 1966, - /* 2860 */ 1608, 1608, 1608, 2002, 1608, 1608, 328, 1968, 623, 1970, - /* 2870 */ 1971, 618, 1985, 613, 1608, 1608, 1608, 1608, 1608, 1608, - /* 2880 */ 620, 1608, 1608, 1608, 1608, 1936, 1608, 619, 1985, 1608, - /* 2890 */ 1608, 1608, 1608, 1608, 1608, 1608, 620, 1608, 1608, 1608, - /* 2900 */ 1608, 1936, 1608, 619, 1608, 1608, 1608, 1608, 1608, 1967, - /* 2910 */ 1608, 1966, 1608, 1608, 1608, 2002, 1608, 1608, 329, 1968, - /* 2920 */ 623, 1970, 1971, 618, 1608, 613, 1608, 1966, 1608, 1608, - /* 2930 */ 1608, 2002, 1967, 1608, 325, 1968, 623, 1970, 1971, 618, - /* 2940 */ 1985, 613, 1608, 1608, 1608, 1608, 1608, 1608, 620, 1608, - /* 2950 */ 1608, 1608, 1608, 1936, 1608, 619, 1608, 1608, 1608, 1608, - /* 2960 */ 1608, 1608, 1608, 1985, 1608, 1608, 1608, 1608, 1608, 1608, - /* 2970 */ 1608, 620, 1608, 1608, 1608, 1608, 1936, 1608, 619, 1966, - /* 2980 */ 1608, 1608, 1608, 2002, 1967, 1608, 330, 1968, 623, 1970, - /* 2990 */ 1971, 618, 1608, 613, 1608, 1608, 1608, 1608, 1608, 1608, - /* 3000 */ 1608, 1608, 621, 1608, 1608, 1608, 2002, 1608, 1608, 308, - /* 3010 */ 1968, 623, 1970, 1971, 618, 1985, 613, 1608, 1608, 1608, - /* 3020 */ 1608, 1608, 1608, 620, 1608, 1608, 1608, 1608, 1936, 1608, - /* 3030 */ 619, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 3040 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 3050 */ 1608, 1608, 1608, 1608, 1966, 1608, 1608, 1608, 2002, 1608, - /* 3060 */ 1608, 307, 1968, 623, 1970, 1971, 618, 1608, 613, + /* 0 */ 35, 282, 1755, 1892, 579, 1892, 233, 2147, 2142, 1807, + /* 10 */ 1972, 2142, 45, 43, 1550, 361, 1890, 601, 1889, 601, + /* 20 */ 370, 1968, 1400, 578, 179, 613, 1768, 2146, 2143, 580, + /* 30 */ 1985, 2143, 2145, 1480, 1554, 1398, 468, 1968, 469, 1660, + /* 40 */ 1425, 613, 38, 37, 1972, 2088, 44, 42, 41, 40, + /* 50 */ 39, 1964, 1970, 352, 474, 1968, 409, 591, 1475, 1425, + /* 60 */ 470, 2003, 624, 18, 486, 1830, 1830, 1964, 1970, 594, + /* 70 */ 1406, 2085, 350, 337, 1954, 84, 630, 477, 624, 469, + /* 80 */ 1660, 1828, 1828, 45, 43, 1964, 1970, 365, 138, 134, + /* 90 */ 2147, 370, 163, 1400, 1636, 14, 624, 333, 1772, 555, + /* 100 */ 48, 1984, 2088, 2142, 1480, 2020, 1398, 1427, 107, 1986, + /* 110 */ 634, 1988, 1989, 629, 84, 624, 527, 721, 2148, 179, + /* 120 */ 176, 614, 2073, 2143, 580, 173, 364, 2069, 2084, 1475, + /* 130 */ 613, 537, 1482, 1483, 18, 130, 2003, 1773, 1509, 579, + /* 140 */ 181, 1406, 507, 2142, 573, 230, 340, 1877, 2099, 262, + /* 150 */ 2081, 590, 1777, 131, 589, 614, 1427, 2142, 578, 179, + /* 160 */ 530, 1455, 1465, 2143, 580, 524, 14, 1481, 1484, 130, + /* 170 */ 229, 375, 578, 179, 1823, 1825, 512, 2143, 580, 1873, + /* 180 */ 173, 1456, 1401, 1610, 1399, 572, 1777, 486, 721, 1106, + /* 190 */ 187, 38, 37, 232, 1510, 44, 42, 41, 40, 39, + /* 200 */ 1328, 1329, 1878, 1482, 1483, 48, 67, 1404, 1405, 66, + /* 210 */ 1454, 1457, 1458, 1459, 1460, 1461, 1462, 1463, 1464, 626, + /* 220 */ 622, 1473, 1474, 1476, 1477, 1478, 1479, 2, 1108, 64, + /* 230 */ 1111, 1112, 1455, 1465, 41, 40, 39, 120, 1481, 1484, + /* 240 */ 119, 118, 117, 116, 115, 114, 113, 112, 111, 164, + /* 250 */ 264, 345, 175, 1401, 1731, 1399, 38, 37, 1377, 1378, + /* 260 */ 44, 42, 41, 40, 39, 1817, 34, 368, 1504, 1505, + /* 270 */ 1506, 1507, 1508, 1512, 1513, 1514, 1515, 191, 1404, 1405, + /* 280 */ 1425, 1454, 1457, 1458, 1459, 1460, 1461, 1462, 1463, 1464, + /* 290 */ 626, 622, 1473, 1474, 1476, 1477, 1478, 1479, 2, 264, + /* 300 */ 11, 45, 43, 1693, 44, 42, 41, 40, 39, 370, + /* 310 */ 1754, 1400, 346, 81, 344, 343, 80, 509, 574, 1985, + /* 320 */ 60, 511, 1480, 1647, 1398, 1232, 656, 655, 654, 1236, + /* 330 */ 653, 1238, 1239, 652, 1241, 649, 1192, 1247, 646, 1249, + /* 340 */ 1250, 643, 640, 510, 1587, 467, 265, 1475, 472, 1666, + /* 350 */ 2003, 536, 18, 1824, 1825, 182, 1426, 1617, 631, 1406, + /* 360 */ 1125, 8, 1124, 1954, 534, 630, 532, 1954, 522, 521, + /* 370 */ 520, 1194, 45, 43, 1485, 216, 135, 516, 1424, 1425, + /* 380 */ 370, 515, 1400, 60, 14, 90, 514, 519, 2088, 1830, + /* 390 */ 1984, 1126, 513, 1480, 2020, 1398, 358, 107, 1986, 634, + /* 400 */ 1988, 1989, 629, 1985, 624, 1828, 721, 141, 2147, 147, + /* 410 */ 2044, 2073, 2142, 31, 2083, 364, 2069, 666, 1475, 38, + /* 420 */ 37, 1482, 1483, 44, 42, 41, 40, 39, 2146, 1753, + /* 430 */ 1406, 1941, 2143, 2144, 2003, 1624, 152, 151, 663, 662, + /* 440 */ 661, 149, 628, 273, 274, 518, 517, 1954, 272, 630, + /* 450 */ 1455, 1465, 1616, 569, 60, 46, 1481, 1484, 415, 129, + /* 460 */ 128, 127, 126, 125, 124, 123, 122, 121, 1275, 1276, + /* 470 */ 182, 1401, 476, 1399, 1984, 472, 1666, 721, 2020, 392, + /* 480 */ 1646, 318, 1986, 634, 1988, 1989, 629, 627, 624, 615, + /* 490 */ 2038, 668, 1482, 1483, 692, 690, 1404, 1405, 326, 1454, + /* 500 */ 1457, 1458, 1459, 1460, 1461, 1462, 1463, 1464, 626, 622, + /* 510 */ 1473, 1474, 1476, 1477, 1478, 1479, 2, 522, 521, 520, + /* 520 */ 614, 1455, 1465, 362, 1954, 135, 516, 1481, 1484, 1625, + /* 530 */ 515, 161, 1830, 182, 54, 514, 519, 575, 570, 564, + /* 540 */ 1779, 513, 1401, 404, 1399, 60, 1622, 98, 1829, 140, + /* 550 */ 120, 1777, 2044, 119, 118, 117, 116, 115, 114, 113, + /* 560 */ 112, 111, 1985, 60, 406, 402, 1543, 1404, 1405, 1770, + /* 570 */ 1454, 1457, 1458, 1459, 1460, 1461, 1462, 1463, 1464, 626, + /* 580 */ 622, 1473, 1474, 1476, 1477, 1478, 1479, 2, 45, 43, + /* 590 */ 1400, 1645, 616, 2003, 2045, 1125, 370, 1124, 1400, 660, + /* 600 */ 614, 631, 1821, 1398, 182, 49, 1954, 668, 630, 1480, + /* 610 */ 385, 1398, 659, 231, 184, 13, 12, 38, 37, 1985, + /* 620 */ 1830, 44, 42, 41, 40, 39, 1126, 363, 11, 1428, + /* 630 */ 591, 1777, 1830, 1984, 1475, 1954, 1828, 2020, 1406, 374, + /* 640 */ 312, 1986, 634, 1988, 1989, 629, 1406, 624, 1828, 408, + /* 650 */ 2003, 407, 614, 555, 1860, 2146, 614, 2142, 631, 45, + /* 660 */ 43, 138, 11, 1954, 9, 630, 413, 370, 1428, 1400, + /* 670 */ 414, 46, 2148, 179, 104, 87, 328, 2143, 580, 540, + /* 680 */ 1480, 538, 1398, 1777, 576, 721, 597, 1777, 139, 614, + /* 690 */ 1984, 1644, 1643, 721, 2020, 182, 1769, 108, 1986, 634, + /* 700 */ 1988, 1989, 629, 423, 624, 1475, 1752, 621, 1482, 1483, + /* 710 */ 1523, 2073, 1425, 182, 614, 2072, 2069, 1406, 416, 451, + /* 720 */ 1777, 593, 177, 2081, 2082, 1406, 136, 2086, 437, 555, + /* 730 */ 1642, 417, 614, 2142, 1511, 1954, 1954, 1455, 1465, 373, + /* 740 */ 1873, 1641, 14, 1481, 1484, 1777, 438, 161, 2148, 179, + /* 750 */ 1401, 189, 1399, 2143, 580, 182, 1779, 681, 1401, 1747, + /* 760 */ 1399, 38, 37, 1777, 721, 44, 42, 41, 40, 39, + /* 770 */ 1426, 1873, 242, 1640, 1954, 1404, 1405, 195, 194, 1482, + /* 780 */ 1483, 1547, 193, 1404, 1405, 1954, 1454, 1457, 1458, 1459, + /* 790 */ 1460, 1461, 1462, 1463, 1464, 626, 622, 1473, 1474, 1476, + /* 800 */ 1477, 1478, 1479, 2, 1639, 432, 32, 614, 1455, 1465, + /* 810 */ 325, 188, 1423, 666, 1481, 1484, 1516, 1954, 618, 445, + /* 820 */ 2045, 484, 458, 1766, 664, 457, 1577, 1821, 50, 1401, + /* 830 */ 3, 1399, 152, 151, 663, 662, 661, 149, 1777, 376, + /* 840 */ 429, 1428, 459, 161, 1490, 431, 680, 161, 1954, 1566, + /* 850 */ 1425, 1501, 1780, 1456, 1404, 1405, 1779, 1454, 1457, 1458, + /* 860 */ 1459, 1460, 1461, 1462, 1463, 1464, 626, 622, 1473, 1474, + /* 870 */ 1476, 1477, 1478, 1479, 2, 566, 1575, 1576, 1578, 1579, + /* 880 */ 162, 665, 1343, 1344, 1821, 302, 341, 1690, 614, 38, + /* 890 */ 37, 1111, 1112, 44, 42, 41, 40, 39, 419, 300, + /* 900 */ 70, 33, 485, 69, 586, 511, 296, 38, 37, 1807, + /* 910 */ 103, 44, 42, 41, 40, 39, 582, 1342, 1345, 1777, + /* 920 */ 100, 199, 464, 462, 144, 27, 132, 510, 455, 2093, + /* 930 */ 1543, 450, 449, 448, 447, 444, 443, 442, 441, 440, + /* 940 */ 436, 435, 434, 433, 342, 426, 425, 424, 1638, 421, + /* 950 */ 420, 339, 698, 697, 696, 695, 380, 60, 694, 693, + /* 960 */ 142, 688, 687, 686, 685, 684, 683, 682, 154, 678, + /* 970 */ 677, 676, 379, 378, 673, 672, 671, 670, 669, 614, + /* 980 */ 724, 38, 37, 150, 241, 44, 42, 41, 40, 39, + /* 990 */ 1635, 1456, 1954, 1774, 289, 150, 106, 38, 37, 1680, + /* 1000 */ 1985, 44, 42, 41, 40, 39, 71, 1732, 222, 172, + /* 1010 */ 1777, 220, 614, 1634, 614, 714, 710, 706, 702, 287, + /* 1020 */ 1762, 523, 583, 1546, 224, 1633, 234, 223, 551, 666, + /* 1030 */ 226, 2003, 1632, 225, 1954, 53, 78, 77, 412, 594, + /* 1040 */ 1764, 186, 1631, 1777, 1954, 1777, 630, 1375, 152, 151, + /* 1050 */ 663, 662, 661, 149, 1409, 105, 79, 1954, 280, 324, + /* 1060 */ 550, 1973, 400, 614, 398, 394, 390, 387, 384, 1954, + /* 1070 */ 614, 1984, 1968, 1985, 614, 2020, 1954, 595, 107, 1986, + /* 1080 */ 634, 1988, 1989, 629, 599, 624, 1954, 1637, 600, 1630, + /* 1090 */ 176, 610, 2073, 1985, 1777, 614, 364, 2069, 1673, 383, + /* 1100 */ 614, 1777, 1964, 1970, 2003, 1777, 1629, 182, 228, 277, + /* 1110 */ 2113, 227, 631, 624, 609, 1628, 1760, 1954, 2100, 630, + /* 1120 */ 525, 1671, 1619, 1620, 2003, 268, 1777, 52, 587, 62, + /* 1130 */ 267, 1777, 631, 1954, 554, 246, 160, 1954, 1627, 630, + /* 1140 */ 150, 237, 555, 528, 1984, 47, 2142, 1369, 2020, 236, + /* 1150 */ 1954, 107, 1986, 634, 1988, 1989, 629, 240, 624, 1954, + /* 1160 */ 1985, 2148, 179, 2162, 1984, 2073, 2143, 580, 2020, 364, + /* 1170 */ 2069, 107, 1986, 634, 1988, 1989, 629, 625, 624, 614, + /* 1180 */ 2107, 1574, 1954, 2162, 614, 2073, 382, 248, 1408, 364, + /* 1190 */ 2069, 2003, 598, 611, 614, 614, 88, 1340, 377, 631, + /* 1200 */ 2120, 13, 12, 1412, 1954, 658, 630, 259, 612, 283, + /* 1210 */ 1777, 567, 270, 367, 366, 1777, 1985, 215, 68, 1882, + /* 1220 */ 148, 381, 543, 1414, 150, 1777, 1777, 62, 47, 555, + /* 1230 */ 47, 1984, 591, 2142, 1480, 2020, 1407, 1667, 107, 1986, + /* 1240 */ 634, 1988, 1989, 629, 584, 624, 1154, 2003, 2148, 179, + /* 1250 */ 2162, 638, 2073, 2143, 580, 631, 364, 2069, 148, 1475, + /* 1260 */ 1954, 1985, 630, 138, 275, 555, 1975, 562, 150, 2142, + /* 1270 */ 606, 1406, 279, 591, 253, 133, 1225, 2004, 148, 1517, + /* 1280 */ 1466, 1155, 295, 1661, 2148, 179, 716, 1984, 1818, 2143, + /* 1290 */ 580, 2020, 2003, 674, 107, 1986, 634, 1988, 1989, 629, + /* 1300 */ 631, 624, 2103, 1253, 138, 1954, 2162, 630, 2073, 675, + /* 1310 */ 1257, 592, 364, 2069, 1977, 1173, 261, 258, 620, 4, + /* 1320 */ 1264, 1, 386, 2136, 178, 2081, 2082, 1262, 136, 2086, + /* 1330 */ 153, 1171, 1984, 391, 338, 1362, 2020, 1411, 290, 107, + /* 1340 */ 1986, 634, 1988, 1989, 629, 192, 624, 418, 1428, 1883, + /* 1350 */ 422, 2162, 453, 2073, 427, 1423, 439, 364, 2069, 1875, + /* 1360 */ 1985, 446, 452, 454, 460, 180, 2081, 2082, 2092, 136, + /* 1370 */ 2086, 461, 196, 463, 465, 1429, 466, 475, 1431, 478, + /* 1380 */ 1426, 202, 479, 1415, 204, 1410, 1430, 480, 1432, 483, + /* 1390 */ 481, 2003, 487, 207, 1128, 209, 82, 83, 504, 631, + /* 1400 */ 213, 505, 508, 506, 1954, 1931, 630, 1767, 1418, 1420, + /* 1410 */ 110, 327, 1930, 542, 544, 86, 219, 291, 146, 1763, + /* 1420 */ 221, 622, 1473, 1474, 1476, 1477, 1478, 1479, 155, 156, + /* 1430 */ 1765, 1984, 235, 1985, 1761, 2020, 157, 158, 107, 1986, + /* 1440 */ 634, 1988, 1989, 629, 545, 624, 238, 552, 2104, 2114, + /* 1450 */ 2048, 559, 2073, 1985, 549, 2119, 364, 2069, 546, 568, + /* 1460 */ 604, 244, 565, 7, 2003, 353, 2118, 247, 577, 571, + /* 1470 */ 2095, 560, 631, 252, 558, 255, 169, 1954, 557, 630, + /* 1480 */ 354, 585, 254, 256, 2003, 257, 2141, 2165, 588, 1543, + /* 1490 */ 137, 1427, 631, 2089, 596, 292, 357, 1954, 602, 630, + /* 1500 */ 266, 93, 603, 260, 1984, 1900, 1899, 1898, 2020, 1985, + /* 1510 */ 360, 107, 1986, 634, 1988, 1989, 629, 293, 624, 607, + /* 1520 */ 608, 294, 95, 2046, 1984, 2073, 59, 97, 2020, 364, + /* 1530 */ 2069, 107, 1986, 634, 1988, 1989, 629, 1778, 624, 2054, + /* 1540 */ 2003, 99, 636, 617, 1822, 2073, 1748, 717, 631, 364, + /* 1550 */ 2069, 297, 720, 1954, 718, 630, 51, 329, 286, 321, + /* 1560 */ 1948, 330, 306, 1947, 1985, 301, 320, 299, 75, 1946, + /* 1570 */ 1945, 1942, 76, 310, 388, 389, 1392, 1393, 185, 393, + /* 1580 */ 1984, 1940, 395, 396, 2020, 397, 1939, 108, 1986, 634, + /* 1590 */ 1988, 1989, 629, 399, 624, 2003, 401, 1937, 403, 1936, + /* 1600 */ 405, 2073, 1938, 631, 1365, 619, 2069, 1364, 1954, 1911, + /* 1610 */ 630, 1910, 410, 411, 1909, 1908, 1868, 1867, 1865, 1319, + /* 1620 */ 1985, 143, 1864, 1863, 1866, 1862, 1861, 1859, 1858, 1857, + /* 1630 */ 190, 1856, 430, 428, 1855, 632, 1985, 1854, 1853, 2020, + /* 1640 */ 1852, 1851, 108, 1986, 634, 1988, 1989, 629, 1850, 624, + /* 1650 */ 1849, 2003, 145, 1840, 1839, 1838, 2073, 1848, 1847, 631, + /* 1660 */ 332, 2069, 1846, 1845, 1954, 1844, 630, 2003, 1321, 1835, + /* 1670 */ 1834, 1833, 456, 1832, 1831, 631, 1843, 1842, 1841, 1837, + /* 1680 */ 1954, 1836, 630, 1695, 1200, 1694, 1692, 1656, 197, 200, + /* 1690 */ 198, 1984, 1974, 1655, 1114, 2020, 1113, 1985, 165, 1986, + /* 1700 */ 634, 1988, 1989, 629, 174, 624, 73, 1984, 471, 1924, + /* 1710 */ 1918, 2020, 201, 473, 166, 1986, 634, 1988, 1989, 629, + /* 1720 */ 1907, 624, 208, 74, 1906, 1886, 1756, 206, 2003, 488, + /* 1730 */ 490, 1687, 492, 1685, 494, 496, 631, 1691, 1689, 556, + /* 1740 */ 2110, 1954, 498, 630, 489, 1683, 493, 497, 500, 501, + /* 1750 */ 502, 1670, 1985, 1147, 1669, 1652, 1758, 1269, 1268, 1757, + /* 1760 */ 61, 218, 1191, 1190, 1183, 1189, 581, 2163, 1984, 1985, + /* 1770 */ 1188, 1681, 2020, 1674, 1185, 108, 1986, 634, 1988, 1989, + /* 1780 */ 629, 1184, 624, 2003, 689, 691, 347, 1182, 348, 2073, + /* 1790 */ 1672, 631, 349, 526, 2070, 529, 1954, 1651, 630, 1650, + /* 1800 */ 2003, 535, 531, 533, 1649, 109, 539, 1382, 631, 1381, + /* 1810 */ 1384, 26, 1923, 1954, 55, 630, 1371, 1917, 159, 1905, + /* 1820 */ 547, 1903, 19, 1984, 16, 2147, 563, 2020, 1589, 239, + /* 1830 */ 165, 1986, 634, 1988, 1989, 629, 561, 624, 1985, 28, + /* 1840 */ 1984, 251, 243, 167, 2020, 245, 553, 166, 1986, 634, + /* 1850 */ 1988, 1989, 629, 548, 624, 351, 249, 1985, 58, 1573, + /* 1860 */ 29, 1604, 250, 1975, 30, 1565, 63, 21, 20, 2003, + /* 1870 */ 89, 1603, 2111, 1609, 359, 5, 6, 631, 1610, 355, + /* 1880 */ 1608, 1607, 1954, 356, 630, 17, 263, 1540, 2003, 1539, + /* 1890 */ 170, 57, 1904, 1902, 1901, 1885, 628, 91, 92, 269, + /* 1900 */ 2164, 1954, 22, 630, 1571, 271, 276, 1884, 65, 1984, + /* 1910 */ 94, 100, 281, 2020, 278, 12, 319, 1986, 634, 1988, + /* 1920 */ 1989, 629, 96, 624, 1985, 23, 1416, 2023, 1984, 1470, + /* 1930 */ 1502, 1468, 2020, 623, 10, 318, 1986, 634, 1988, 1989, + /* 1940 */ 629, 1447, 624, 1492, 2039, 56, 605, 1491, 36, 171, + /* 1950 */ 1467, 217, 15, 183, 1231, 2003, 24, 635, 1439, 25, + /* 1960 */ 369, 637, 633, 631, 1254, 372, 168, 639, 1954, 1251, + /* 1970 */ 630, 641, 503, 499, 495, 491, 214, 642, 644, 1248, + /* 1980 */ 1985, 645, 647, 1242, 648, 650, 1240, 657, 1246, 1245, + /* 1990 */ 284, 651, 101, 1244, 102, 1984, 1985, 1243, 1263, 2020, + /* 2000 */ 72, 1259, 319, 1986, 634, 1988, 1989, 629, 1145, 624, + /* 2010 */ 1179, 2003, 85, 1178, 667, 212, 371, 1177, 1176, 631, + /* 2020 */ 1175, 1174, 1172, 1170, 1954, 1169, 630, 2003, 1168, 1198, + /* 2030 */ 679, 1166, 1163, 285, 1165, 631, 1164, 1162, 1161, 1160, + /* 2040 */ 1954, 1195, 630, 1193, 1157, 1151, 1156, 1153, 1152, 1150, + /* 2050 */ 1688, 1984, 699, 700, 701, 2020, 1686, 1985, 319, 1986, + /* 2060 */ 634, 1988, 1989, 629, 703, 624, 704, 541, 1684, 705, + /* 2070 */ 707, 2020, 708, 709, 314, 1986, 634, 1988, 1989, 629, + /* 2080 */ 1682, 624, 211, 205, 711, 712, 713, 210, 2003, 1668, + /* 2090 */ 482, 1648, 715, 288, 1103, 719, 631, 1402, 298, 722, + /* 2100 */ 1623, 1954, 723, 630, 1623, 1623, 203, 1623, 1623, 1623, + /* 2110 */ 1623, 1623, 1985, 1623, 1623, 1623, 1623, 1623, 1623, 1623, + /* 2120 */ 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1984, 1985, + /* 2130 */ 1623, 1623, 2020, 1623, 1623, 303, 1986, 634, 1988, 1989, + /* 2140 */ 629, 1623, 624, 2003, 1623, 1623, 1623, 1623, 1623, 1623, + /* 2150 */ 1623, 631, 1623, 1623, 1623, 1623, 1954, 1623, 630, 1623, + /* 2160 */ 2003, 1623, 1623, 1623, 1623, 1623, 1623, 1623, 631, 1623, + /* 2170 */ 1623, 1623, 1623, 1954, 1623, 630, 1623, 1623, 1623, 1623, + /* 2180 */ 1623, 1623, 1623, 1984, 1623, 1623, 1985, 2020, 1623, 1623, + /* 2190 */ 304, 1986, 634, 1988, 1989, 629, 1623, 624, 1623, 1623, + /* 2200 */ 1984, 1623, 1623, 1985, 2020, 1623, 1623, 305, 1986, 634, + /* 2210 */ 1988, 1989, 629, 1623, 624, 1623, 1623, 2003, 1623, 1623, + /* 2220 */ 1623, 1623, 1623, 1623, 1623, 631, 1623, 1623, 1623, 1623, + /* 2230 */ 1954, 1623, 630, 1623, 2003, 1623, 1623, 1623, 1623, 1623, + /* 2240 */ 1623, 1623, 631, 1623, 1623, 1623, 1623, 1954, 1985, 630, + /* 2250 */ 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1984, 1623, 1623, + /* 2260 */ 1623, 2020, 1623, 1623, 311, 1986, 634, 1988, 1989, 629, + /* 2270 */ 1623, 624, 1623, 1623, 1984, 1623, 1623, 1623, 2020, 2003, + /* 2280 */ 1623, 315, 1986, 634, 1988, 1989, 629, 631, 624, 1623, + /* 2290 */ 1623, 1623, 1954, 1623, 630, 1623, 1623, 1623, 1623, 1623, + /* 2300 */ 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1985, + /* 2310 */ 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1984, + /* 2320 */ 1623, 1623, 1623, 2020, 1623, 1985, 307, 1986, 634, 1988, + /* 2330 */ 1989, 629, 1623, 624, 1623, 1623, 1623, 1623, 1623, 1623, + /* 2340 */ 2003, 1623, 1623, 1623, 1623, 1623, 1623, 1623, 631, 1623, + /* 2350 */ 1623, 1623, 1623, 1954, 1623, 630, 2003, 1623, 1623, 1623, + /* 2360 */ 1623, 1623, 1623, 1623, 631, 1623, 1623, 1623, 1623, 1954, + /* 2370 */ 1623, 630, 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1623, + /* 2380 */ 1984, 1623, 1623, 1623, 2020, 1623, 1985, 316, 1986, 634, + /* 2390 */ 1988, 1989, 629, 1623, 624, 1623, 1984, 1623, 1623, 1623, + /* 2400 */ 2020, 1623, 1623, 308, 1986, 634, 1988, 1989, 629, 1623, + /* 2410 */ 624, 1623, 1623, 1623, 1623, 1623, 1623, 2003, 1623, 1623, + /* 2420 */ 1623, 1623, 1623, 1623, 1623, 631, 1623, 1623, 1623, 1623, + /* 2430 */ 1954, 1623, 630, 1623, 1623, 1623, 1623, 1623, 1623, 1623, + /* 2440 */ 1623, 1985, 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1623, + /* 2450 */ 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1984, 1985, 1623, + /* 2460 */ 1623, 2020, 1623, 1623, 317, 1986, 634, 1988, 1989, 629, + /* 2470 */ 1623, 624, 2003, 1623, 1623, 1623, 1623, 1623, 1623, 1623, + /* 2480 */ 631, 1623, 1623, 1623, 1623, 1954, 1623, 630, 1623, 2003, + /* 2490 */ 1623, 1623, 1623, 1623, 1623, 1623, 1623, 631, 1623, 1623, + /* 2500 */ 1623, 1623, 1954, 1623, 630, 1623, 1623, 1623, 1623, 1623, + /* 2510 */ 1623, 1623, 1984, 1623, 1623, 1985, 2020, 1623, 1623, 309, + /* 2520 */ 1986, 634, 1988, 1989, 629, 1623, 624, 1623, 1623, 1984, + /* 2530 */ 1623, 1623, 1985, 2020, 1623, 1623, 322, 1986, 634, 1988, + /* 2540 */ 1989, 629, 1623, 624, 1623, 1623, 2003, 1623, 1623, 1623, + /* 2550 */ 1623, 1623, 1623, 1623, 631, 1623, 1623, 1623, 1623, 1954, + /* 2560 */ 1623, 630, 1623, 2003, 1623, 1623, 1623, 1623, 1623, 1623, + /* 2570 */ 1623, 631, 1623, 1623, 1623, 1623, 1954, 1985, 630, 1623, + /* 2580 */ 1623, 1623, 1623, 1623, 1623, 1623, 1984, 1623, 1623, 1623, + /* 2590 */ 2020, 1623, 1623, 323, 1986, 634, 1988, 1989, 629, 1623, + /* 2600 */ 624, 1623, 1623, 1984, 1623, 1623, 1623, 2020, 2003, 1623, + /* 2610 */ 1997, 1986, 634, 1988, 1989, 629, 631, 624, 1623, 1623, + /* 2620 */ 1623, 1954, 1623, 630, 1623, 1623, 1623, 1623, 1623, 1623, + /* 2630 */ 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1985, 1623, + /* 2640 */ 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1984, 1623, + /* 2650 */ 1623, 1623, 2020, 1623, 1985, 1996, 1986, 634, 1988, 1989, + /* 2660 */ 629, 1623, 624, 1623, 1623, 1623, 1623, 1623, 1623, 2003, + /* 2670 */ 1623, 1623, 1623, 1623, 1623, 1623, 1623, 631, 1623, 1623, + /* 2680 */ 1623, 1623, 1954, 1623, 630, 2003, 1623, 1623, 1623, 1623, + /* 2690 */ 1623, 1623, 1623, 631, 1623, 1623, 1623, 1623, 1954, 1623, + /* 2700 */ 630, 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1984, + /* 2710 */ 1623, 1623, 1623, 2020, 1623, 1985, 1995, 1986, 634, 1988, + /* 2720 */ 1989, 629, 1623, 624, 1623, 1984, 1623, 1623, 1623, 2020, + /* 2730 */ 1623, 1623, 334, 1986, 634, 1988, 1989, 629, 1623, 624, + /* 2740 */ 1623, 1623, 1623, 1623, 1623, 1623, 2003, 1623, 1623, 1623, + /* 2750 */ 1623, 1623, 1623, 1623, 631, 1623, 1623, 1623, 1623, 1954, + /* 2760 */ 1623, 630, 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1623, + /* 2770 */ 1985, 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1623, 1623, + /* 2780 */ 1623, 1623, 1623, 1623, 1623, 1623, 1984, 1985, 1623, 1623, + /* 2790 */ 2020, 1623, 1623, 335, 1986, 634, 1988, 1989, 629, 1623, + /* 2800 */ 624, 2003, 1623, 1623, 1623, 1623, 1623, 1623, 1623, 631, + /* 2810 */ 1623, 1623, 1623, 1623, 1954, 1623, 630, 1623, 2003, 1623, + /* 2820 */ 1623, 1623, 1623, 1623, 1623, 1623, 631, 1623, 1623, 1623, + /* 2830 */ 1623, 1954, 1623, 630, 1623, 1623, 1623, 1623, 1623, 1623, + /* 2840 */ 1623, 1984, 1623, 1623, 1985, 2020, 1623, 1623, 331, 1986, + /* 2850 */ 634, 1988, 1989, 629, 1623, 624, 1623, 1623, 1984, 1623, + /* 2860 */ 1623, 1985, 2020, 1623, 1623, 336, 1986, 634, 1988, 1989, + /* 2870 */ 629, 1623, 624, 1623, 1623, 2003, 1623, 1623, 1623, 1623, + /* 2880 */ 1623, 1623, 1623, 631, 1623, 1623, 1623, 1623, 1954, 1623, + /* 2890 */ 630, 1623, 2003, 1623, 1623, 1623, 1623, 1623, 1623, 1623, + /* 2900 */ 631, 1623, 1623, 1623, 1623, 1954, 1623, 630, 1623, 1623, + /* 2910 */ 1623, 1623, 1623, 1623, 1623, 632, 1623, 1623, 1623, 2020, + /* 2920 */ 1623, 1623, 314, 1986, 634, 1988, 1989, 629, 1623, 624, + /* 2930 */ 1623, 1623, 1984, 1623, 1623, 1623, 2020, 1623, 1623, 313, + /* 2940 */ 1986, 634, 1988, 1989, 629, 1623, 624, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 420, 421, 332, 374, 334, 335, 332, 370, 334, 335, - /* 10 */ 373, 374, 12, 13, 14, 361, 387, 388, 360, 14, - /* 20 */ 20, 336, 22, 8, 9, 20, 372, 12, 13, 14, - /* 30 */ 15, 16, 359, 33, 333, 35, 352, 336, 337, 355, - /* 40 */ 431, 20, 8, 9, 435, 20, 12, 13, 14, 15, - /* 50 */ 16, 39, 20, 380, 381, 401, 402, 403, 58, 450, - /* 60 */ 451, 376, 58, 63, 455, 456, 412, 8, 9, 328, - /* 70 */ 70, 12, 13, 14, 15, 16, 333, 20, 336, 336, - /* 80 */ 337, 8, 9, 12, 13, 12, 13, 14, 15, 16, - /* 90 */ 3, 20, 358, 22, 0, 95, 415, 63, 94, 418, - /* 100 */ 359, 97, 95, 44, 33, 371, 35, 20, 367, 367, - /* 110 */ 327, 96, 329, 372, 336, 374, 95, 117, 24, 25, - /* 120 */ 26, 27, 28, 29, 30, 31, 32, 95, 350, 58, - /* 130 */ 340, 343, 132, 133, 63, 357, 348, 79, 104, 398, - /* 140 */ 4, 70, 62, 402, 354, 367, 405, 406, 407, 408, - /* 150 */ 409, 410, 362, 412, 20, 431, 359, 44, 417, 435, - /* 160 */ 419, 161, 162, 366, 423, 424, 95, 167, 168, 427, - /* 170 */ 428, 429, 375, 431, 432, 451, 20, 435, 437, 455, - /* 180 */ 456, 431, 182, 96, 184, 435, 445, 21, 117, 132, - /* 190 */ 133, 58, 450, 451, 160, 137, 138, 455, 456, 95, - /* 200 */ 34, 451, 36, 132, 133, 455, 456, 207, 208, 96, + /* 0 */ 421, 422, 0, 374, 432, 374, 352, 432, 436, 355, + /* 10 */ 361, 436, 12, 13, 14, 384, 387, 388, 387, 388, + /* 20 */ 20, 372, 22, 451, 452, 20, 361, 452, 456, 457, + /* 30 */ 328, 456, 457, 33, 14, 35, 332, 372, 334, 335, + /* 40 */ 20, 20, 8, 9, 361, 405, 12, 13, 14, 15, + /* 50 */ 16, 402, 403, 404, 14, 372, 389, 336, 58, 20, + /* 60 */ 20, 359, 413, 63, 62, 359, 359, 402, 403, 367, + /* 70 */ 70, 431, 366, 366, 372, 342, 374, 332, 413, 334, + /* 80 */ 335, 375, 375, 12, 13, 402, 403, 404, 367, 356, + /* 90 */ 3, 20, 327, 22, 329, 95, 413, 63, 365, 432, + /* 100 */ 95, 399, 405, 436, 33, 403, 35, 20, 406, 407, + /* 110 */ 408, 409, 410, 411, 342, 413, 4, 117, 451, 452, + /* 120 */ 418, 336, 420, 456, 457, 359, 424, 425, 431, 58, + /* 130 */ 20, 19, 132, 133, 63, 350, 359, 365, 104, 432, + /* 140 */ 438, 70, 357, 436, 367, 33, 380, 381, 446, 428, + /* 150 */ 429, 430, 367, 432, 433, 336, 20, 436, 451, 452, + /* 160 */ 48, 161, 162, 456, 457, 53, 95, 167, 168, 350, + /* 170 */ 58, 370, 451, 452, 373, 374, 357, 456, 457, 367, + /* 180 */ 359, 161, 182, 96, 184, 408, 367, 62, 117, 4, + /* 190 */ 378, 8, 9, 126, 160, 12, 13, 14, 15, 16, + /* 200 */ 161, 162, 381, 132, 133, 95, 94, 207, 208, 97, /* 210 */ 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, - /* 220 */ 220, 221, 222, 223, 224, 225, 226, 227, 95, 95, - /* 230 */ 97, 95, 161, 162, 404, 20, 163, 21, 167, 168, - /* 240 */ 24, 25, 26, 27, 28, 29, 30, 31, 32, 106, - /* 250 */ 163, 127, 245, 182, 404, 184, 244, 8, 9, 328, - /* 260 */ 430, 12, 13, 14, 15, 16, 232, 233, 234, 235, - /* 270 */ 236, 237, 238, 239, 240, 241, 242, 245, 207, 208, - /* 280 */ 430, 210, 211, 212, 213, 214, 215, 216, 217, 218, - /* 290 */ 219, 220, 221, 222, 223, 224, 225, 226, 227, 177, - /* 300 */ 229, 12, 13, 372, 361, 22, 14, 328, 359, 20, - /* 310 */ 22, 22, 20, 189, 190, 372, 367, 193, 35, 195, - /* 320 */ 198, 199, 33, 35, 35, 108, 109, 110, 111, 112, - /* 330 */ 113, 114, 115, 116, 117, 118, 0, 120, 121, 122, - /* 340 */ 123, 124, 125, 33, 401, 402, 403, 58, 35, 245, - /* 350 */ 389, 372, 63, 70, 359, 412, 407, 4, 48, 70, - /* 360 */ 20, 366, 22, 20, 54, 55, 56, 57, 58, 328, - /* 370 */ 375, 328, 12, 13, 14, 35, 161, 162, 245, 245, - /* 380 */ 20, 245, 22, 70, 95, 12, 13, 14, 15, 16, - /* 390 */ 62, 51, 431, 33, 20, 35, 435, 20, 328, 22, - /* 400 */ 117, 65, 66, 67, 94, 4, 117, 97, 0, 73, - /* 410 */ 74, 450, 451, 372, 78, 372, 455, 456, 58, 83, - /* 420 */ 84, 132, 133, 174, 328, 89, 126, 127, 51, 21, - /* 430 */ 70, 131, 24, 25, 26, 27, 28, 29, 30, 31, - /* 440 */ 32, 336, 372, 20, 43, 328, 45, 46, 70, 374, - /* 450 */ 161, 162, 336, 161, 359, 95, 167, 168, 229, 384, - /* 460 */ 231, 366, 387, 388, 336, 182, 350, 184, 372, 44, - /* 470 */ 375, 182, 367, 184, 164, 165, 328, 117, 350, 169, - /* 480 */ 8, 9, 172, 367, 12, 13, 14, 15, 16, 372, - /* 490 */ 207, 208, 132, 133, 342, 367, 207, 208, 188, 210, + /* 220 */ 220, 221, 222, 223, 224, 225, 226, 227, 43, 4, + /* 230 */ 45, 46, 161, 162, 14, 15, 16, 21, 167, 168, + /* 240 */ 24, 25, 26, 27, 28, 29, 30, 31, 32, 343, + /* 250 */ 163, 37, 358, 182, 348, 184, 8, 9, 191, 192, + /* 260 */ 12, 13, 14, 15, 16, 371, 232, 233, 234, 235, + /* 270 */ 236, 237, 238, 239, 240, 241, 242, 58, 207, 208, + /* 280 */ 20, 210, 211, 212, 213, 214, 215, 216, 217, 218, + /* 290 */ 219, 220, 221, 222, 223, 224, 225, 226, 227, 163, + /* 300 */ 229, 12, 13, 0, 12, 13, 14, 15, 16, 20, + /* 310 */ 0, 22, 98, 94, 100, 101, 97, 103, 20, 328, + /* 320 */ 95, 107, 33, 328, 35, 108, 109, 110, 111, 112, + /* 330 */ 113, 114, 115, 116, 117, 118, 35, 120, 121, 122, + /* 340 */ 123, 124, 125, 129, 96, 333, 58, 58, 336, 337, + /* 350 */ 359, 21, 63, 373, 374, 245, 20, 174, 367, 70, + /* 360 */ 20, 39, 22, 372, 34, 374, 36, 372, 65, 66, + /* 370 */ 67, 70, 12, 13, 14, 35, 73, 74, 20, 20, + /* 380 */ 20, 78, 22, 95, 95, 97, 83, 84, 405, 359, + /* 390 */ 399, 51, 89, 33, 403, 35, 366, 406, 407, 408, + /* 400 */ 409, 410, 411, 328, 413, 375, 117, 416, 432, 418, + /* 410 */ 419, 420, 436, 2, 431, 424, 425, 107, 58, 8, + /* 420 */ 9, 132, 133, 12, 13, 14, 15, 16, 452, 0, + /* 430 */ 70, 0, 456, 457, 359, 0, 126, 127, 128, 129, + /* 440 */ 130, 131, 367, 126, 127, 345, 346, 372, 131, 374, + /* 450 */ 161, 162, 269, 166, 95, 95, 167, 168, 336, 24, + /* 460 */ 25, 26, 27, 28, 29, 30, 31, 32, 132, 133, + /* 470 */ 245, 182, 333, 184, 399, 336, 337, 117, 403, 48, + /* 480 */ 328, 406, 407, 408, 409, 410, 411, 412, 413, 414, + /* 490 */ 415, 62, 132, 133, 345, 346, 207, 208, 376, 210, /* 500 */ 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, - /* 510 */ 221, 222, 223, 224, 225, 226, 227, 365, 269, 20, - /* 520 */ 372, 161, 162, 336, 35, 328, 431, 167, 168, 336, - /* 530 */ 435, 426, 427, 428, 429, 336, 431, 432, 20, 65, - /* 540 */ 66, 67, 182, 350, 184, 450, 451, 73, 74, 350, - /* 550 */ 455, 456, 78, 328, 367, 132, 133, 83, 84, 70, - /* 560 */ 367, 328, 328, 89, 373, 374, 367, 207, 208, 372, + /* 510 */ 221, 222, 223, 224, 225, 226, 227, 65, 66, 67, + /* 520 */ 336, 161, 162, 351, 372, 73, 74, 167, 168, 0, + /* 530 */ 78, 359, 359, 245, 350, 83, 84, 250, 251, 252, + /* 540 */ 368, 89, 182, 177, 184, 95, 325, 340, 375, 416, + /* 550 */ 21, 367, 419, 24, 25, 26, 27, 28, 29, 30, + /* 560 */ 31, 32, 328, 95, 198, 199, 244, 207, 208, 362, /* 570 */ 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, /* 580 */ 220, 221, 222, 223, 224, 225, 226, 227, 12, 13, - /* 590 */ 167, 168, 359, 0, 342, 351, 20, 372, 22, 246, - /* 600 */ 367, 325, 416, 359, 418, 372, 372, 374, 356, 33, - /* 610 */ 126, 35, 368, 229, 427, 428, 429, 365, 431, 432, - /* 620 */ 8, 9, 328, 404, 12, 13, 14, 15, 16, 4, - /* 630 */ 166, 398, 20, 336, 58, 402, 70, 207, 405, 406, - /* 640 */ 407, 408, 409, 410, 19, 412, 70, 350, 415, 430, - /* 650 */ 417, 418, 419, 359, 357, 62, 423, 424, 33, 12, - /* 660 */ 13, 367, 163, 336, 367, 389, 372, 20, 374, 22, - /* 670 */ 416, 95, 418, 48, 328, 191, 192, 350, 53, 328, - /* 680 */ 33, 163, 35, 58, 361, 95, 256, 257, 258, 259, - /* 690 */ 260, 266, 398, 117, 367, 372, 402, 345, 346, 405, - /* 700 */ 406, 407, 408, 409, 410, 58, 412, 431, 132, 133, - /* 710 */ 0, 435, 393, 351, 250, 251, 252, 70, 372, 94, - /* 720 */ 0, 359, 97, 372, 401, 402, 450, 451, 14, 360, - /* 730 */ 368, 455, 456, 328, 20, 412, 369, 161, 162, 372, - /* 740 */ 446, 447, 95, 167, 168, 336, 8, 9, 160, 360, - /* 750 */ 12, 13, 14, 15, 16, 2, 359, 3, 182, 350, - /* 760 */ 184, 8, 9, 107, 117, 12, 13, 14, 15, 16, - /* 770 */ 360, 181, 62, 183, 328, 163, 367, 372, 381, 132, - /* 780 */ 133, 0, 328, 207, 208, 129, 210, 211, 212, 213, + /* 590 */ 22, 328, 417, 359, 419, 20, 20, 22, 22, 369, + /* 600 */ 336, 367, 372, 35, 245, 95, 372, 62, 374, 33, + /* 610 */ 389, 35, 106, 127, 350, 1, 2, 8, 9, 328, + /* 620 */ 359, 12, 13, 14, 15, 16, 51, 366, 229, 20, + /* 630 */ 336, 367, 359, 399, 58, 372, 375, 403, 70, 366, + /* 640 */ 406, 407, 408, 409, 410, 411, 70, 413, 375, 181, + /* 650 */ 359, 183, 336, 432, 0, 3, 336, 436, 367, 12, + /* 660 */ 13, 367, 229, 372, 231, 374, 350, 20, 20, 22, + /* 670 */ 350, 95, 451, 452, 340, 189, 190, 456, 457, 193, + /* 680 */ 33, 195, 35, 367, 450, 117, 389, 367, 354, 336, + /* 690 */ 399, 328, 328, 117, 403, 245, 362, 406, 407, 408, + /* 700 */ 409, 410, 411, 350, 413, 58, 0, 63, 132, 133, + /* 710 */ 96, 420, 20, 245, 336, 424, 425, 70, 22, 79, + /* 720 */ 367, 427, 428, 429, 430, 70, 432, 433, 350, 432, + /* 730 */ 328, 35, 336, 436, 160, 372, 372, 161, 162, 351, + /* 740 */ 367, 328, 95, 167, 168, 367, 350, 359, 451, 452, + /* 750 */ 182, 378, 184, 456, 457, 245, 368, 347, 182, 349, + /* 760 */ 184, 8, 9, 367, 117, 12, 13, 14, 15, 16, + /* 770 */ 20, 367, 163, 328, 372, 207, 208, 137, 138, 132, + /* 780 */ 133, 4, 378, 207, 208, 372, 210, 211, 212, 213, /* 790 */ 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, - /* 800 */ 224, 225, 226, 227, 14, 15, 16, 336, 161, 162, - /* 810 */ 18, 42, 20, 44, 167, 168, 351, 340, 372, 27, - /* 820 */ 232, 350, 30, 360, 359, 33, 372, 107, 360, 182, - /* 830 */ 242, 184, 328, 368, 96, 245, 345, 346, 367, 362, - /* 840 */ 48, 361, 50, 360, 359, 53, 126, 127, 128, 129, - /* 850 */ 130, 131, 372, 368, 207, 208, 20, 210, 211, 212, + /* 800 */ 224, 225, 226, 227, 328, 151, 232, 336, 161, 162, + /* 810 */ 18, 163, 20, 107, 167, 168, 242, 372, 417, 27, + /* 820 */ 419, 350, 30, 360, 369, 33, 207, 372, 42, 182, + /* 830 */ 44, 184, 126, 127, 128, 129, 130, 131, 367, 351, + /* 840 */ 48, 20, 50, 359, 14, 53, 70, 359, 372, 96, + /* 850 */ 20, 207, 368, 161, 207, 208, 368, 210, 211, 212, /* 860 */ 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, - /* 870 */ 223, 224, 225, 226, 227, 161, 372, 347, 20, 349, - /* 880 */ 18, 401, 402, 2, 0, 23, 94, 0, 107, 8, - /* 890 */ 9, 367, 412, 12, 13, 14, 15, 16, 106, 37, - /* 900 */ 38, 359, 378, 41, 367, 367, 329, 126, 127, 128, - /* 910 */ 129, 130, 131, 1, 2, 378, 378, 375, 45, 46, - /* 920 */ 95, 59, 60, 61, 369, 382, 369, 372, 136, 372, - /* 930 */ 105, 139, 140, 141, 142, 143, 144, 145, 146, 147, - /* 940 */ 148, 149, 150, 151, 152, 153, 154, 155, 389, 157, + /* 870 */ 223, 224, 225, 226, 227, 256, 257, 258, 259, 260, + /* 880 */ 18, 369, 132, 133, 372, 23, 94, 0, 336, 8, + /* 890 */ 9, 45, 46, 12, 13, 14, 15, 16, 106, 37, + /* 900 */ 38, 2, 350, 41, 44, 107, 352, 8, 9, 355, + /* 910 */ 95, 12, 13, 14, 15, 16, 264, 167, 168, 367, + /* 920 */ 105, 59, 60, 61, 42, 44, 44, 129, 136, 243, + /* 930 */ 244, 139, 140, 141, 142, 143, 144, 145, 146, 147, + /* 940 */ 148, 149, 150, 151, 152, 153, 154, 155, 328, 157, /* 950 */ 158, 159, 65, 66, 67, 68, 69, 95, 71, 72, /* 960 */ 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, - /* 970 */ 83, 84, 85, 86, 87, 88, 89, 90, 91, 328, - /* 980 */ 459, 336, 107, 328, 352, 336, 389, 355, 389, 44, - /* 990 */ 431, 243, 244, 336, 435, 350, 134, 161, 348, 350, - /* 1000 */ 328, 126, 127, 128, 129, 130, 131, 350, 96, 450, - /* 1010 */ 451, 42, 367, 44, 455, 456, 367, 99, 264, 106, - /* 1020 */ 102, 163, 99, 372, 367, 102, 163, 372, 431, 336, - /* 1030 */ 431, 359, 435, 170, 435, 151, 174, 175, 176, 367, - /* 1040 */ 336, 179, 336, 350, 372, 35, 374, 450, 451, 450, - /* 1050 */ 451, 448, 455, 456, 455, 456, 350, 0, 389, 197, - /* 1060 */ 367, 336, 200, 58, 202, 203, 204, 205, 206, 156, - /* 1070 */ 398, 367, 328, 367, 402, 350, 35, 405, 406, 407, - /* 1080 */ 408, 409, 410, 336, 412, 99, 44, 99, 102, 417, - /* 1090 */ 102, 419, 367, 0, 338, 423, 424, 350, 336, 336, - /* 1100 */ 431, 0, 97, 359, 435, 48, 0, 245, 336, 63, - /* 1110 */ 37, 367, 350, 350, 367, 22, 372, 445, 374, 450, - /* 1120 */ 451, 47, 350, 22, 455, 456, 328, 336, 22, 367, - /* 1130 */ 367, 427, 428, 429, 44, 431, 432, 442, 96, 367, - /* 1140 */ 44, 350, 398, 132, 133, 359, 402, 44, 338, 405, - /* 1150 */ 406, 407, 408, 409, 410, 13, 412, 359, 367, 44, - /* 1160 */ 44, 417, 382, 419, 44, 367, 335, 423, 424, 95, - /* 1170 */ 372, 98, 374, 100, 101, 44, 103, 35, 434, 13, - /* 1180 */ 107, 0, 371, 336, 44, 44, 96, 328, 1, 2, - /* 1190 */ 382, 44, 96, 433, 184, 44, 398, 350, 452, 96, - /* 1200 */ 402, 35, 129, 405, 406, 407, 408, 409, 410, 425, - /* 1210 */ 412, 96, 96, 268, 367, 417, 96, 419, 359, 436, - /* 1220 */ 44, 423, 424, 44, 247, 184, 367, 96, 12, 13, - /* 1230 */ 49, 372, 434, 374, 400, 180, 96, 96, 22, 44, - /* 1240 */ 48, 399, 391, 96, 42, 379, 20, 96, 379, 33, - /* 1250 */ 328, 35, 382, 207, 160, 377, 20, 398, 379, 377, - /* 1260 */ 336, 402, 336, 377, 405, 406, 407, 408, 409, 410, - /* 1270 */ 328, 412, 96, 93, 58, 96, 417, 344, 419, 336, - /* 1280 */ 336, 359, 423, 424, 336, 20, 70, 330, 330, 367, - /* 1290 */ 20, 96, 395, 434, 372, 342, 374, 342, 20, 374, - /* 1300 */ 20, 359, 337, 20, 390, 342, 342, 337, 342, 367, - /* 1310 */ 342, 336, 52, 342, 372, 339, 374, 339, 359, 330, - /* 1320 */ 398, 359, 330, 336, 402, 196, 359, 405, 406, 407, - /* 1330 */ 408, 409, 410, 117, 412, 397, 395, 340, 359, 417, - /* 1340 */ 398, 419, 359, 359, 402, 423, 424, 405, 406, 407, - /* 1350 */ 408, 409, 410, 359, 412, 359, 434, 359, 359, 417, - /* 1360 */ 359, 419, 372, 372, 187, 423, 424, 394, 340, 374, - /* 1370 */ 19, 336, 255, 328, 372, 382, 434, 441, 254, 372, - /* 1380 */ 441, 372, 382, 261, 33, 372, 173, 440, 385, 385, - /* 1390 */ 444, 441, 443, 263, 262, 439, 248, 270, 182, 48, - /* 1400 */ 184, 267, 265, 244, 359, 54, 55, 56, 57, 58, - /* 1410 */ 438, 460, 367, 453, 400, 454, 367, 372, 20, 374, - /* 1420 */ 404, 336, 340, 207, 208, 337, 372, 165, 385, 385, - /* 1430 */ 372, 372, 328, 372, 372, 372, 220, 221, 222, 223, - /* 1440 */ 224, 225, 226, 398, 355, 94, 367, 402, 97, 383, - /* 1450 */ 405, 406, 407, 408, 409, 410, 95, 412, 340, 340, - /* 1460 */ 422, 95, 417, 359, 419, 363, 340, 372, 423, 424, - /* 1470 */ 349, 367, 336, 36, 331, 330, 372, 396, 374, 392, - /* 1480 */ 326, 130, 0, 386, 386, 341, 328, 0, 353, 189, - /* 1490 */ 0, 353, 353, 0, 42, 0, 35, 201, 35, 35, - /* 1500 */ 35, 0, 398, 201, 35, 35, 402, 201, 0, 405, - /* 1510 */ 406, 407, 408, 409, 410, 201, 412, 359, 0, 35, - /* 1520 */ 169, 417, 0, 419, 22, 367, 35, 423, 424, 0, - /* 1530 */ 372, 184, 374, 0, 182, 0, 178, 186, 0, 188, - /* 1540 */ 177, 0, 0, 47, 0, 0, 42, 328, 0, 0, - /* 1550 */ 0, 0, 0, 0, 0, 0, 398, 151, 35, 0, - /* 1560 */ 402, 151, 0, 405, 406, 407, 408, 409, 410, 0, - /* 1570 */ 412, 0, 0, 0, 0, 417, 0, 419, 359, 0, - /* 1580 */ 0, 423, 424, 0, 0, 0, 367, 0, 0, 0, - /* 1590 */ 0, 372, 0, 374, 135, 0, 0, 22, 42, 0, - /* 1600 */ 0, 0, 0, 0, 0, 328, 0, 0, 0, 0, - /* 1610 */ 42, 44, 35, 47, 0, 58, 58, 398, 0, 14, - /* 1620 */ 39, 402, 40, 0, 405, 406, 407, 408, 409, 410, - /* 1630 */ 14, 412, 0, 47, 0, 173, 359, 39, 419, 0, - /* 1640 */ 47, 0, 423, 424, 367, 0, 0, 35, 0, 372, - /* 1650 */ 39, 374, 39, 0, 35, 64, 39, 35, 48, 328, - /* 1660 */ 39, 0, 35, 48, 39, 0, 0, 0, 48, 0, - /* 1670 */ 35, 22, 48, 0, 104, 398, 22, 35, 35, 402, - /* 1680 */ 44, 44, 405, 406, 407, 408, 409, 410, 328, 412, - /* 1690 */ 359, 35, 35, 35, 35, 35, 419, 0, 367, 102, - /* 1700 */ 423, 424, 22, 372, 0, 374, 22, 0, 22, 35, - /* 1710 */ 0, 35, 50, 0, 35, 0, 22, 35, 20, 359, - /* 1720 */ 35, 194, 96, 95, 0, 35, 163, 367, 0, 398, - /* 1730 */ 22, 185, 372, 402, 374, 0, 405, 406, 407, 408, - /* 1740 */ 409, 410, 0, 412, 3, 95, 328, 44, 249, 228, - /* 1750 */ 419, 96, 47, 253, 423, 424, 44, 3, 398, 95, - /* 1760 */ 44, 47, 402, 44, 96, 405, 406, 407, 408, 409, - /* 1770 */ 410, 411, 412, 413, 414, 44, 95, 359, 35, 249, - /* 1780 */ 249, 35, 35, 96, 95, 367, 95, 35, 96, 96, - /* 1790 */ 372, 95, 374, 163, 165, 96, 163, 35, 35, 0, - /* 1800 */ 328, 44, 171, 47, 243, 96, 47, 96, 170, 0, - /* 1810 */ 170, 0, 95, 166, 96, 95, 398, 96, 95, 0, - /* 1820 */ 402, 95, 39, 405, 406, 407, 408, 409, 410, 95, - /* 1830 */ 412, 359, 105, 47, 164, 44, 2, 22, 22, 367, - /* 1840 */ 95, 47, 47, 95, 372, 96, 374, 96, 95, 35, - /* 1850 */ 106, 119, 35, 95, 328, 35, 96, 96, 207, 95, - /* 1860 */ 95, 209, 96, 95, 35, 96, 95, 35, 35, 96, - /* 1870 */ 398, 95, 228, 96, 402, 457, 458, 405, 406, 407, - /* 1880 */ 408, 409, 410, 95, 412, 359, 96, 95, 22, 119, - /* 1890 */ 44, 419, 107, 367, 119, 95, 424, 119, 372, 95, - /* 1900 */ 374, 230, 35, 228, 22, 95, 328, 64, 63, 35, - /* 1910 */ 35, 35, 35, 35, 35, 35, 35, 70, 35, 35, - /* 1920 */ 92, 35, 328, 35, 398, 35, 22, 70, 402, 44, - /* 1930 */ 35, 405, 406, 407, 408, 409, 410, 359, 412, 22, - /* 1940 */ 35, 35, 35, 35, 35, 367, 35, 35, 35, 0, - /* 1950 */ 372, 35, 374, 359, 39, 0, 35, 39, 48, 0, - /* 1960 */ 35, 367, 48, 39, 0, 35, 372, 48, 374, 0, - /* 1970 */ 48, 39, 35, 447, 35, 0, 398, 20, 22, 21, - /* 1980 */ 402, 461, 22, 405, 406, 407, 408, 409, 410, 21, - /* 1990 */ 412, 461, 398, 22, 461, 461, 402, 461, 461, 405, - /* 2000 */ 406, 407, 408, 409, 410, 461, 412, 461, 461, 461, - /* 2010 */ 461, 328, 461, 461, 461, 461, 461, 461, 461, 461, - /* 2020 */ 461, 461, 461, 461, 461, 461, 461, 449, 461, 328, - /* 2030 */ 461, 461, 461, 461, 461, 461, 461, 461, 461, 461, - /* 2040 */ 461, 461, 359, 461, 461, 461, 461, 364, 461, 461, - /* 2050 */ 367, 461, 458, 461, 461, 372, 461, 374, 461, 461, - /* 2060 */ 359, 461, 461, 461, 461, 461, 461, 461, 367, 461, - /* 2070 */ 461, 461, 461, 372, 461, 374, 461, 461, 461, 461, - /* 2080 */ 461, 398, 461, 328, 461, 402, 461, 461, 405, 406, - /* 2090 */ 407, 408, 409, 410, 461, 412, 461, 461, 461, 398, - /* 2100 */ 328, 461, 461, 402, 461, 461, 405, 406, 407, 408, - /* 2110 */ 409, 410, 461, 412, 359, 414, 461, 461, 461, 364, - /* 2120 */ 461, 461, 367, 461, 461, 461, 461, 372, 461, 374, - /* 2130 */ 461, 359, 461, 461, 461, 461, 364, 461, 461, 367, - /* 2140 */ 461, 461, 461, 461, 372, 461, 374, 461, 461, 461, - /* 2150 */ 461, 461, 328, 398, 461, 461, 461, 402, 461, 461, - /* 2160 */ 405, 406, 407, 408, 409, 410, 461, 412, 328, 461, - /* 2170 */ 398, 461, 461, 461, 402, 461, 461, 405, 406, 407, - /* 2180 */ 408, 409, 410, 359, 412, 461, 461, 461, 461, 461, - /* 2190 */ 461, 367, 461, 461, 461, 461, 372, 461, 374, 359, - /* 2200 */ 461, 461, 461, 461, 461, 461, 461, 367, 461, 461, - /* 2210 */ 461, 461, 372, 461, 374, 461, 461, 461, 461, 461, - /* 2220 */ 328, 461, 398, 461, 461, 461, 402, 461, 461, 405, - /* 2230 */ 406, 407, 408, 409, 410, 461, 412, 461, 398, 461, - /* 2240 */ 461, 461, 402, 461, 461, 405, 406, 407, 408, 409, - /* 2250 */ 410, 359, 412, 461, 461, 461, 461, 461, 461, 367, - /* 2260 */ 461, 461, 461, 461, 372, 461, 374, 461, 461, 461, - /* 2270 */ 461, 461, 461, 461, 328, 461, 461, 461, 461, 461, - /* 2280 */ 461, 461, 461, 461, 461, 461, 461, 461, 461, 461, - /* 2290 */ 398, 461, 461, 461, 402, 328, 461, 405, 406, 407, - /* 2300 */ 408, 409, 410, 461, 412, 359, 461, 461, 461, 461, - /* 2310 */ 461, 461, 461, 367, 461, 461, 461, 461, 372, 461, - /* 2320 */ 374, 461, 461, 461, 461, 461, 359, 461, 461, 461, - /* 2330 */ 461, 461, 461, 461, 367, 461, 461, 461, 461, 372, - /* 2340 */ 461, 374, 461, 461, 398, 461, 461, 461, 402, 461, - /* 2350 */ 461, 405, 406, 407, 408, 409, 410, 461, 412, 461, - /* 2360 */ 461, 461, 461, 461, 461, 398, 461, 461, 461, 402, - /* 2370 */ 461, 328, 405, 406, 407, 408, 409, 410, 461, 412, - /* 2380 */ 461, 461, 461, 461, 461, 461, 461, 461, 461, 328, - /* 2390 */ 461, 461, 461, 461, 461, 461, 461, 461, 461, 461, - /* 2400 */ 461, 461, 359, 461, 461, 461, 461, 461, 461, 461, - /* 2410 */ 367, 461, 461, 461, 461, 372, 461, 374, 461, 461, - /* 2420 */ 359, 461, 461, 461, 461, 461, 461, 461, 367, 461, - /* 2430 */ 461, 461, 461, 372, 461, 374, 461, 461, 461, 461, - /* 2440 */ 461, 398, 461, 328, 461, 402, 461, 461, 405, 406, - /* 2450 */ 407, 408, 409, 410, 461, 412, 461, 461, 461, 398, - /* 2460 */ 328, 461, 461, 402, 461, 461, 405, 406, 407, 408, - /* 2470 */ 409, 410, 461, 412, 359, 461, 461, 461, 461, 461, - /* 2480 */ 461, 461, 367, 461, 461, 461, 461, 372, 461, 374, - /* 2490 */ 461, 359, 461, 461, 461, 461, 461, 461, 461, 367, - /* 2500 */ 461, 461, 461, 461, 372, 461, 374, 461, 461, 461, - /* 2510 */ 461, 461, 328, 398, 461, 461, 461, 402, 461, 461, - /* 2520 */ 405, 406, 407, 408, 409, 410, 461, 412, 328, 461, - /* 2530 */ 398, 461, 461, 461, 402, 461, 461, 405, 406, 407, - /* 2540 */ 408, 409, 410, 359, 412, 461, 461, 461, 461, 461, - /* 2550 */ 461, 367, 461, 461, 461, 461, 372, 461, 374, 359, - /* 2560 */ 461, 461, 461, 461, 461, 461, 461, 367, 461, 461, - /* 2570 */ 461, 461, 372, 461, 374, 461, 461, 461, 461, 461, - /* 2580 */ 328, 461, 398, 461, 461, 461, 402, 461, 461, 405, - /* 2590 */ 406, 407, 408, 409, 410, 461, 412, 461, 398, 461, - /* 2600 */ 461, 461, 402, 328, 461, 405, 406, 407, 408, 409, - /* 2610 */ 410, 359, 412, 461, 461, 461, 461, 461, 461, 367, - /* 2620 */ 461, 461, 461, 461, 372, 461, 374, 461, 461, 461, - /* 2630 */ 461, 461, 461, 461, 359, 461, 461, 461, 461, 461, - /* 2640 */ 461, 461, 367, 461, 461, 461, 461, 372, 461, 374, - /* 2650 */ 398, 461, 461, 461, 402, 328, 461, 405, 406, 407, - /* 2660 */ 408, 409, 410, 461, 412, 461, 461, 461, 461, 461, - /* 2670 */ 461, 461, 461, 398, 461, 461, 461, 402, 461, 461, - /* 2680 */ 405, 406, 407, 408, 409, 410, 359, 412, 461, 461, - /* 2690 */ 461, 461, 461, 461, 367, 461, 461, 461, 461, 372, - /* 2700 */ 461, 374, 461, 461, 461, 461, 461, 461, 461, 461, - /* 2710 */ 461, 461, 461, 461, 461, 461, 461, 461, 328, 461, - /* 2720 */ 461, 461, 461, 461, 461, 398, 461, 461, 461, 402, - /* 2730 */ 461, 461, 405, 406, 407, 408, 409, 410, 461, 412, - /* 2740 */ 461, 461, 461, 461, 461, 461, 461, 461, 461, 359, - /* 2750 */ 461, 461, 461, 461, 461, 461, 461, 367, 461, 461, - /* 2760 */ 461, 461, 372, 461, 374, 461, 461, 461, 461, 461, - /* 2770 */ 328, 461, 461, 461, 461, 461, 461, 461, 461, 461, - /* 2780 */ 461, 461, 461, 461, 461, 461, 461, 461, 398, 328, - /* 2790 */ 461, 461, 402, 461, 461, 405, 406, 407, 408, 409, - /* 2800 */ 410, 359, 412, 461, 461, 461, 461, 461, 461, 367, - /* 2810 */ 461, 461, 461, 461, 372, 461, 374, 461, 461, 461, - /* 2820 */ 359, 461, 461, 461, 461, 461, 461, 461, 367, 461, - /* 2830 */ 461, 461, 461, 372, 461, 374, 461, 461, 461, 461, - /* 2840 */ 398, 328, 461, 461, 402, 461, 461, 405, 406, 407, - /* 2850 */ 408, 409, 410, 461, 412, 461, 461, 328, 461, 398, - /* 2860 */ 461, 461, 461, 402, 461, 461, 405, 406, 407, 408, - /* 2870 */ 409, 410, 359, 412, 461, 461, 461, 461, 461, 461, - /* 2880 */ 367, 461, 461, 461, 461, 372, 461, 374, 359, 461, - /* 2890 */ 461, 461, 461, 461, 461, 461, 367, 461, 461, 461, - /* 2900 */ 461, 372, 461, 374, 461, 461, 461, 461, 461, 328, - /* 2910 */ 461, 398, 461, 461, 461, 402, 461, 461, 405, 406, - /* 2920 */ 407, 408, 409, 410, 461, 412, 461, 398, 461, 461, - /* 2930 */ 461, 402, 328, 461, 405, 406, 407, 408, 409, 410, - /* 2940 */ 359, 412, 461, 461, 461, 461, 461, 461, 367, 461, - /* 2950 */ 461, 461, 461, 372, 461, 374, 461, 461, 461, 461, - /* 2960 */ 461, 461, 461, 359, 461, 461, 461, 461, 461, 461, - /* 2970 */ 461, 367, 461, 461, 461, 461, 372, 461, 374, 398, - /* 2980 */ 461, 461, 461, 402, 328, 461, 405, 406, 407, 408, - /* 2990 */ 409, 410, 461, 412, 461, 461, 461, 461, 461, 461, - /* 3000 */ 461, 461, 398, 461, 461, 461, 402, 461, 461, 405, - /* 3010 */ 406, 407, 408, 409, 410, 359, 412, 461, 461, 461, - /* 3020 */ 461, 461, 461, 367, 461, 461, 461, 461, 372, 461, - /* 3030 */ 374, 461, 461, 461, 461, 461, 461, 461, 461, 461, - /* 3040 */ 461, 461, 461, 461, 461, 461, 461, 461, 461, 461, - /* 3050 */ 461, 461, 461, 461, 398, 461, 461, 461, 402, 461, - /* 3060 */ 461, 405, 406, 407, 408, 409, 410, 461, 412, 325, - /* 3070 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3080 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3090 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3100 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3110 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3120 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3130 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3140 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3150 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3160 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3170 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3180 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3190 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3200 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3210 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3220 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3230 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3240 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3250 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3260 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3270 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3280 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3290 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3300 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3310 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3320 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3330 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3340 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3350 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3360 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3370 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3380 */ 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - /* 3390 */ 325, 325, 325, 325, + /* 970 */ 83, 84, 85, 86, 87, 88, 89, 90, 91, 336, + /* 980 */ 19, 8, 9, 44, 163, 12, 13, 14, 15, 16, + /* 990 */ 328, 161, 372, 350, 33, 44, 134, 8, 9, 0, + /* 1000 */ 328, 12, 13, 14, 15, 16, 106, 348, 99, 48, + /* 1010 */ 367, 102, 336, 328, 336, 54, 55, 56, 57, 58, + /* 1020 */ 360, 22, 44, 246, 99, 328, 350, 102, 350, 107, + /* 1030 */ 99, 359, 328, 102, 372, 96, 174, 175, 176, 367, + /* 1040 */ 360, 179, 328, 367, 372, 367, 374, 96, 126, 127, + /* 1050 */ 128, 129, 130, 131, 35, 94, 156, 372, 97, 197, + /* 1060 */ 393, 361, 200, 336, 202, 203, 204, 205, 206, 372, + /* 1070 */ 336, 399, 372, 328, 336, 403, 372, 350, 406, 407, + /* 1080 */ 408, 409, 410, 411, 350, 413, 372, 329, 350, 328, + /* 1090 */ 418, 130, 420, 328, 367, 336, 424, 425, 0, 389, + /* 1100 */ 336, 367, 402, 403, 359, 367, 328, 245, 99, 350, + /* 1110 */ 382, 102, 367, 413, 350, 328, 360, 372, 446, 374, + /* 1120 */ 22, 0, 132, 133, 359, 164, 367, 163, 268, 44, + /* 1130 */ 169, 367, 367, 372, 170, 44, 163, 372, 328, 374, + /* 1140 */ 44, 360, 432, 22, 399, 44, 436, 186, 403, 188, + /* 1150 */ 372, 406, 407, 408, 409, 410, 411, 58, 413, 372, + /* 1160 */ 328, 451, 452, 418, 399, 420, 456, 457, 403, 424, + /* 1170 */ 425, 406, 407, 408, 409, 410, 411, 360, 413, 336, + /* 1180 */ 435, 96, 372, 418, 336, 420, 389, 96, 35, 424, + /* 1190 */ 425, 359, 96, 350, 336, 336, 97, 96, 350, 367, + /* 1200 */ 435, 1, 2, 184, 372, 360, 374, 460, 350, 350, + /* 1210 */ 367, 449, 44, 12, 13, 367, 328, 338, 44, 382, + /* 1220 */ 44, 338, 389, 22, 44, 367, 367, 44, 44, 432, + /* 1230 */ 44, 399, 336, 436, 33, 403, 35, 0, 406, 407, + /* 1240 */ 408, 409, 410, 411, 266, 413, 35, 359, 451, 452, + /* 1250 */ 418, 44, 420, 456, 457, 367, 424, 425, 44, 58, + /* 1260 */ 372, 328, 374, 367, 96, 432, 47, 435, 44, 436, + /* 1270 */ 96, 70, 96, 336, 443, 44, 96, 359, 44, 96, + /* 1280 */ 96, 70, 96, 335, 451, 452, 49, 399, 371, 456, + /* 1290 */ 457, 403, 359, 13, 406, 407, 408, 409, 410, 411, + /* 1300 */ 367, 413, 382, 96, 367, 372, 418, 374, 420, 13, + /* 1310 */ 96, 434, 424, 425, 95, 35, 453, 426, 117, 247, + /* 1320 */ 96, 437, 401, 435, 428, 429, 430, 96, 432, 433, + /* 1330 */ 96, 35, 399, 48, 400, 180, 403, 184, 391, 406, + /* 1340 */ 407, 408, 409, 410, 411, 42, 413, 379, 20, 382, + /* 1350 */ 379, 418, 160, 420, 377, 20, 336, 424, 425, 336, + /* 1360 */ 328, 379, 377, 377, 93, 428, 429, 430, 435, 432, + /* 1370 */ 433, 344, 336, 336, 336, 20, 330, 330, 20, 395, + /* 1380 */ 20, 342, 374, 182, 342, 184, 20, 337, 20, 337, + /* 1390 */ 390, 359, 336, 342, 52, 342, 342, 342, 339, 367, + /* 1400 */ 342, 339, 359, 330, 372, 372, 374, 359, 207, 208, + /* 1410 */ 336, 330, 372, 196, 398, 95, 359, 395, 397, 359, + /* 1420 */ 359, 220, 221, 222, 223, 224, 225, 226, 359, 359, + /* 1430 */ 359, 399, 340, 328, 359, 403, 359, 359, 406, 407, + /* 1440 */ 408, 409, 410, 411, 187, 413, 340, 336, 382, 382, + /* 1450 */ 418, 372, 420, 328, 374, 442, 424, 425, 394, 255, + /* 1460 */ 254, 385, 372, 261, 359, 372, 442, 385, 173, 372, + /* 1470 */ 445, 263, 367, 444, 262, 440, 442, 372, 248, 374, + /* 1480 */ 270, 265, 441, 439, 359, 401, 455, 461, 267, 244, + /* 1490 */ 367, 20, 367, 405, 336, 385, 337, 372, 372, 374, + /* 1500 */ 340, 340, 372, 454, 399, 372, 372, 372, 403, 328, + /* 1510 */ 372, 406, 407, 408, 409, 410, 411, 385, 413, 165, + /* 1520 */ 383, 355, 340, 418, 399, 420, 95, 340, 403, 424, + /* 1530 */ 425, 406, 407, 408, 409, 410, 411, 367, 413, 423, + /* 1540 */ 359, 95, 363, 418, 372, 420, 349, 36, 367, 424, + /* 1550 */ 425, 336, 330, 372, 331, 374, 392, 386, 340, 396, + /* 1560 */ 0, 386, 353, 0, 328, 326, 353, 341, 189, 0, + /* 1570 */ 0, 0, 42, 353, 35, 201, 35, 35, 35, 201, + /* 1580 */ 399, 0, 35, 35, 403, 201, 0, 406, 407, 408, + /* 1590 */ 409, 410, 411, 201, 413, 359, 35, 0, 22, 0, + /* 1600 */ 35, 420, 0, 367, 184, 424, 425, 182, 372, 0, + /* 1610 */ 374, 0, 178, 177, 0, 0, 0, 0, 0, 47, + /* 1620 */ 328, 42, 0, 0, 0, 0, 0, 0, 0, 0, + /* 1630 */ 151, 0, 151, 35, 0, 399, 328, 0, 0, 403, + /* 1640 */ 0, 0, 406, 407, 408, 409, 410, 411, 0, 413, + /* 1650 */ 0, 359, 42, 0, 0, 0, 420, 0, 0, 367, + /* 1660 */ 424, 425, 0, 0, 372, 0, 374, 359, 22, 0, + /* 1670 */ 0, 0, 135, 0, 0, 367, 0, 0, 0, 0, + /* 1680 */ 372, 0, 374, 0, 35, 0, 0, 0, 58, 42, + /* 1690 */ 58, 399, 47, 0, 14, 403, 14, 328, 406, 407, + /* 1700 */ 408, 409, 410, 411, 44, 413, 39, 399, 47, 0, + /* 1710 */ 0, 403, 40, 47, 406, 407, 408, 409, 410, 411, + /* 1720 */ 0, 413, 173, 39, 0, 0, 0, 39, 359, 35, + /* 1730 */ 39, 0, 35, 0, 39, 35, 367, 0, 0, 447, + /* 1740 */ 448, 372, 39, 374, 48, 0, 48, 48, 35, 48, + /* 1750 */ 39, 0, 328, 64, 0, 0, 0, 35, 22, 0, + /* 1760 */ 104, 102, 35, 35, 22, 35, 458, 459, 399, 328, + /* 1770 */ 35, 0, 403, 0, 35, 406, 407, 408, 409, 410, + /* 1780 */ 411, 35, 413, 359, 44, 44, 22, 35, 22, 420, + /* 1790 */ 0, 367, 22, 50, 425, 35, 372, 0, 374, 0, + /* 1800 */ 359, 22, 35, 35, 0, 20, 194, 35, 367, 35, + /* 1810 */ 96, 95, 0, 372, 163, 374, 35, 0, 185, 0, + /* 1820 */ 22, 0, 44, 399, 249, 3, 253, 403, 96, 165, + /* 1830 */ 406, 407, 408, 409, 410, 411, 228, 413, 328, 95, + /* 1840 */ 399, 47, 95, 95, 403, 96, 171, 406, 407, 408, + /* 1850 */ 409, 410, 411, 163, 413, 163, 95, 328, 44, 96, + /* 1860 */ 95, 35, 44, 47, 44, 96, 3, 44, 249, 359, + /* 1870 */ 95, 35, 448, 96, 364, 170, 170, 367, 96, 35, + /* 1880 */ 35, 35, 372, 35, 374, 249, 47, 96, 359, 96, + /* 1890 */ 47, 44, 0, 0, 0, 0, 367, 95, 39, 96, + /* 1900 */ 459, 372, 95, 374, 96, 95, 95, 0, 95, 399, + /* 1910 */ 39, 105, 47, 403, 164, 2, 406, 407, 408, 409, + /* 1920 */ 410, 411, 95, 413, 328, 44, 22, 95, 399, 96, + /* 1930 */ 207, 96, 403, 95, 230, 406, 407, 408, 409, 410, + /* 1940 */ 411, 22, 413, 228, 415, 243, 166, 228, 95, 47, + /* 1950 */ 96, 33, 95, 47, 22, 359, 95, 106, 96, 95, + /* 1960 */ 364, 35, 209, 367, 96, 35, 48, 95, 372, 96, + /* 1970 */ 374, 35, 54, 55, 56, 57, 58, 95, 35, 96, + /* 1980 */ 328, 95, 35, 96, 95, 35, 96, 107, 119, 119, + /* 1990 */ 44, 95, 95, 119, 95, 399, 328, 119, 35, 403, + /* 2000 */ 95, 22, 406, 407, 408, 409, 410, 411, 64, 413, + /* 2010 */ 35, 359, 94, 35, 63, 97, 364, 35, 35, 367, + /* 2020 */ 35, 35, 35, 35, 372, 35, 374, 359, 35, 70, + /* 2030 */ 92, 35, 22, 44, 35, 367, 35, 35, 35, 35, + /* 2040 */ 372, 70, 374, 35, 35, 22, 35, 35, 35, 35, + /* 2050 */ 0, 399, 35, 48, 39, 403, 0, 328, 406, 407, + /* 2060 */ 408, 409, 410, 411, 35, 413, 48, 399, 0, 39, + /* 2070 */ 35, 403, 48, 39, 406, 407, 408, 409, 410, 411, + /* 2080 */ 0, 413, 164, 165, 35, 48, 39, 169, 359, 0, + /* 2090 */ 172, 0, 35, 22, 35, 21, 367, 22, 22, 21, + /* 2100 */ 462, 372, 20, 374, 462, 462, 188, 462, 462, 462, + /* 2110 */ 462, 462, 328, 462, 462, 462, 462, 462, 462, 462, + /* 2120 */ 462, 462, 462, 462, 462, 462, 462, 462, 399, 328, + /* 2130 */ 462, 462, 403, 462, 462, 406, 407, 408, 409, 410, + /* 2140 */ 411, 462, 413, 359, 462, 462, 462, 462, 462, 462, + /* 2150 */ 462, 367, 462, 462, 462, 462, 372, 462, 374, 462, + /* 2160 */ 359, 462, 462, 462, 462, 462, 462, 462, 367, 462, + /* 2170 */ 462, 462, 462, 372, 462, 374, 462, 462, 462, 462, + /* 2180 */ 462, 462, 462, 399, 462, 462, 328, 403, 462, 462, + /* 2190 */ 406, 407, 408, 409, 410, 411, 462, 413, 462, 462, + /* 2200 */ 399, 462, 462, 328, 403, 462, 462, 406, 407, 408, + /* 2210 */ 409, 410, 411, 462, 413, 462, 462, 359, 462, 462, + /* 2220 */ 462, 462, 462, 462, 462, 367, 462, 462, 462, 462, + /* 2230 */ 372, 462, 374, 462, 359, 462, 462, 462, 462, 462, + /* 2240 */ 462, 462, 367, 462, 462, 462, 462, 372, 328, 374, + /* 2250 */ 462, 462, 462, 462, 462, 462, 462, 399, 462, 462, + /* 2260 */ 462, 403, 462, 462, 406, 407, 408, 409, 410, 411, + /* 2270 */ 462, 413, 462, 462, 399, 462, 462, 462, 403, 359, + /* 2280 */ 462, 406, 407, 408, 409, 410, 411, 367, 413, 462, + /* 2290 */ 462, 462, 372, 462, 374, 462, 462, 462, 462, 462, + /* 2300 */ 462, 462, 462, 462, 462, 462, 462, 462, 462, 328, + /* 2310 */ 462, 462, 462, 462, 462, 462, 462, 462, 462, 399, + /* 2320 */ 462, 462, 462, 403, 462, 328, 406, 407, 408, 409, + /* 2330 */ 410, 411, 462, 413, 462, 462, 462, 462, 462, 462, + /* 2340 */ 359, 462, 462, 462, 462, 462, 462, 462, 367, 462, + /* 2350 */ 462, 462, 462, 372, 462, 374, 359, 462, 462, 462, + /* 2360 */ 462, 462, 462, 462, 367, 462, 462, 462, 462, 372, + /* 2370 */ 462, 374, 462, 462, 462, 462, 462, 462, 462, 462, + /* 2380 */ 399, 462, 462, 462, 403, 462, 328, 406, 407, 408, + /* 2390 */ 409, 410, 411, 462, 413, 462, 399, 462, 462, 462, + /* 2400 */ 403, 462, 462, 406, 407, 408, 409, 410, 411, 462, + /* 2410 */ 413, 462, 462, 462, 462, 462, 462, 359, 462, 462, + /* 2420 */ 462, 462, 462, 462, 462, 367, 462, 462, 462, 462, + /* 2430 */ 372, 462, 374, 462, 462, 462, 462, 462, 462, 462, + /* 2440 */ 462, 328, 462, 462, 462, 462, 462, 462, 462, 462, + /* 2450 */ 462, 462, 462, 462, 462, 462, 462, 399, 328, 462, + /* 2460 */ 462, 403, 462, 462, 406, 407, 408, 409, 410, 411, + /* 2470 */ 462, 413, 359, 462, 462, 462, 462, 462, 462, 462, + /* 2480 */ 367, 462, 462, 462, 462, 372, 462, 374, 462, 359, + /* 2490 */ 462, 462, 462, 462, 462, 462, 462, 367, 462, 462, + /* 2500 */ 462, 462, 372, 462, 374, 462, 462, 462, 462, 462, + /* 2510 */ 462, 462, 399, 462, 462, 328, 403, 462, 462, 406, + /* 2520 */ 407, 408, 409, 410, 411, 462, 413, 462, 462, 399, + /* 2530 */ 462, 462, 328, 403, 462, 462, 406, 407, 408, 409, + /* 2540 */ 410, 411, 462, 413, 462, 462, 359, 462, 462, 462, + /* 2550 */ 462, 462, 462, 462, 367, 462, 462, 462, 462, 372, + /* 2560 */ 462, 374, 462, 359, 462, 462, 462, 462, 462, 462, + /* 2570 */ 462, 367, 462, 462, 462, 462, 372, 328, 374, 462, + /* 2580 */ 462, 462, 462, 462, 462, 462, 399, 462, 462, 462, + /* 2590 */ 403, 462, 462, 406, 407, 408, 409, 410, 411, 462, + /* 2600 */ 413, 462, 462, 399, 462, 462, 462, 403, 359, 462, + /* 2610 */ 406, 407, 408, 409, 410, 411, 367, 413, 462, 462, + /* 2620 */ 462, 372, 462, 374, 462, 462, 462, 462, 462, 462, + /* 2630 */ 462, 462, 462, 462, 462, 462, 462, 462, 328, 462, + /* 2640 */ 462, 462, 462, 462, 462, 462, 462, 462, 399, 462, + /* 2650 */ 462, 462, 403, 462, 328, 406, 407, 408, 409, 410, + /* 2660 */ 411, 462, 413, 462, 462, 462, 462, 462, 462, 359, + /* 2670 */ 462, 462, 462, 462, 462, 462, 462, 367, 462, 462, + /* 2680 */ 462, 462, 372, 462, 374, 359, 462, 462, 462, 462, + /* 2690 */ 462, 462, 462, 367, 462, 462, 462, 462, 372, 462, + /* 2700 */ 374, 462, 462, 462, 462, 462, 462, 462, 462, 399, + /* 2710 */ 462, 462, 462, 403, 462, 328, 406, 407, 408, 409, + /* 2720 */ 410, 411, 462, 413, 462, 399, 462, 462, 462, 403, + /* 2730 */ 462, 462, 406, 407, 408, 409, 410, 411, 462, 413, + /* 2740 */ 462, 462, 462, 462, 462, 462, 359, 462, 462, 462, + /* 2750 */ 462, 462, 462, 462, 367, 462, 462, 462, 462, 372, + /* 2760 */ 462, 374, 462, 462, 462, 462, 462, 462, 462, 462, + /* 2770 */ 328, 462, 462, 462, 462, 462, 462, 462, 462, 462, + /* 2780 */ 462, 462, 462, 462, 462, 462, 399, 328, 462, 462, + /* 2790 */ 403, 462, 462, 406, 407, 408, 409, 410, 411, 462, + /* 2800 */ 413, 359, 462, 462, 462, 462, 462, 462, 462, 367, + /* 2810 */ 462, 462, 462, 462, 372, 462, 374, 462, 359, 462, + /* 2820 */ 462, 462, 462, 462, 462, 462, 367, 462, 462, 462, + /* 2830 */ 462, 372, 462, 374, 462, 462, 462, 462, 462, 462, + /* 2840 */ 462, 399, 462, 462, 328, 403, 462, 462, 406, 407, + /* 2850 */ 408, 409, 410, 411, 462, 413, 462, 462, 399, 462, + /* 2860 */ 462, 328, 403, 462, 462, 406, 407, 408, 409, 410, + /* 2870 */ 411, 462, 413, 462, 462, 359, 462, 462, 462, 462, + /* 2880 */ 462, 462, 462, 367, 462, 462, 462, 462, 372, 462, + /* 2890 */ 374, 462, 359, 462, 462, 462, 462, 462, 462, 462, + /* 2900 */ 367, 462, 462, 462, 462, 372, 462, 374, 462, 462, + /* 2910 */ 462, 462, 462, 462, 462, 399, 462, 462, 462, 403, + /* 2920 */ 462, 462, 406, 407, 408, 409, 410, 411, 462, 413, + /* 2930 */ 462, 462, 399, 462, 462, 462, 403, 462, 462, 406, + /* 2940 */ 407, 408, 409, 410, 411, 462, 413, }; -#define YY_SHIFT_COUNT (713) +#define YY_SHIFT_COUNT (724) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (1975) +#define YY_SHIFT_MAX (2091) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 862, 0, 71, 0, 289, 289, 289, 289, 289, 289, /* 10 */ 289, 289, 289, 289, 289, 360, 576, 576, 647, 576, /* 20 */ 576, 576, 576, 576, 576, 576, 576, 576, 576, 576, /* 30 */ 576, 576, 576, 576, 576, 576, 576, 576, 576, 576, - /* 40 */ 576, 576, 576, 576, 576, 576, 576, 576, 32, 134, - /* 50 */ 21, 590, 133, 7, 104, 7, 21, 21, 1216, 1216, - /* 60 */ 7, 1216, 1216, 136, 7, 423, 25, 25, 423, 401, - /* 70 */ 401, 215, 57, 5, 5, 25, 25, 25, 25, 25, - /* 80 */ 25, 25, 25, 25, 25, 80, 25, 25, 156, 25, - /* 90 */ 25, 25, 343, 25, 25, 343, 25, 343, 343, 343, - /* 100 */ 25, 328, 792, 34, 34, 216, 474, 283, 283, 283, - /* 110 */ 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, - /* 120 */ 283, 283, 283, 283, 283, 283, 1073, 87, 215, 57, - /* 130 */ 593, 313, 499, 499, 499, 710, 229, 229, 313, 374, - /* 140 */ 374, 374, 143, 384, 343, 378, 343, 378, 378, 143, - /* 150 */ 566, 217, 217, 217, 217, 217, 217, 217, 1351, 408, - /* 160 */ 336, 612, 249, 430, 340, 464, 292, 714, 377, 518, - /* 170 */ 873, 656, 858, 748, 12, 754, 748, 769, 353, 836, - /* 180 */ 977, 1192, 1055, 1202, 1226, 1202, 1094, 1236, 1236, 1202, - /* 190 */ 1094, 1094, 1180, 1236, 1236, 1236, 1265, 1265, 1270, 80, - /* 200 */ 1278, 80, 1280, 1283, 80, 1280, 80, 80, 80, 1236, - /* 210 */ 80, 1260, 1260, 1265, 343, 343, 343, 343, 343, 343, - /* 220 */ 343, 343, 343, 343, 343, 1236, 1265, 378, 378, 1129, - /* 230 */ 1270, 328, 1177, 1278, 328, 1236, 1226, 1226, 378, 1117, - /* 240 */ 1124, 378, 1117, 1124, 378, 378, 343, 1122, 1213, 1117, - /* 250 */ 1130, 1132, 1148, 977, 1127, 1134, 1137, 1159, 374, 1398, - /* 260 */ 1236, 1280, 328, 1124, 378, 378, 378, 378, 378, 1124, - /* 270 */ 378, 1262, 328, 143, 328, 374, 1361, 1366, 378, 566, - /* 280 */ 1236, 328, 1437, 1265, 3069, 3069, 3069, 3069, 3069, 3069, - /* 290 */ 3069, 3069, 3069, 887, 310, 94, 625, 15, 59, 738, - /* 300 */ 720, 753, 881, 73, 781, 472, 472, 472, 472, 472, - /* 310 */ 472, 472, 472, 472, 875, 124, 373, 373, 122, 4, - /* 320 */ 58, 166, 484, 300, 300, 790, 912, 588, 790, 790, - /* 330 */ 790, 113, 1057, 288, 969, 913, 884, 918, 923, 986, - /* 340 */ 988, 1093, 1101, 1106, 1005, 1042, 1090, 1011, 425, 945, - /* 350 */ 863, 1096, 1103, 1115, 1116, 1120, 1187, 1131, 1010, 1041, - /* 360 */ 1046, 1140, 1074, 1141, 1147, 1151, 1176, 1179, 1195, 825, - /* 370 */ 1142, 1166, 489, 1181, 1482, 1487, 1300, 1490, 1493, 1452, - /* 380 */ 1495, 1461, 1296, 1463, 1464, 1465, 1302, 1501, 1469, 1470, - /* 390 */ 1306, 1508, 1314, 1518, 1484, 1522, 1502, 1529, 1491, 1347, - /* 400 */ 1352, 1533, 1535, 1358, 1363, 1538, 1541, 1496, 1542, 1544, - /* 410 */ 1545, 1504, 1548, 1549, 1550, 1551, 1552, 1553, 1554, 1555, - /* 420 */ 1406, 1523, 1559, 1410, 1562, 1569, 1571, 1572, 1573, 1574, - /* 430 */ 1576, 1579, 1580, 1583, 1584, 1585, 1587, 1588, 1589, 1556, - /* 440 */ 1590, 1592, 1599, 1600, 1601, 1575, 1602, 1603, 1604, 1459, - /* 450 */ 1595, 1596, 1577, 1606, 1557, 1607, 1558, 1608, 1609, 1568, - /* 460 */ 1581, 1567, 1566, 1605, 1586, 1616, 1593, 1614, 1582, 1598, - /* 470 */ 1618, 1623, 1632, 1611, 1462, 1634, 1639, 1641, 1591, 1645, - /* 480 */ 1646, 1612, 1610, 1613, 1648, 1619, 1615, 1617, 1653, 1622, - /* 490 */ 1620, 1621, 1661, 1627, 1624, 1625, 1665, 1666, 1667, 1669, - /* 500 */ 1570, 1597, 1635, 1649, 1673, 1642, 1643, 1656, 1657, 1636, - /* 510 */ 1637, 1658, 1659, 1654, 1660, 1697, 1680, 1704, 1684, 1662, - /* 520 */ 1707, 1686, 1674, 1710, 1676, 1713, 1679, 1715, 1694, 1698, - /* 530 */ 1682, 1685, 1527, 1626, 1628, 1724, 1563, 1690, 1728, 1546, - /* 540 */ 1708, 1630, 1629, 1735, 1742, 1633, 1631, 1741, 1703, 1499, - /* 550 */ 1650, 1655, 1664, 1638, 1521, 1640, 1500, 1668, 1712, 1687, - /* 560 */ 1681, 1689, 1691, 1692, 1716, 1705, 1714, 1696, 1719, 1530, - /* 570 */ 1693, 1699, 1754, 1731, 1531, 1743, 1746, 1747, 1752, 1762, - /* 580 */ 1763, 1709, 1711, 1756, 1561, 1757, 1759, 1799, 1809, 1811, - /* 590 */ 1717, 1718, 1721, 1720, 1723, 1647, 1726, 1819, 1783, 1670, - /* 600 */ 1734, 1727, 1566, 1786, 1791, 1644, 1671, 1675, 1834, 1815, - /* 610 */ 1651, 1745, 1749, 1748, 1751, 1753, 1760, 1794, 1758, 1764, - /* 620 */ 1795, 1761, 1816, 1652, 1765, 1744, 1766, 1814, 1817, 1768, - /* 630 */ 1769, 1820, 1771, 1773, 1829, 1776, 1777, 1832, 1788, 1790, - /* 640 */ 1833, 1792, 1732, 1770, 1775, 1778, 1866, 1785, 1800, 1846, - /* 650 */ 1804, 1867, 1810, 1846, 1846, 1882, 1843, 1845, 1874, 1875, - /* 660 */ 1876, 1877, 1878, 1879, 1880, 1881, 1883, 1884, 1847, 1828, - /* 670 */ 1885, 1886, 1888, 1890, 1904, 1895, 1905, 1906, 1857, 1636, - /* 680 */ 1907, 1637, 1908, 1909, 1911, 1912, 1917, 1913, 1949, 1916, - /* 690 */ 1910, 1915, 1955, 1921, 1914, 1918, 1959, 1925, 1919, 1924, - /* 700 */ 1964, 1930, 1922, 1932, 1969, 1937, 1939, 1975, 1956, 1958, - /* 710 */ 1960, 1971, 1968, 1957, + /* 40 */ 576, 576, 576, 576, 576, 576, 576, 576, 110, 359, + /* 50 */ 5, 468, 288, 450, 510, 450, 5, 5, 1201, 1201, + /* 60 */ 450, 1201, 1201, 225, 450, 750, 21, 21, 750, 185, + /* 70 */ 185, 39, 336, 40, 40, 21, 21, 21, 21, 21, + /* 80 */ 21, 21, 21, 21, 21, 125, 260, 21, 21, 298, + /* 90 */ 21, 260, 21, 21, 21, 21, 260, 21, 21, 260, + /* 100 */ 21, 260, 260, 260, 21, 545, 792, 34, 34, 216, + /* 110 */ 452, 568, 568, 568, 568, 568, 568, 568, 568, 568, + /* 120 */ 568, 568, 568, 568, 568, 568, 568, 568, 568, 568, + /* 130 */ 214, 87, 39, 336, 2, 301, 136, 136, 136, 429, + /* 140 */ 433, 433, 301, 358, 358, 358, 506, 399, 260, 655, + /* 150 */ 260, 655, 655, 506, 776, 217, 217, 217, 217, 217, + /* 160 */ 217, 217, 961, 529, 303, 609, 183, 619, 340, 287, + /* 170 */ 20, 830, 575, 648, 846, 798, 821, 686, 322, 652, + /* 180 */ 686, 786, 777, 692, 1072, 1285, 1155, 1303, 1328, 1303, + /* 190 */ 1192, 1335, 1335, 1303, 1192, 1192, 1271, 1335, 1335, 1335, + /* 200 */ 1355, 1355, 1358, 125, 1360, 125, 1366, 1368, 125, 1366, + /* 210 */ 125, 125, 125, 1335, 125, 1342, 1342, 1355, 260, 260, + /* 220 */ 260, 260, 260, 260, 260, 260, 260, 260, 260, 1335, + /* 230 */ 1355, 655, 655, 1217, 1320, 1358, 545, 1257, 1360, 545, + /* 240 */ 1335, 1328, 1328, 655, 1204, 1206, 655, 1204, 1206, 655, + /* 250 */ 655, 260, 1202, 1295, 1204, 1208, 1212, 1230, 1072, 1210, + /* 260 */ 1221, 1216, 1245, 358, 1471, 1335, 1366, 545, 545, 1206, + /* 270 */ 655, 655, 655, 655, 655, 1206, 655, 1354, 545, 506, + /* 280 */ 545, 358, 1431, 1446, 655, 776, 1335, 545, 1511, 1355, + /* 290 */ 2947, 2947, 2947, 2947, 2947, 2947, 2947, 2947, 2947, 887, + /* 300 */ 1918, 435, 112, 248, 881, 753, 310, 411, 899, 973, + /* 310 */ 706, 989, 989, 989, 989, 989, 989, 989, 989, 989, + /* 320 */ 922, 486, 292, 292, 366, 219, 640, 330, 67, 317, + /* 330 */ 317, 220, 614, 574, 220, 220, 220, 939, 431, 696, + /* 340 */ 882, 900, 654, 909, 925, 931, 1009, 999, 1098, 1121, + /* 350 */ 951, 1099, 1085, 1091, 990, 978, 860, 964, 1096, 1101, + /* 360 */ 1168, 1174, 1176, 1180, 1200, 1183, 1019, 1153, 644, 1184, + /* 370 */ 1219, 1186, 1207, 1214, 1224, 1231, 1234, 815, 1280, 1296, + /* 380 */ 1211, 1237, 1560, 1563, 1379, 1569, 1570, 1530, 1571, 1539, + /* 390 */ 1374, 1541, 1542, 1543, 1378, 1581, 1547, 1548, 1384, 1586, + /* 400 */ 1392, 1602, 1561, 1597, 1576, 1599, 1565, 1420, 1425, 1609, + /* 410 */ 1611, 1434, 1436, 1614, 1615, 1572, 1616, 1617, 1618, 1579, + /* 420 */ 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1629, 1479, 1598, + /* 430 */ 1631, 1481, 1634, 1637, 1638, 1640, 1641, 1648, 1650, 1657, + /* 440 */ 1658, 1662, 1663, 1665, 1676, 1677, 1678, 1610, 1653, 1654, + /* 450 */ 1655, 1679, 1681, 1646, 1669, 1670, 1671, 1537, 1673, 1674, + /* 460 */ 1649, 1683, 1630, 1685, 1632, 1686, 1687, 1647, 1667, 1660, + /* 470 */ 1645, 1680, 1661, 1682, 1666, 1693, 1672, 1684, 1709, 1710, + /* 480 */ 1720, 1688, 1549, 1724, 1725, 1726, 1689, 1737, 1738, 1694, + /* 490 */ 1696, 1691, 1731, 1697, 1698, 1695, 1733, 1700, 1699, 1703, + /* 500 */ 1745, 1713, 1701, 1711, 1751, 1754, 1755, 1756, 1656, 1659, + /* 510 */ 1722, 1736, 1759, 1727, 1728, 1730, 1735, 1740, 1741, 1739, + /* 520 */ 1746, 1742, 1752, 1771, 1764, 1773, 1766, 1743, 1790, 1770, + /* 530 */ 1760, 1797, 1767, 1799, 1768, 1804, 1779, 1785, 1772, 1774, + /* 540 */ 1612, 1714, 1716, 1812, 1651, 1781, 1817, 1633, 1798, 1690, + /* 550 */ 1664, 1819, 1821, 1692, 1675, 1822, 1778, 1575, 1744, 1732, + /* 560 */ 1747, 1705, 1608, 1706, 1573, 1749, 1814, 1763, 1748, 1761, + /* 570 */ 1765, 1769, 1818, 1794, 1816, 1775, 1820, 1619, 1777, 1782, + /* 580 */ 1863, 1823, 1636, 1826, 1836, 1844, 1845, 1846, 1848, 1791, + /* 590 */ 1793, 1839, 1702, 1847, 1843, 1892, 1893, 1894, 1895, 1802, + /* 600 */ 1859, 1807, 1803, 1808, 1810, 1811, 1780, 1813, 1907, 1871, + /* 610 */ 1750, 1827, 1806, 1645, 1865, 1881, 1715, 1704, 1719, 1913, + /* 620 */ 1904, 1723, 1832, 1833, 1838, 1835, 1853, 1854, 1902, 1857, + /* 630 */ 1861, 1906, 1862, 1919, 1753, 1864, 1851, 1868, 1926, 1930, + /* 640 */ 1872, 1873, 1936, 1882, 1883, 1943, 1886, 1887, 1947, 1889, + /* 650 */ 1890, 1950, 1896, 1869, 1870, 1874, 1878, 1932, 1880, 1897, + /* 660 */ 1946, 1899, 1963, 1905, 1946, 1946, 1979, 1944, 1951, 1975, + /* 670 */ 1978, 1982, 1983, 1985, 1986, 1987, 1988, 1990, 1993, 1959, + /* 680 */ 1938, 1989, 1996, 1999, 2001, 2010, 2002, 2003, 2004, 1971, + /* 690 */ 1740, 2008, 1741, 2009, 2011, 2012, 2013, 2023, 2014, 2050, + /* 700 */ 2017, 2005, 2015, 2056, 2029, 2018, 2030, 2068, 2035, 2024, + /* 710 */ 2034, 2080, 2049, 2037, 2047, 2089, 2057, 2059, 2091, 2071, + /* 720 */ 2074, 2075, 2076, 2078, 2082, }; -#define YY_REDUCE_COUNT (292) -#define YY_REDUCE_MIN (-420) -#define YY_REDUCE_MAX (2656) +#define YY_REDUCE_COUNT (298) +#define YY_REDUCE_MIN (-428) +#define YY_REDUCE_MAX (2533) static const short yy_reduce_ofst[] = { - /* 0 */ 276, -259, 233, 672, 744, 798, 859, 922, 942, 1045, - /* 10 */ 1104, 1158, 1219, 1277, 1331, 1360, 294, 1418, 1472, 1526, - /* 20 */ 1578, 1594, 1683, 1701, 1755, 1772, 1824, 1840, 1892, 1946, - /* 30 */ 1967, 2043, 2061, 2115, 2132, 2184, 2200, 2252, 2275, 2327, - /* 40 */ 2390, 2442, 2461, 2513, 2529, 2581, 2604, 2656, -258, 95, - /* 50 */ 105, -39, 559, 597, 599, 669, 187, 704, -346, -57, - /* 60 */ -391, 323, 480, -276, -250, 75, -222, 297, -371, -330, - /* 70 */ -326, -327, -363, -299, -257, 116, 128, 193, 199, 327, - /* 80 */ 409, 471, 645, 649, 657, 252, 693, 706, -51, 725, - /* 90 */ 747, 762, 244, 763, 772, -203, 791, 465, -5, 362, - /* 100 */ 847, -210, -315, -420, -420, -217, -212, -69, -21, 41, - /* 110 */ 43, 70, 96, 117, 148, 197, 225, 234, 346, 351, - /* 120 */ 405, 446, 454, 504, 651, 655, -266, -170, 397, 191, - /* 130 */ 152, 352, -170, -150, 219, 477, 186, 254, 491, 524, - /* 140 */ 537, 538, -316, -319, 485, 367, 542, 555, 557, 632, - /* 150 */ 530, -342, 369, 389, 410, 463, 468, 483, 319, 577, - /* 160 */ 650, 543, 521, 603, 756, 695, 786, 786, 810, 780, - /* 170 */ 831, 811, 808, 760, 760, 746, 760, 784, 783, 786, - /* 180 */ 834, 842, 851, 866, 870, 869, 878, 924, 926, 879, - /* 190 */ 882, 886, 933, 943, 944, 948, 957, 958, 897, 953, - /* 200 */ 925, 955, 965, 914, 963, 970, 964, 966, 968, 975, - /* 210 */ 971, 976, 978, 989, 959, 962, 967, 979, 983, 984, - /* 220 */ 994, 996, 998, 999, 1001, 987, 992, 990, 991, 938, - /* 230 */ 941, 997, 973, 995, 1028, 1035, 993, 1000, 1002, 936, - /* 240 */ 1003, 1007, 939, 1004, 1009, 1013, 786, 946, 949, 950, - /* 250 */ 947, 956, 972, 1014, 951, 961, 960, 760, 1049, 1016, - /* 260 */ 1085, 1088, 1082, 1043, 1054, 1058, 1059, 1061, 1062, 1044, - /* 270 */ 1063, 1066, 1118, 1089, 1119, 1079, 1038, 1102, 1095, 1121, - /* 280 */ 1136, 1126, 1143, 1145, 1087, 1081, 1097, 1098, 1135, 1138, - /* 290 */ 1139, 1144, 1154, + /* 0 */ 221, -298, -9, 672, 745, 765, 832, 888, 933, 1032, + /* 10 */ 1105, 1125, 291, 1181, 1236, 75, 1292, 1308, 1369, 1424, + /* 20 */ 234, 1441, 1510, 1529, 1596, 1652, 1668, 1729, 1784, 1801, + /* 30 */ 1858, 1875, 1920, 1981, 1997, 2058, 2113, 2130, 2187, 2204, + /* 40 */ 2249, 2310, 2326, 2387, 2442, 2459, 2516, 2533, -279, -293, + /* 50 */ 294, -333, 297, 710, 797, 833, 896, 937, -351, -317, + /* 60 */ -428, -335, 700, -425, -24, -369, -215, -181, -371, -296, + /* 70 */ -255, -234, -199, 12, 139, 184, 264, 316, 320, 353, + /* 80 */ 378, 396, 471, 552, 643, -267, -294, 676, 678, -223, + /* 90 */ 727, 30, 734, 738, 759, 764, 172, 843, 858, 261, + /* 100 */ 859, 388, 273, 488, 848, 334, 122, -421, -421, -235, + /* 110 */ -94, -5, 152, 263, 363, 364, 402, 413, 445, 476, + /* 120 */ 620, 662, 685, 697, 704, 714, 761, 778, 787, 810, + /* 130 */ -106, -360, -179, -20, -228, 100, -360, -303, -17, 207, + /* 140 */ 175, 401, 149, -188, 373, 404, -346, 133, 484, 230, + /* 150 */ 173, 455, 512, 554, 410, 463, 660, 680, 756, 781, + /* 160 */ 817, 845, 667, 758, 659, 728, 747, 762, 879, 831, + /* 170 */ 918, 918, 883, 837, 948, 917, 920, 877, 877, 863, + /* 180 */ 877, 891, 884, 918, 921, 934, 947, 968, 967, 971, + /* 190 */ 977, 1020, 1023, 982, 985, 986, 1027, 1036, 1037, 1038, + /* 200 */ 1046, 1047, 984, 1039, 1008, 1042, 1050, 1000, 1051, 1052, + /* 210 */ 1053, 1054, 1055, 1056, 1058, 1059, 1062, 1073, 1043, 1048, + /* 220 */ 1057, 1060, 1061, 1069, 1070, 1071, 1075, 1077, 1078, 1074, + /* 230 */ 1081, 1033, 1040, 1016, 1021, 1022, 1092, 1064, 1080, 1106, + /* 240 */ 1111, 1066, 1067, 1079, 1013, 1076, 1090, 1024, 1082, 1093, + /* 250 */ 1097, 918, 1025, 1029, 1034, 1041, 1035, 1044, 1084, 1026, + /* 260 */ 1031, 1049, 877, 1123, 1088, 1158, 1159, 1160, 1161, 1110, + /* 270 */ 1126, 1130, 1133, 1134, 1135, 1132, 1138, 1137, 1182, 1166, + /* 280 */ 1187, 1170, 1116, 1179, 1172, 1197, 1215, 1218, 1223, 1222, + /* 290 */ 1164, 1163, 1171, 1175, 1209, 1213, 1220, 1226, 1239, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 10 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 20 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 30 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 40 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 50 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 60 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 70 */ 1606, 1861, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 80 */ 1606, 1606, 1606, 1606, 1606, 1684, 1606, 1606, 1606, 1606, - /* 90 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 100 */ 1606, 1682, 1854, 2057, 1606, 1606, 1606, 1606, 1606, 1606, - /* 110 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 120 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 2069, 1606, 1606, - /* 130 */ 1684, 1606, 2069, 2069, 2069, 1682, 2029, 2029, 1606, 1606, - /* 140 */ 1606, 1606, 1791, 1606, 1606, 1606, 1606, 1606, 1606, 1791, - /* 150 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1903, 1606, - /* 160 */ 1606, 2094, 2148, 1606, 1606, 2097, 1606, 1606, 1606, 1866, - /* 170 */ 1606, 1744, 2084, 2061, 2075, 2132, 2062, 2059, 2078, 1606, - /* 180 */ 2088, 1606, 1896, 1859, 1606, 1859, 1856, 1606, 1606, 1859, - /* 190 */ 1856, 1856, 1735, 1606, 1606, 1606, 1606, 1606, 1606, 1684, - /* 200 */ 1606, 1684, 1606, 1606, 1684, 1606, 1684, 1684, 1684, 1606, - /* 210 */ 1684, 1663, 1663, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 220 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1916, - /* 230 */ 1606, 1682, 1905, 1606, 1682, 1606, 1606, 1606, 1606, 2105, - /* 240 */ 2103, 1606, 2105, 2103, 1606, 1606, 1606, 2117, 2113, 2105, - /* 250 */ 2121, 2119, 2090, 2088, 2151, 2138, 2134, 2075, 1606, 1606, - /* 260 */ 1606, 1606, 1682, 2103, 1606, 1606, 1606, 1606, 1606, 2103, - /* 270 */ 1606, 1606, 1682, 1606, 1682, 1606, 1606, 1760, 1606, 1606, - /* 280 */ 1606, 1682, 1638, 1606, 1898, 1909, 1881, 1881, 1794, 1794, - /* 290 */ 1794, 1685, 1611, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 300 */ 1606, 1606, 1606, 1606, 1606, 2116, 2115, 1984, 1606, 2033, - /* 310 */ 2032, 2031, 2022, 1983, 1756, 1606, 1982, 1981, 1606, 1606, - /* 320 */ 1606, 1606, 1606, 1872, 1871, 1975, 1606, 1606, 1976, 1974, - /* 330 */ 1973, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 340 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 2135, 2139, - /* 350 */ 1606, 1606, 1606, 1606, 1606, 1606, 2058, 1606, 1606, 1606, - /* 360 */ 1606, 1606, 1958, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 370 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 380 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 390 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 400 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 410 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 420 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 430 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 440 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 450 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 460 */ 1606, 1643, 1963, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 470 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 480 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 490 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 500 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1723, - /* 510 */ 1722, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 520 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 530 */ 1606, 1606, 1606, 1966, 1606, 1606, 1606, 1606, 1606, 1606, - /* 540 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 2131, 2091, 1606, - /* 550 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 560 */ 1606, 1606, 1606, 1606, 1606, 1606, 1958, 1606, 2114, 1606, - /* 570 */ 1606, 2129, 1606, 2133, 1606, 1606, 1606, 1606, 1606, 1606, - /* 580 */ 1606, 2068, 2064, 1606, 1606, 2060, 1606, 1606, 1606, 1606, - /* 590 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 600 */ 1606, 1606, 1957, 1606, 2019, 1606, 1606, 1606, 2053, 1606, - /* 610 */ 1606, 2004, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 620 */ 1606, 1966, 1606, 1969, 1606, 1606, 1606, 1606, 1606, 1788, - /* 630 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 640 */ 1606, 1606, 1773, 1771, 1770, 1769, 1606, 1766, 1606, 1801, - /* 650 */ 1606, 1606, 1606, 1797, 1796, 1606, 1606, 1606, 1606, 1606, - /* 660 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 670 */ 1703, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1695, - /* 680 */ 1606, 1694, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 690 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 700 */ 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606, - /* 710 */ 1606, 1606, 1606, 1606, + /* 0 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 10 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 20 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 30 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 40 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 50 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 60 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 70 */ 1621, 1876, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 80 */ 1621, 1621, 1621, 1621, 1621, 1699, 1621, 1621, 1621, 1621, + /* 90 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 100 */ 1621, 1621, 1621, 1621, 1621, 1697, 1869, 2075, 1621, 1621, + /* 110 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 120 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 130 */ 1621, 2087, 1621, 1621, 1699, 1621, 2087, 2087, 2087, 1697, + /* 140 */ 2047, 2047, 1621, 1621, 1621, 1621, 1806, 1621, 1621, 1621, + /* 150 */ 1621, 1621, 1621, 1806, 1621, 1621, 1621, 1621, 1621, 1621, + /* 160 */ 1621, 1621, 1919, 1621, 1621, 2112, 2166, 1621, 1621, 2115, + /* 170 */ 1621, 1621, 1621, 1881, 1621, 1759, 2102, 2079, 2093, 2150, + /* 180 */ 2080, 2077, 2096, 1621, 2106, 1621, 1912, 1874, 1621, 1874, + /* 190 */ 1871, 1621, 1621, 1874, 1871, 1871, 1750, 1621, 1621, 1621, + /* 200 */ 1621, 1621, 1621, 1699, 1621, 1699, 1621, 1621, 1699, 1621, + /* 210 */ 1699, 1699, 1699, 1621, 1699, 1678, 1678, 1621, 1621, 1621, + /* 220 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 230 */ 1621, 1621, 1621, 1934, 1925, 1621, 1697, 1921, 1621, 1697, + /* 240 */ 1621, 1621, 1621, 1621, 2123, 2121, 1621, 2123, 2121, 1621, + /* 250 */ 1621, 1621, 2135, 2131, 2123, 2139, 2137, 2108, 2106, 2169, + /* 260 */ 2156, 2152, 2093, 1621, 1621, 1621, 1621, 1697, 1697, 2121, + /* 270 */ 1621, 1621, 1621, 1621, 1621, 2121, 1621, 1621, 1697, 1621, + /* 280 */ 1697, 1621, 1621, 1775, 1621, 1621, 1621, 1697, 1653, 1621, + /* 290 */ 1914, 1927, 1897, 1897, 1809, 1809, 1809, 1700, 1626, 1621, + /* 300 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 310 */ 1621, 2134, 2133, 2002, 1621, 2051, 2050, 2049, 2040, 2001, + /* 320 */ 1771, 1621, 2000, 1999, 1621, 1621, 1621, 1621, 1621, 1888, + /* 330 */ 1887, 1993, 1621, 1621, 1994, 1992, 1991, 1621, 1621, 1621, + /* 340 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 350 */ 1621, 1621, 1621, 1621, 1621, 2153, 2157, 1621, 1621, 1621, + /* 360 */ 1621, 1621, 1621, 1621, 2076, 1621, 1621, 1621, 1621, 1621, + /* 370 */ 1976, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 380 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 390 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 400 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 410 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 420 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 430 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 440 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 450 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 460 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1658, + /* 470 */ 1981, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 480 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 490 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 500 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 510 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1738, 1737, 1621, + /* 520 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 530 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 540 */ 1621, 1984, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 550 */ 1621, 1621, 1621, 1621, 1621, 2149, 2109, 1621, 1621, 1621, + /* 560 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 570 */ 1621, 1621, 1621, 1621, 1976, 1621, 2132, 1621, 1621, 2147, + /* 580 */ 1621, 2151, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 2086, + /* 590 */ 2082, 1621, 1621, 2078, 1621, 1621, 1621, 1621, 1621, 1621, + /* 600 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 610 */ 1621, 1621, 1621, 1975, 1621, 2037, 1621, 1621, 1621, 2071, + /* 620 */ 1621, 1621, 2022, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 630 */ 1621, 1621, 1984, 1621, 1987, 1621, 1621, 1621, 1621, 1621, + /* 640 */ 1803, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 650 */ 1621, 1621, 1621, 1788, 1786, 1785, 1784, 1621, 1781, 1621, + /* 660 */ 1816, 1621, 1621, 1621, 1812, 1811, 1621, 1621, 1621, 1621, + /* 670 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 680 */ 1621, 1718, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 690 */ 1710, 1621, 1709, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 700 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 710 */ 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, 1621, + /* 720 */ 1621, 1621, 1621, 1621, 1621, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1882,70 +1826,71 @@ static const char *const yyTokenName[] = { /* 394 */ "bufsize_opt", /* 395 */ "stream_name", /* 396 */ "stream_options", - /* 397 */ "subtable_opt", - /* 398 */ "expression", - /* 399 */ "dnode_list", - /* 400 */ "where_clause_opt", - /* 401 */ "signed", - /* 402 */ "literal_func", - /* 403 */ "literal_list", - /* 404 */ "table_alias", - /* 405 */ "expr_or_subquery", - /* 406 */ "pseudo_column", - /* 407 */ "column_reference", - /* 408 */ "function_expression", - /* 409 */ "case_when_expression", - /* 410 */ "star_func", - /* 411 */ "star_func_para_list", - /* 412 */ "noarg_func", - /* 413 */ "other_para_list", - /* 414 */ "star_func_para", - /* 415 */ "when_then_list", - /* 416 */ "case_when_else_opt", - /* 417 */ "common_expression", - /* 418 */ "when_then_expr", - /* 419 */ "predicate", - /* 420 */ "compare_op", - /* 421 */ "in_op", - /* 422 */ "in_predicate_value", - /* 423 */ "boolean_value_expression", - /* 424 */ "boolean_primary", - /* 425 */ "from_clause_opt", - /* 426 */ "table_reference_list", - /* 427 */ "table_reference", - /* 428 */ "table_primary", - /* 429 */ "joined_table", - /* 430 */ "alias_opt", - /* 431 */ "subquery", - /* 432 */ "parenthesized_joined_table", - /* 433 */ "join_type", - /* 434 */ "search_condition", - /* 435 */ "query_specification", - /* 436 */ "set_quantifier_opt", - /* 437 */ "select_list", - /* 438 */ "partition_by_clause_opt", - /* 439 */ "range_opt", - /* 440 */ "every_opt", - /* 441 */ "fill_opt", - /* 442 */ "twindow_clause_opt", - /* 443 */ "group_by_clause_opt", - /* 444 */ "having_clause_opt", - /* 445 */ "select_item", - /* 446 */ "partition_list", - /* 447 */ "partition_item", - /* 448 */ "fill_mode", - /* 449 */ "group_by_list", - /* 450 */ "query_expression", - /* 451 */ "query_simple", - /* 452 */ "order_by_clause_opt", - /* 453 */ "slimit_clause_opt", - /* 454 */ "limit_clause_opt", - /* 455 */ "union_query_expression", - /* 456 */ "query_simple_or_subquery", - /* 457 */ "sort_specification_list", - /* 458 */ "sort_specification", - /* 459 */ "ordering_specification_opt", - /* 460 */ "null_ordering_opt", + /* 397 */ "col_list_opt", + /* 398 */ "subtable_opt", + /* 399 */ "expression", + /* 400 */ "dnode_list", + /* 401 */ "where_clause_opt", + /* 402 */ "signed", + /* 403 */ "literal_func", + /* 404 */ "literal_list", + /* 405 */ "table_alias", + /* 406 */ "expr_or_subquery", + /* 407 */ "pseudo_column", + /* 408 */ "column_reference", + /* 409 */ "function_expression", + /* 410 */ "case_when_expression", + /* 411 */ "star_func", + /* 412 */ "star_func_para_list", + /* 413 */ "noarg_func", + /* 414 */ "other_para_list", + /* 415 */ "star_func_para", + /* 416 */ "when_then_list", + /* 417 */ "case_when_else_opt", + /* 418 */ "common_expression", + /* 419 */ "when_then_expr", + /* 420 */ "predicate", + /* 421 */ "compare_op", + /* 422 */ "in_op", + /* 423 */ "in_predicate_value", + /* 424 */ "boolean_value_expression", + /* 425 */ "boolean_primary", + /* 426 */ "from_clause_opt", + /* 427 */ "table_reference_list", + /* 428 */ "table_reference", + /* 429 */ "table_primary", + /* 430 */ "joined_table", + /* 431 */ "alias_opt", + /* 432 */ "subquery", + /* 433 */ "parenthesized_joined_table", + /* 434 */ "join_type", + /* 435 */ "search_condition", + /* 436 */ "query_specification", + /* 437 */ "set_quantifier_opt", + /* 438 */ "select_list", + /* 439 */ "partition_by_clause_opt", + /* 440 */ "range_opt", + /* 441 */ "every_opt", + /* 442 */ "fill_opt", + /* 443 */ "twindow_clause_opt", + /* 444 */ "group_by_clause_opt", + /* 445 */ "having_clause_opt", + /* 446 */ "select_item", + /* 447 */ "partition_list", + /* 448 */ "partition_item", + /* 449 */ "fill_mode", + /* 450 */ "group_by_list", + /* 451 */ "query_expression", + /* 452 */ "query_simple", + /* 453 */ "order_by_clause_opt", + /* 454 */ "slimit_clause_opt", + /* 455 */ "limit_clause_opt", + /* 456 */ "union_query_expression", + /* 457 */ "query_simple_or_subquery", + /* 458 */ "sort_specification_list", + /* 459 */ "sort_specification", + /* 460 */ "ordering_specification_opt", + /* 461 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -2214,290 +2159,293 @@ static const char *const yyRuleName[] = { /* 258 */ "tag_item ::= column_name column_alias", /* 259 */ "tag_item ::= column_name AS column_alias", /* 260 */ "cmd ::= CREATE SMA INDEX not_exists_opt full_table_name ON full_table_name index_options", - /* 261 */ "cmd ::= DROP INDEX exists_opt full_table_name", - /* 262 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", - /* 263 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", - /* 264 */ "func_list ::= func", - /* 265 */ "func_list ::= func_list NK_COMMA func", - /* 266 */ "func ::= sma_func_name NK_LP expression_list NK_RP", - /* 267 */ "sma_func_name ::= function_name", - /* 268 */ "sma_func_name ::= COUNT", - /* 269 */ "sma_func_name ::= FIRST", - /* 270 */ "sma_func_name ::= LAST", - /* 271 */ "sma_func_name ::= LAST_ROW", - /* 272 */ "sma_stream_opt ::=", - /* 273 */ "sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal", - /* 274 */ "sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal", - /* 275 */ "sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal", - /* 276 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery", - /* 277 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", - /* 278 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", - /* 279 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name", - /* 280 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name", - /* 281 */ "cmd ::= DROP TOPIC exists_opt topic_name", - /* 282 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", - /* 283 */ "cmd ::= DESC full_table_name", - /* 284 */ "cmd ::= DESCRIBE full_table_name", - /* 285 */ "cmd ::= RESET QUERY CACHE", - /* 286 */ "cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery", - /* 287 */ "analyze_opt ::=", - /* 288 */ "analyze_opt ::= ANALYZE", - /* 289 */ "explain_options ::=", - /* 290 */ "explain_options ::= explain_options VERBOSE NK_BOOL", - /* 291 */ "explain_options ::= explain_options RATIO NK_FLOAT", - /* 292 */ "cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt", - /* 293 */ "cmd ::= DROP FUNCTION exists_opt function_name", - /* 294 */ "agg_func_opt ::=", - /* 295 */ "agg_func_opt ::= AGGREGATE", - /* 296 */ "bufsize_opt ::=", - /* 297 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", - /* 298 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name tags_def_opt subtable_opt AS query_or_subquery", - /* 299 */ "cmd ::= DROP STREAM exists_opt stream_name", - /* 300 */ "stream_options ::=", - /* 301 */ "stream_options ::= stream_options TRIGGER AT_ONCE", - /* 302 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", - /* 303 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", - /* 304 */ "stream_options ::= stream_options WATERMARK duration_literal", - /* 305 */ "stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER", - /* 306 */ "stream_options ::= stream_options FILL_HISTORY NK_INTEGER", - /* 307 */ "subtable_opt ::=", - /* 308 */ "subtable_opt ::= SUBTABLE NK_LP expression NK_RP", - /* 309 */ "cmd ::= KILL CONNECTION NK_INTEGER", - /* 310 */ "cmd ::= KILL QUERY NK_STRING", - /* 311 */ "cmd ::= KILL TRANSACTION NK_INTEGER", - /* 312 */ "cmd ::= BALANCE VGROUP", - /* 313 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", - /* 314 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", - /* 315 */ "cmd ::= SPLIT VGROUP NK_INTEGER", - /* 316 */ "dnode_list ::= DNODE NK_INTEGER", - /* 317 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", - /* 318 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", - /* 319 */ "cmd ::= query_or_subquery", - /* 320 */ "cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery", - /* 321 */ "cmd ::= INSERT INTO full_table_name query_or_subquery", - /* 322 */ "literal ::= NK_INTEGER", - /* 323 */ "literal ::= NK_FLOAT", - /* 324 */ "literal ::= NK_STRING", - /* 325 */ "literal ::= NK_BOOL", - /* 326 */ "literal ::= TIMESTAMP NK_STRING", - /* 327 */ "literal ::= duration_literal", - /* 328 */ "literal ::= NULL", - /* 329 */ "literal ::= NK_QUESTION", - /* 330 */ "duration_literal ::= NK_VARIABLE", - /* 331 */ "signed ::= NK_INTEGER", - /* 332 */ "signed ::= NK_PLUS NK_INTEGER", - /* 333 */ "signed ::= NK_MINUS NK_INTEGER", - /* 334 */ "signed ::= NK_FLOAT", - /* 335 */ "signed ::= NK_PLUS NK_FLOAT", - /* 336 */ "signed ::= NK_MINUS NK_FLOAT", - /* 337 */ "signed_literal ::= signed", - /* 338 */ "signed_literal ::= NK_STRING", - /* 339 */ "signed_literal ::= NK_BOOL", - /* 340 */ "signed_literal ::= TIMESTAMP NK_STRING", - /* 341 */ "signed_literal ::= duration_literal", - /* 342 */ "signed_literal ::= NULL", - /* 343 */ "signed_literal ::= literal_func", - /* 344 */ "signed_literal ::= NK_QUESTION", - /* 345 */ "literal_list ::= signed_literal", - /* 346 */ "literal_list ::= literal_list NK_COMMA signed_literal", - /* 347 */ "db_name ::= NK_ID", - /* 348 */ "table_name ::= NK_ID", - /* 349 */ "column_name ::= NK_ID", - /* 350 */ "function_name ::= NK_ID", - /* 351 */ "table_alias ::= NK_ID", - /* 352 */ "column_alias ::= NK_ID", - /* 353 */ "user_name ::= NK_ID", - /* 354 */ "topic_name ::= NK_ID", - /* 355 */ "stream_name ::= NK_ID", - /* 356 */ "cgroup_name ::= NK_ID", - /* 357 */ "expr_or_subquery ::= expression", - /* 358 */ "expression ::= literal", - /* 359 */ "expression ::= pseudo_column", - /* 360 */ "expression ::= column_reference", - /* 361 */ "expression ::= function_expression", - /* 362 */ "expression ::= case_when_expression", - /* 363 */ "expression ::= NK_LP expression NK_RP", - /* 364 */ "expression ::= NK_PLUS expr_or_subquery", - /* 365 */ "expression ::= NK_MINUS expr_or_subquery", - /* 366 */ "expression ::= expr_or_subquery NK_PLUS expr_or_subquery", - /* 367 */ "expression ::= expr_or_subquery NK_MINUS expr_or_subquery", - /* 368 */ "expression ::= expr_or_subquery NK_STAR expr_or_subquery", - /* 369 */ "expression ::= expr_or_subquery NK_SLASH expr_or_subquery", - /* 370 */ "expression ::= expr_or_subquery NK_REM expr_or_subquery", - /* 371 */ "expression ::= column_reference NK_ARROW NK_STRING", - /* 372 */ "expression ::= expr_or_subquery NK_BITAND expr_or_subquery", - /* 373 */ "expression ::= expr_or_subquery NK_BITOR expr_or_subquery", - /* 374 */ "expression_list ::= expr_or_subquery", - /* 375 */ "expression_list ::= expression_list NK_COMMA expr_or_subquery", - /* 376 */ "column_reference ::= column_name", - /* 377 */ "column_reference ::= table_name NK_DOT column_name", - /* 378 */ "pseudo_column ::= ROWTS", - /* 379 */ "pseudo_column ::= TBNAME", - /* 380 */ "pseudo_column ::= table_name NK_DOT TBNAME", - /* 381 */ "pseudo_column ::= QSTART", - /* 382 */ "pseudo_column ::= QEND", - /* 383 */ "pseudo_column ::= QDURATION", - /* 384 */ "pseudo_column ::= WSTART", - /* 385 */ "pseudo_column ::= WEND", - /* 386 */ "pseudo_column ::= WDURATION", - /* 387 */ "pseudo_column ::= IROWTS", - /* 388 */ "pseudo_column ::= ISFILLED", - /* 389 */ "pseudo_column ::= QTAGS", - /* 390 */ "function_expression ::= function_name NK_LP expression_list NK_RP", - /* 391 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", - /* 392 */ "function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP", - /* 393 */ "function_expression ::= literal_func", - /* 394 */ "literal_func ::= noarg_func NK_LP NK_RP", - /* 395 */ "literal_func ::= NOW", - /* 396 */ "noarg_func ::= NOW", - /* 397 */ "noarg_func ::= TODAY", - /* 398 */ "noarg_func ::= TIMEZONE", - /* 399 */ "noarg_func ::= DATABASE", - /* 400 */ "noarg_func ::= CLIENT_VERSION", - /* 401 */ "noarg_func ::= SERVER_VERSION", - /* 402 */ "noarg_func ::= SERVER_STATUS", - /* 403 */ "noarg_func ::= CURRENT_USER", - /* 404 */ "noarg_func ::= USER", - /* 405 */ "star_func ::= COUNT", - /* 406 */ "star_func ::= FIRST", - /* 407 */ "star_func ::= LAST", - /* 408 */ "star_func ::= LAST_ROW", - /* 409 */ "star_func_para_list ::= NK_STAR", - /* 410 */ "star_func_para_list ::= other_para_list", - /* 411 */ "other_para_list ::= star_func_para", - /* 412 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", - /* 413 */ "star_func_para ::= expr_or_subquery", - /* 414 */ "star_func_para ::= table_name NK_DOT NK_STAR", - /* 415 */ "case_when_expression ::= CASE when_then_list case_when_else_opt END", - /* 416 */ "case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END", - /* 417 */ "when_then_list ::= when_then_expr", - /* 418 */ "when_then_list ::= when_then_list when_then_expr", - /* 419 */ "when_then_expr ::= WHEN common_expression THEN common_expression", - /* 420 */ "case_when_else_opt ::=", - /* 421 */ "case_when_else_opt ::= ELSE common_expression", - /* 422 */ "predicate ::= expr_or_subquery compare_op expr_or_subquery", - /* 423 */ "predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery", - /* 424 */ "predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery", - /* 425 */ "predicate ::= expr_or_subquery IS NULL", - /* 426 */ "predicate ::= expr_or_subquery IS NOT NULL", - /* 427 */ "predicate ::= expr_or_subquery in_op in_predicate_value", - /* 428 */ "compare_op ::= NK_LT", - /* 429 */ "compare_op ::= NK_GT", - /* 430 */ "compare_op ::= NK_LE", - /* 431 */ "compare_op ::= NK_GE", - /* 432 */ "compare_op ::= NK_NE", - /* 433 */ "compare_op ::= NK_EQ", - /* 434 */ "compare_op ::= LIKE", - /* 435 */ "compare_op ::= NOT LIKE", - /* 436 */ "compare_op ::= MATCH", - /* 437 */ "compare_op ::= NMATCH", - /* 438 */ "compare_op ::= CONTAINS", - /* 439 */ "in_op ::= IN", - /* 440 */ "in_op ::= NOT IN", - /* 441 */ "in_predicate_value ::= NK_LP literal_list NK_RP", - /* 442 */ "boolean_value_expression ::= boolean_primary", - /* 443 */ "boolean_value_expression ::= NOT boolean_primary", - /* 444 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", - /* 445 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", - /* 446 */ "boolean_primary ::= predicate", - /* 447 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 448 */ "common_expression ::= expr_or_subquery", - /* 449 */ "common_expression ::= boolean_value_expression", - /* 450 */ "from_clause_opt ::=", - /* 451 */ "from_clause_opt ::= FROM table_reference_list", - /* 452 */ "table_reference_list ::= table_reference", - /* 453 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 454 */ "table_reference ::= table_primary", - /* 455 */ "table_reference ::= joined_table", - /* 456 */ "table_primary ::= table_name alias_opt", - /* 457 */ "table_primary ::= db_name NK_DOT table_name alias_opt", - /* 458 */ "table_primary ::= subquery alias_opt", - /* 459 */ "table_primary ::= parenthesized_joined_table", - /* 460 */ "alias_opt ::=", - /* 461 */ "alias_opt ::= table_alias", - /* 462 */ "alias_opt ::= AS table_alias", - /* 463 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 464 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 465 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 466 */ "join_type ::=", - /* 467 */ "join_type ::= INNER", - /* 468 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 469 */ "set_quantifier_opt ::=", - /* 470 */ "set_quantifier_opt ::= DISTINCT", - /* 471 */ "set_quantifier_opt ::= ALL", - /* 472 */ "select_list ::= select_item", - /* 473 */ "select_list ::= select_list NK_COMMA select_item", - /* 474 */ "select_item ::= NK_STAR", - /* 475 */ "select_item ::= common_expression", - /* 476 */ "select_item ::= common_expression column_alias", - /* 477 */ "select_item ::= common_expression AS column_alias", - /* 478 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 479 */ "where_clause_opt ::=", - /* 480 */ "where_clause_opt ::= WHERE search_condition", - /* 481 */ "partition_by_clause_opt ::=", - /* 482 */ "partition_by_clause_opt ::= PARTITION BY partition_list", - /* 483 */ "partition_list ::= partition_item", - /* 484 */ "partition_list ::= partition_list NK_COMMA partition_item", - /* 485 */ "partition_item ::= expr_or_subquery", - /* 486 */ "partition_item ::= expr_or_subquery column_alias", - /* 487 */ "partition_item ::= expr_or_subquery AS column_alias", - /* 488 */ "twindow_clause_opt ::=", - /* 489 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", - /* 490 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP", - /* 491 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 492 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 493 */ "twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition", - /* 494 */ "sliding_opt ::=", - /* 495 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 496 */ "fill_opt ::=", - /* 497 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 498 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 499 */ "fill_mode ::= NONE", - /* 500 */ "fill_mode ::= PREV", - /* 501 */ "fill_mode ::= NULL", - /* 502 */ "fill_mode ::= LINEAR", - /* 503 */ "fill_mode ::= NEXT", - /* 504 */ "group_by_clause_opt ::=", - /* 505 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 506 */ "group_by_list ::= expr_or_subquery", - /* 507 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", - /* 508 */ "having_clause_opt ::=", - /* 509 */ "having_clause_opt ::= HAVING search_condition", - /* 510 */ "range_opt ::=", - /* 511 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", - /* 512 */ "every_opt ::=", - /* 513 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", - /* 514 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 515 */ "query_simple ::= query_specification", - /* 516 */ "query_simple ::= union_query_expression", - /* 517 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", - /* 518 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", - /* 519 */ "query_simple_or_subquery ::= query_simple", - /* 520 */ "query_simple_or_subquery ::= subquery", - /* 521 */ "query_or_subquery ::= query_expression", - /* 522 */ "query_or_subquery ::= subquery", - /* 523 */ "order_by_clause_opt ::=", - /* 524 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 525 */ "slimit_clause_opt ::=", - /* 526 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 527 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 528 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 529 */ "limit_clause_opt ::=", - /* 530 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 531 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 532 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 533 */ "subquery ::= NK_LP query_expression NK_RP", - /* 534 */ "subquery ::= NK_LP subquery NK_RP", - /* 535 */ "search_condition ::= common_expression", - /* 536 */ "sort_specification_list ::= sort_specification", - /* 537 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 538 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", - /* 539 */ "ordering_specification_opt ::=", - /* 540 */ "ordering_specification_opt ::= ASC", - /* 541 */ "ordering_specification_opt ::= DESC", - /* 542 */ "null_ordering_opt ::=", - /* 543 */ "null_ordering_opt ::= NULLS FIRST", - /* 544 */ "null_ordering_opt ::= NULLS LAST", + /* 261 */ "cmd ::= CREATE INDEX not_exists_opt full_table_name ON full_table_name NK_LP col_name_list NK_RP", + /* 262 */ "cmd ::= DROP INDEX exists_opt full_table_name", + /* 263 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", + /* 264 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", + /* 265 */ "func_list ::= func", + /* 266 */ "func_list ::= func_list NK_COMMA func", + /* 267 */ "func ::= sma_func_name NK_LP expression_list NK_RP", + /* 268 */ "sma_func_name ::= function_name", + /* 269 */ "sma_func_name ::= COUNT", + /* 270 */ "sma_func_name ::= FIRST", + /* 271 */ "sma_func_name ::= LAST", + /* 272 */ "sma_func_name ::= LAST_ROW", + /* 273 */ "sma_stream_opt ::=", + /* 274 */ "sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal", + /* 275 */ "sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal", + /* 276 */ "sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal", + /* 277 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery", + /* 278 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", + /* 279 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", + /* 280 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name", + /* 281 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name", + /* 282 */ "cmd ::= DROP TOPIC exists_opt topic_name", + /* 283 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", + /* 284 */ "cmd ::= DESC full_table_name", + /* 285 */ "cmd ::= DESCRIBE full_table_name", + /* 286 */ "cmd ::= RESET QUERY CACHE", + /* 287 */ "cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery", + /* 288 */ "analyze_opt ::=", + /* 289 */ "analyze_opt ::= ANALYZE", + /* 290 */ "explain_options ::=", + /* 291 */ "explain_options ::= explain_options VERBOSE NK_BOOL", + /* 292 */ "explain_options ::= explain_options RATIO NK_FLOAT", + /* 293 */ "cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt", + /* 294 */ "cmd ::= DROP FUNCTION exists_opt function_name", + /* 295 */ "agg_func_opt ::=", + /* 296 */ "agg_func_opt ::= AGGREGATE", + /* 297 */ "bufsize_opt ::=", + /* 298 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", + /* 299 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tags_def_opt subtable_opt AS query_or_subquery", + /* 300 */ "cmd ::= DROP STREAM exists_opt stream_name", + /* 301 */ "col_list_opt ::=", + /* 302 */ "col_list_opt ::= NK_LP col_name_list NK_RP", + /* 303 */ "stream_options ::=", + /* 304 */ "stream_options ::= stream_options TRIGGER AT_ONCE", + /* 305 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", + /* 306 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", + /* 307 */ "stream_options ::= stream_options WATERMARK duration_literal", + /* 308 */ "stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER", + /* 309 */ "stream_options ::= stream_options FILL_HISTORY NK_INTEGER", + /* 310 */ "subtable_opt ::=", + /* 311 */ "subtable_opt ::= SUBTABLE NK_LP expression NK_RP", + /* 312 */ "cmd ::= KILL CONNECTION NK_INTEGER", + /* 313 */ "cmd ::= KILL QUERY NK_STRING", + /* 314 */ "cmd ::= KILL TRANSACTION NK_INTEGER", + /* 315 */ "cmd ::= BALANCE VGROUP", + /* 316 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", + /* 317 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", + /* 318 */ "cmd ::= SPLIT VGROUP NK_INTEGER", + /* 319 */ "dnode_list ::= DNODE NK_INTEGER", + /* 320 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", + /* 321 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", + /* 322 */ "cmd ::= query_or_subquery", + /* 323 */ "cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery", + /* 324 */ "cmd ::= INSERT INTO full_table_name query_or_subquery", + /* 325 */ "literal ::= NK_INTEGER", + /* 326 */ "literal ::= NK_FLOAT", + /* 327 */ "literal ::= NK_STRING", + /* 328 */ "literal ::= NK_BOOL", + /* 329 */ "literal ::= TIMESTAMP NK_STRING", + /* 330 */ "literal ::= duration_literal", + /* 331 */ "literal ::= NULL", + /* 332 */ "literal ::= NK_QUESTION", + /* 333 */ "duration_literal ::= NK_VARIABLE", + /* 334 */ "signed ::= NK_INTEGER", + /* 335 */ "signed ::= NK_PLUS NK_INTEGER", + /* 336 */ "signed ::= NK_MINUS NK_INTEGER", + /* 337 */ "signed ::= NK_FLOAT", + /* 338 */ "signed ::= NK_PLUS NK_FLOAT", + /* 339 */ "signed ::= NK_MINUS NK_FLOAT", + /* 340 */ "signed_literal ::= signed", + /* 341 */ "signed_literal ::= NK_STRING", + /* 342 */ "signed_literal ::= NK_BOOL", + /* 343 */ "signed_literal ::= TIMESTAMP NK_STRING", + /* 344 */ "signed_literal ::= duration_literal", + /* 345 */ "signed_literal ::= NULL", + /* 346 */ "signed_literal ::= literal_func", + /* 347 */ "signed_literal ::= NK_QUESTION", + /* 348 */ "literal_list ::= signed_literal", + /* 349 */ "literal_list ::= literal_list NK_COMMA signed_literal", + /* 350 */ "db_name ::= NK_ID", + /* 351 */ "table_name ::= NK_ID", + /* 352 */ "column_name ::= NK_ID", + /* 353 */ "function_name ::= NK_ID", + /* 354 */ "table_alias ::= NK_ID", + /* 355 */ "column_alias ::= NK_ID", + /* 356 */ "user_name ::= NK_ID", + /* 357 */ "topic_name ::= NK_ID", + /* 358 */ "stream_name ::= NK_ID", + /* 359 */ "cgroup_name ::= NK_ID", + /* 360 */ "expr_or_subquery ::= expression", + /* 361 */ "expression ::= literal", + /* 362 */ "expression ::= pseudo_column", + /* 363 */ "expression ::= column_reference", + /* 364 */ "expression ::= function_expression", + /* 365 */ "expression ::= case_when_expression", + /* 366 */ "expression ::= NK_LP expression NK_RP", + /* 367 */ "expression ::= NK_PLUS expr_or_subquery", + /* 368 */ "expression ::= NK_MINUS expr_or_subquery", + /* 369 */ "expression ::= expr_or_subquery NK_PLUS expr_or_subquery", + /* 370 */ "expression ::= expr_or_subquery NK_MINUS expr_or_subquery", + /* 371 */ "expression ::= expr_or_subquery NK_STAR expr_or_subquery", + /* 372 */ "expression ::= expr_or_subquery NK_SLASH expr_or_subquery", + /* 373 */ "expression ::= expr_or_subquery NK_REM expr_or_subquery", + /* 374 */ "expression ::= column_reference NK_ARROW NK_STRING", + /* 375 */ "expression ::= expr_or_subquery NK_BITAND expr_or_subquery", + /* 376 */ "expression ::= expr_or_subquery NK_BITOR expr_or_subquery", + /* 377 */ "expression_list ::= expr_or_subquery", + /* 378 */ "expression_list ::= expression_list NK_COMMA expr_or_subquery", + /* 379 */ "column_reference ::= column_name", + /* 380 */ "column_reference ::= table_name NK_DOT column_name", + /* 381 */ "pseudo_column ::= ROWTS", + /* 382 */ "pseudo_column ::= TBNAME", + /* 383 */ "pseudo_column ::= table_name NK_DOT TBNAME", + /* 384 */ "pseudo_column ::= QSTART", + /* 385 */ "pseudo_column ::= QEND", + /* 386 */ "pseudo_column ::= QDURATION", + /* 387 */ "pseudo_column ::= WSTART", + /* 388 */ "pseudo_column ::= WEND", + /* 389 */ "pseudo_column ::= WDURATION", + /* 390 */ "pseudo_column ::= IROWTS", + /* 391 */ "pseudo_column ::= ISFILLED", + /* 392 */ "pseudo_column ::= QTAGS", + /* 393 */ "function_expression ::= function_name NK_LP expression_list NK_RP", + /* 394 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", + /* 395 */ "function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP", + /* 396 */ "function_expression ::= literal_func", + /* 397 */ "literal_func ::= noarg_func NK_LP NK_RP", + /* 398 */ "literal_func ::= NOW", + /* 399 */ "noarg_func ::= NOW", + /* 400 */ "noarg_func ::= TODAY", + /* 401 */ "noarg_func ::= TIMEZONE", + /* 402 */ "noarg_func ::= DATABASE", + /* 403 */ "noarg_func ::= CLIENT_VERSION", + /* 404 */ "noarg_func ::= SERVER_VERSION", + /* 405 */ "noarg_func ::= SERVER_STATUS", + /* 406 */ "noarg_func ::= CURRENT_USER", + /* 407 */ "noarg_func ::= USER", + /* 408 */ "star_func ::= COUNT", + /* 409 */ "star_func ::= FIRST", + /* 410 */ "star_func ::= LAST", + /* 411 */ "star_func ::= LAST_ROW", + /* 412 */ "star_func_para_list ::= NK_STAR", + /* 413 */ "star_func_para_list ::= other_para_list", + /* 414 */ "other_para_list ::= star_func_para", + /* 415 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", + /* 416 */ "star_func_para ::= expr_or_subquery", + /* 417 */ "star_func_para ::= table_name NK_DOT NK_STAR", + /* 418 */ "case_when_expression ::= CASE when_then_list case_when_else_opt END", + /* 419 */ "case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END", + /* 420 */ "when_then_list ::= when_then_expr", + /* 421 */ "when_then_list ::= when_then_list when_then_expr", + /* 422 */ "when_then_expr ::= WHEN common_expression THEN common_expression", + /* 423 */ "case_when_else_opt ::=", + /* 424 */ "case_when_else_opt ::= ELSE common_expression", + /* 425 */ "predicate ::= expr_or_subquery compare_op expr_or_subquery", + /* 426 */ "predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery", + /* 427 */ "predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery", + /* 428 */ "predicate ::= expr_or_subquery IS NULL", + /* 429 */ "predicate ::= expr_or_subquery IS NOT NULL", + /* 430 */ "predicate ::= expr_or_subquery in_op in_predicate_value", + /* 431 */ "compare_op ::= NK_LT", + /* 432 */ "compare_op ::= NK_GT", + /* 433 */ "compare_op ::= NK_LE", + /* 434 */ "compare_op ::= NK_GE", + /* 435 */ "compare_op ::= NK_NE", + /* 436 */ "compare_op ::= NK_EQ", + /* 437 */ "compare_op ::= LIKE", + /* 438 */ "compare_op ::= NOT LIKE", + /* 439 */ "compare_op ::= MATCH", + /* 440 */ "compare_op ::= NMATCH", + /* 441 */ "compare_op ::= CONTAINS", + /* 442 */ "in_op ::= IN", + /* 443 */ "in_op ::= NOT IN", + /* 444 */ "in_predicate_value ::= NK_LP literal_list NK_RP", + /* 445 */ "boolean_value_expression ::= boolean_primary", + /* 446 */ "boolean_value_expression ::= NOT boolean_primary", + /* 447 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 448 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 449 */ "boolean_primary ::= predicate", + /* 450 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 451 */ "common_expression ::= expr_or_subquery", + /* 452 */ "common_expression ::= boolean_value_expression", + /* 453 */ "from_clause_opt ::=", + /* 454 */ "from_clause_opt ::= FROM table_reference_list", + /* 455 */ "table_reference_list ::= table_reference", + /* 456 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 457 */ "table_reference ::= table_primary", + /* 458 */ "table_reference ::= joined_table", + /* 459 */ "table_primary ::= table_name alias_opt", + /* 460 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 461 */ "table_primary ::= subquery alias_opt", + /* 462 */ "table_primary ::= parenthesized_joined_table", + /* 463 */ "alias_opt ::=", + /* 464 */ "alias_opt ::= table_alias", + /* 465 */ "alias_opt ::= AS table_alias", + /* 466 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 467 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 468 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 469 */ "join_type ::=", + /* 470 */ "join_type ::= INNER", + /* 471 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 472 */ "set_quantifier_opt ::=", + /* 473 */ "set_quantifier_opt ::= DISTINCT", + /* 474 */ "set_quantifier_opt ::= ALL", + /* 475 */ "select_list ::= select_item", + /* 476 */ "select_list ::= select_list NK_COMMA select_item", + /* 477 */ "select_item ::= NK_STAR", + /* 478 */ "select_item ::= common_expression", + /* 479 */ "select_item ::= common_expression column_alias", + /* 480 */ "select_item ::= common_expression AS column_alias", + /* 481 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 482 */ "where_clause_opt ::=", + /* 483 */ "where_clause_opt ::= WHERE search_condition", + /* 484 */ "partition_by_clause_opt ::=", + /* 485 */ "partition_by_clause_opt ::= PARTITION BY partition_list", + /* 486 */ "partition_list ::= partition_item", + /* 487 */ "partition_list ::= partition_list NK_COMMA partition_item", + /* 488 */ "partition_item ::= expr_or_subquery", + /* 489 */ "partition_item ::= expr_or_subquery column_alias", + /* 490 */ "partition_item ::= expr_or_subquery AS column_alias", + /* 491 */ "twindow_clause_opt ::=", + /* 492 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", + /* 493 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP", + /* 494 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 495 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 496 */ "twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition", + /* 497 */ "sliding_opt ::=", + /* 498 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 499 */ "fill_opt ::=", + /* 500 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 501 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 502 */ "fill_mode ::= NONE", + /* 503 */ "fill_mode ::= PREV", + /* 504 */ "fill_mode ::= NULL", + /* 505 */ "fill_mode ::= LINEAR", + /* 506 */ "fill_mode ::= NEXT", + /* 507 */ "group_by_clause_opt ::=", + /* 508 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 509 */ "group_by_list ::= expr_or_subquery", + /* 510 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", + /* 511 */ "having_clause_opt ::=", + /* 512 */ "having_clause_opt ::= HAVING search_condition", + /* 513 */ "range_opt ::=", + /* 514 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", + /* 515 */ "every_opt ::=", + /* 516 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", + /* 517 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 518 */ "query_simple ::= query_specification", + /* 519 */ "query_simple ::= union_query_expression", + /* 520 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", + /* 521 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", + /* 522 */ "query_simple_or_subquery ::= query_simple", + /* 523 */ "query_simple_or_subquery ::= subquery", + /* 524 */ "query_or_subquery ::= query_expression", + /* 525 */ "query_or_subquery ::= subquery", + /* 526 */ "order_by_clause_opt ::=", + /* 527 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 528 */ "slimit_clause_opt ::=", + /* 529 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 530 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 531 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 532 */ "limit_clause_opt ::=", + /* 533 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 534 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 535 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 536 */ "subquery ::= NK_LP query_expression NK_RP", + /* 537 */ "subquery ::= NK_LP subquery NK_RP", + /* 538 */ "search_condition ::= common_expression", + /* 539 */ "sort_specification_list ::= sort_specification", + /* 540 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 541 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", + /* 542 */ "ordering_specification_opt ::=", + /* 543 */ "ordering_specification_opt ::= ASC", + /* 544 */ "ordering_specification_opt ::= DESC", + /* 545 */ "null_ordering_opt ::=", + /* 546 */ "null_ordering_opt ::= NULLS FIRST", + /* 547 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -2652,49 +2600,49 @@ static void yy_destructor( case 389: /* query_or_subquery */ case 392: /* explain_options */ case 396: /* stream_options */ - case 397: /* subtable_opt */ - case 398: /* expression */ - case 400: /* where_clause_opt */ - case 401: /* signed */ - case 402: /* literal_func */ - case 405: /* expr_or_subquery */ - case 406: /* pseudo_column */ - case 407: /* column_reference */ - case 408: /* function_expression */ - case 409: /* case_when_expression */ - case 414: /* star_func_para */ - case 416: /* case_when_else_opt */ - case 417: /* common_expression */ - case 418: /* when_then_expr */ - case 419: /* predicate */ - case 422: /* in_predicate_value */ - case 423: /* boolean_value_expression */ - case 424: /* boolean_primary */ - case 425: /* from_clause_opt */ - case 426: /* table_reference_list */ - case 427: /* table_reference */ - case 428: /* table_primary */ - case 429: /* joined_table */ - case 431: /* subquery */ - case 432: /* parenthesized_joined_table */ - case 434: /* search_condition */ - case 435: /* query_specification */ - case 439: /* range_opt */ - case 440: /* every_opt */ - case 441: /* fill_opt */ - case 442: /* twindow_clause_opt */ - case 444: /* having_clause_opt */ - case 445: /* select_item */ - case 447: /* partition_item */ - case 450: /* query_expression */ - case 451: /* query_simple */ - case 453: /* slimit_clause_opt */ - case 454: /* limit_clause_opt */ - case 455: /* union_query_expression */ - case 456: /* query_simple_or_subquery */ - case 458: /* sort_specification */ + case 398: /* subtable_opt */ + case 399: /* expression */ + case 401: /* where_clause_opt */ + case 402: /* signed */ + case 403: /* literal_func */ + case 406: /* expr_or_subquery */ + case 407: /* pseudo_column */ + case 408: /* column_reference */ + case 409: /* function_expression */ + case 410: /* case_when_expression */ + case 415: /* star_func_para */ + case 417: /* case_when_else_opt */ + case 418: /* common_expression */ + case 419: /* when_then_expr */ + case 420: /* predicate */ + case 423: /* in_predicate_value */ + case 424: /* boolean_value_expression */ + case 425: /* boolean_primary */ + case 426: /* from_clause_opt */ + case 427: /* table_reference_list */ + case 428: /* table_reference */ + case 429: /* table_primary */ + case 430: /* joined_table */ + case 432: /* subquery */ + case 433: /* parenthesized_joined_table */ + case 435: /* search_condition */ + case 436: /* query_specification */ + case 440: /* range_opt */ + case 441: /* every_opt */ + case 442: /* fill_opt */ + case 443: /* twindow_clause_opt */ + case 445: /* having_clause_opt */ + case 446: /* select_item */ + case 448: /* partition_item */ + case 451: /* query_expression */ + case 452: /* query_simple */ + case 454: /* slimit_clause_opt */ + case 455: /* limit_clause_opt */ + case 456: /* union_query_expression */ + case 457: /* query_simple_or_subquery */ + case 459: /* sort_specification */ { - nodesDestroyNode((yypminor->yy600)); + nodesDestroyNode((yypminor->yy476)); } break; case 326: /* account_options */ @@ -2718,10 +2666,10 @@ static void yy_destructor( case 388: /* sma_func_name */ case 390: /* cgroup_name */ case 395: /* stream_name */ - case 404: /* table_alias */ - case 410: /* star_func */ - case 412: /* noarg_func */ - case 430: /* alias_opt */ + case 405: /* table_alias */ + case 411: /* star_func */ + case 413: /* noarg_func */ + case 431: /* alias_opt */ { } @@ -2743,7 +2691,7 @@ static void yy_destructor( case 342: /* exists_opt */ case 391: /* analyze_opt */ case 393: /* agg_func_opt */ - case 436: /* set_quantifier_opt */ + case 437: /* set_quantifier_opt */ { } @@ -2763,20 +2711,21 @@ static void yy_destructor( case 370: /* rollup_func_list */ case 380: /* tag_list_opt */ case 384: /* func_list */ - case 399: /* dnode_list */ - case 403: /* literal_list */ - case 411: /* star_func_para_list */ - case 413: /* other_para_list */ - case 415: /* when_then_list */ - case 437: /* select_list */ - case 438: /* partition_by_clause_opt */ - case 443: /* group_by_clause_opt */ - case 446: /* partition_list */ - case 449: /* group_by_list */ - case 452: /* order_by_clause_opt */ - case 457: /* sort_specification_list */ + case 397: /* col_list_opt */ + case 400: /* dnode_list */ + case 404: /* literal_list */ + case 412: /* star_func_para_list */ + case 414: /* other_para_list */ + case 416: /* when_then_list */ + case 438: /* select_list */ + case 439: /* partition_by_clause_opt */ + case 444: /* group_by_clause_opt */ + case 447: /* partition_list */ + case 450: /* group_by_list */ + case 453: /* order_by_clause_opt */ + case 458: /* sort_specification_list */ { - nodesDestroyList((yypminor->yy601)); + nodesDestroyList((yypminor->yy376)); } break; case 348: /* alter_db_option */ @@ -2790,28 +2739,28 @@ static void yy_destructor( } break; - case 420: /* compare_op */ - case 421: /* in_op */ + case 421: /* compare_op */ + case 422: /* in_op */ { } break; - case 433: /* join_type */ + case 434: /* join_type */ { } break; - case 448: /* fill_mode */ + case 449: /* fill_mode */ { } break; - case 459: /* ordering_specification_opt */ + case 460: /* ordering_specification_opt */ { } break; - case 460: /* null_ordering_opt */ + case 461: /* null_ordering_opt */ { } @@ -2939,18 +2888,15 @@ static YYACTIONTYPE yy_find_shift_action( do{ i = yy_shift_ofst[stateno]; assert( i>=0 ); - assert( i<=YY_ACTTAB_COUNT ); - assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); + /* assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); */ assert( iLookAhead!=YYNOCODE ); assert( iLookAhead < YYNTOKEN ); i += iLookAhead; - assert( i<(int)YY_NLOOKAHEAD ); - if( yy_lookahead[i]!=iLookAhead ){ + if( i>=YY_NLOOKAHEAD || yy_lookahead[i]!=iLookAhead ){ #ifdef YYFALLBACK YYCODETYPE iFallback; /* Fallback token */ - assert( iLookAhead %s\n", @@ -2965,8 +2911,16 @@ static YYACTIONTYPE yy_find_shift_action( #ifdef YYWILDCARD { int j = i - iLookAhead + YYWILDCARD; - assert( j<(int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])) ); - if( yy_lookahead[j]==YYWILDCARD && iLookAhead>0 ){ + if( +#if YY_SHIFT_MIN+YYWILDCARD<0 + j>=0 && +#endif +#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT + j0 + ){ #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", @@ -2980,7 +2934,6 @@ static YYACTIONTYPE yy_find_shift_action( #endif /* YYWILDCARD */ return yy_default[stateno]; }else{ - assert( i>=0 && iyytos; #ifndef NDEBUG if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - yysize = yyRuleInfoNRhs[yyruleno]; + yysize = yyRuleInfo[yyruleno].nrhs; if( yysize ){ - fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", + fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n", yyTracePrompt, - yyruleno, yyRuleName[yyruleno], - yyrulenoyytos - yypParser->yystack)>yypParser->yyhwm ){ yypParser->yyhwm++; @@ -4336,78 +3743,78 @@ static YYACTIONTYPE yy_reduce( yy_destructor(yypParser,328,&yymsp[0].minor); break; case 24: /* cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ -{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-3].minor.yy77, &yymsp[-1].minor.yy0, yymsp[0].minor.yy287); } +{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-3].minor.yy701, &yymsp[-1].minor.yy0, yymsp[0].minor.yy47); } break; case 25: /* cmd ::= ALTER USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy77, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy701, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } break; case 26: /* cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy77, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy701, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } break; case 27: /* cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy77, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy701, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } break; case 28: /* cmd ::= DROP USER user_name */ -{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy77); } +{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy701); } break; case 29: /* sysinfo_opt ::= */ -{ yymsp[1].minor.yy287 = 1; } +{ yymsp[1].minor.yy47 = 1; } break; case 30: /* sysinfo_opt ::= SYSINFO NK_INTEGER */ -{ yymsp[-1].minor.yy287 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } +{ yymsp[-1].minor.yy47 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } break; case 31: /* cmd ::= GRANT privileges ON priv_level TO user_name */ -{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-4].minor.yy717, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy77); } +{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-4].minor.yy921, &yymsp[-2].minor.yy701, &yymsp[0].minor.yy701); } break; case 32: /* cmd ::= REVOKE privileges ON priv_level FROM user_name */ -{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-4].minor.yy717, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy77); } +{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-4].minor.yy921, &yymsp[-2].minor.yy701, &yymsp[0].minor.yy701); } break; case 33: /* privileges ::= ALL */ -{ yymsp[0].minor.yy717 = PRIVILEGE_TYPE_ALL; } +{ yymsp[0].minor.yy921 = PRIVILEGE_TYPE_ALL; } break; case 34: /* privileges ::= priv_type_list */ case 36: /* priv_type_list ::= priv_type */ yytestcase(yyruleno==36); -{ yylhsminor.yy717 = yymsp[0].minor.yy717; } - yymsp[0].minor.yy717 = yylhsminor.yy717; +{ yylhsminor.yy921 = yymsp[0].minor.yy921; } + yymsp[0].minor.yy921 = yylhsminor.yy921; break; case 35: /* privileges ::= SUBSCRIBE */ -{ yymsp[0].minor.yy717 = PRIVILEGE_TYPE_SUBSCRIBE; } +{ yymsp[0].minor.yy921 = PRIVILEGE_TYPE_SUBSCRIBE; } break; case 37: /* priv_type_list ::= priv_type_list NK_COMMA priv_type */ -{ yylhsminor.yy717 = yymsp[-2].minor.yy717 | yymsp[0].minor.yy717; } - yymsp[-2].minor.yy717 = yylhsminor.yy717; +{ yylhsminor.yy921 = yymsp[-2].minor.yy921 | yymsp[0].minor.yy921; } + yymsp[-2].minor.yy921 = yylhsminor.yy921; break; case 38: /* priv_type ::= READ */ -{ yymsp[0].minor.yy717 = PRIVILEGE_TYPE_READ; } +{ yymsp[0].minor.yy921 = PRIVILEGE_TYPE_READ; } break; case 39: /* priv_type ::= WRITE */ -{ yymsp[0].minor.yy717 = PRIVILEGE_TYPE_WRITE; } +{ yymsp[0].minor.yy921 = PRIVILEGE_TYPE_WRITE; } break; case 40: /* priv_level ::= NK_STAR NK_DOT NK_STAR */ -{ yylhsminor.yy77 = yymsp[-2].minor.yy0; } - yymsp[-2].minor.yy77 = yylhsminor.yy77; +{ yylhsminor.yy701 = yymsp[-2].minor.yy0; } + yymsp[-2].minor.yy701 = yylhsminor.yy701; break; case 41: /* priv_level ::= db_name NK_DOT NK_STAR */ -{ yylhsminor.yy77 = yymsp[-2].minor.yy77; } - yymsp[-2].minor.yy77 = yylhsminor.yy77; +{ yylhsminor.yy701 = yymsp[-2].minor.yy701; } + yymsp[-2].minor.yy701 = yylhsminor.yy701; break; case 42: /* priv_level ::= topic_name */ - case 267: /* sma_func_name ::= function_name */ yytestcase(yyruleno==267); - case 461: /* alias_opt ::= table_alias */ yytestcase(yyruleno==461); -{ yylhsminor.yy77 = yymsp[0].minor.yy77; } - yymsp[0].minor.yy77 = yylhsminor.yy77; + case 268: /* sma_func_name ::= function_name */ yytestcase(yyruleno==268); + case 464: /* alias_opt ::= table_alias */ yytestcase(yyruleno==464); +{ yylhsminor.yy701 = yymsp[0].minor.yy701; } + yymsp[0].minor.yy701 = yylhsminor.yy701; break; case 43: /* cmd ::= CREATE DNODE dnode_endpoint */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy77, NULL); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy701, NULL); } break; case 44: /* cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy701, &yymsp[0].minor.yy0); } break; case 45: /* cmd ::= DROP DNODE NK_INTEGER force_opt */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy841); } +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy845); } break; case 46: /* cmd ::= DROP DNODE dnode_endpoint force_opt */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy77, yymsp[0].minor.yy841); } +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy701, yymsp[0].minor.yy845); } break; case 47: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, NULL); } @@ -4424,49 +3831,49 @@ static YYACTIONTYPE yy_reduce( case 51: /* dnode_endpoint ::= NK_STRING */ case 52: /* dnode_endpoint ::= NK_ID */ yytestcase(yyruleno==52); case 53: /* dnode_endpoint ::= NK_IPTOKEN */ yytestcase(yyruleno==53); - case 268: /* sma_func_name ::= COUNT */ yytestcase(yyruleno==268); - case 269: /* sma_func_name ::= FIRST */ yytestcase(yyruleno==269); - case 270: /* sma_func_name ::= LAST */ yytestcase(yyruleno==270); - case 271: /* sma_func_name ::= LAST_ROW */ yytestcase(yyruleno==271); - case 347: /* db_name ::= NK_ID */ yytestcase(yyruleno==347); - case 348: /* table_name ::= NK_ID */ yytestcase(yyruleno==348); - case 349: /* column_name ::= NK_ID */ yytestcase(yyruleno==349); - case 350: /* function_name ::= NK_ID */ yytestcase(yyruleno==350); - case 351: /* table_alias ::= NK_ID */ yytestcase(yyruleno==351); - case 352: /* column_alias ::= NK_ID */ yytestcase(yyruleno==352); - case 353: /* user_name ::= NK_ID */ yytestcase(yyruleno==353); - case 354: /* topic_name ::= NK_ID */ yytestcase(yyruleno==354); - case 355: /* stream_name ::= NK_ID */ yytestcase(yyruleno==355); - case 356: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==356); - case 396: /* noarg_func ::= NOW */ yytestcase(yyruleno==396); - case 397: /* noarg_func ::= TODAY */ yytestcase(yyruleno==397); - case 398: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==398); - case 399: /* noarg_func ::= DATABASE */ yytestcase(yyruleno==399); - case 400: /* noarg_func ::= CLIENT_VERSION */ yytestcase(yyruleno==400); - case 401: /* noarg_func ::= SERVER_VERSION */ yytestcase(yyruleno==401); - case 402: /* noarg_func ::= SERVER_STATUS */ yytestcase(yyruleno==402); - case 403: /* noarg_func ::= CURRENT_USER */ yytestcase(yyruleno==403); - case 404: /* noarg_func ::= USER */ yytestcase(yyruleno==404); - case 405: /* star_func ::= COUNT */ yytestcase(yyruleno==405); - case 406: /* star_func ::= FIRST */ yytestcase(yyruleno==406); - case 407: /* star_func ::= LAST */ yytestcase(yyruleno==407); - case 408: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==408); -{ yylhsminor.yy77 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy77 = yylhsminor.yy77; + case 269: /* sma_func_name ::= COUNT */ yytestcase(yyruleno==269); + case 270: /* sma_func_name ::= FIRST */ yytestcase(yyruleno==270); + case 271: /* sma_func_name ::= LAST */ yytestcase(yyruleno==271); + case 272: /* sma_func_name ::= LAST_ROW */ yytestcase(yyruleno==272); + case 350: /* db_name ::= NK_ID */ yytestcase(yyruleno==350); + case 351: /* table_name ::= NK_ID */ yytestcase(yyruleno==351); + case 352: /* column_name ::= NK_ID */ yytestcase(yyruleno==352); + case 353: /* function_name ::= NK_ID */ yytestcase(yyruleno==353); + case 354: /* table_alias ::= NK_ID */ yytestcase(yyruleno==354); + case 355: /* column_alias ::= NK_ID */ yytestcase(yyruleno==355); + case 356: /* user_name ::= NK_ID */ yytestcase(yyruleno==356); + case 357: /* topic_name ::= NK_ID */ yytestcase(yyruleno==357); + case 358: /* stream_name ::= NK_ID */ yytestcase(yyruleno==358); + case 359: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==359); + case 399: /* noarg_func ::= NOW */ yytestcase(yyruleno==399); + case 400: /* noarg_func ::= TODAY */ yytestcase(yyruleno==400); + case 401: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==401); + case 402: /* noarg_func ::= DATABASE */ yytestcase(yyruleno==402); + case 403: /* noarg_func ::= CLIENT_VERSION */ yytestcase(yyruleno==403); + case 404: /* noarg_func ::= SERVER_VERSION */ yytestcase(yyruleno==404); + case 405: /* noarg_func ::= SERVER_STATUS */ yytestcase(yyruleno==405); + case 406: /* noarg_func ::= CURRENT_USER */ yytestcase(yyruleno==406); + case 407: /* noarg_func ::= USER */ yytestcase(yyruleno==407); + case 408: /* star_func ::= COUNT */ yytestcase(yyruleno==408); + case 409: /* star_func ::= FIRST */ yytestcase(yyruleno==409); + case 410: /* star_func ::= LAST */ yytestcase(yyruleno==410); + case 411: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==411); +{ yylhsminor.yy701 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy701 = yylhsminor.yy701; break; case 54: /* force_opt ::= */ case 73: /* not_exists_opt ::= */ yytestcase(yyruleno==73); case 75: /* exists_opt ::= */ yytestcase(yyruleno==75); - case 287: /* analyze_opt ::= */ yytestcase(yyruleno==287); - case 294: /* agg_func_opt ::= */ yytestcase(yyruleno==294); - case 469: /* set_quantifier_opt ::= */ yytestcase(yyruleno==469); -{ yymsp[1].minor.yy841 = false; } + case 288: /* analyze_opt ::= */ yytestcase(yyruleno==288); + case 295: /* agg_func_opt ::= */ yytestcase(yyruleno==295); + case 472: /* set_quantifier_opt ::= */ yytestcase(yyruleno==472); +{ yymsp[1].minor.yy845 = false; } break; case 55: /* force_opt ::= FORCE */ - case 288: /* analyze_opt ::= ANALYZE */ yytestcase(yyruleno==288); - case 295: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==295); - case 470: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==470); -{ yymsp[0].minor.yy841 = true; } + case 289: /* analyze_opt ::= ANALYZE */ yytestcase(yyruleno==289); + case 296: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==296); + case 473: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==473); +{ yymsp[0].minor.yy845 = true; } break; case 56: /* cmd ::= ALTER LOCAL NK_STRING */ { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[0].minor.yy0, NULL); } @@ -4499,206 +3906,206 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_MNODE_STMT, &yymsp[0].minor.yy0); } break; case 66: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ -{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy841, &yymsp[-1].minor.yy77, yymsp[0].minor.yy600); } +{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy845, &yymsp[-1].minor.yy701, yymsp[0].minor.yy476); } break; case 67: /* cmd ::= DROP DATABASE exists_opt db_name */ -{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy841, &yymsp[0].minor.yy77); } +{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy845, &yymsp[0].minor.yy701); } break; case 68: /* cmd ::= USE db_name */ -{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy77); } +{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy701); } break; case 69: /* cmd ::= ALTER DATABASE db_name alter_db_options */ -{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy77, yymsp[0].minor.yy600); } +{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy701, yymsp[0].minor.yy476); } break; case 70: /* cmd ::= FLUSH DATABASE db_name */ -{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy77); } +{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy701); } break; case 71: /* cmd ::= TRIM DATABASE db_name speed_opt */ -{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy77, yymsp[0].minor.yy248); } +{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy701, yymsp[0].minor.yy508); } break; case 72: /* not_exists_opt ::= IF NOT EXISTS */ -{ yymsp[-2].minor.yy841 = true; } +{ yymsp[-2].minor.yy845 = true; } break; case 74: /* exists_opt ::= IF EXISTS */ -{ yymsp[-1].minor.yy841 = true; } +{ yymsp[-1].minor.yy845 = true; } break; case 76: /* db_options ::= */ -{ yymsp[1].minor.yy600 = createDefaultDatabaseOptions(pCxt); } +{ yymsp[1].minor.yy476 = createDefaultDatabaseOptions(pCxt); } break; case 77: /* db_options ::= db_options BUFFER NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 78: /* db_options ::= db_options CACHEMODEL NK_STRING */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 79: /* db_options ::= db_options CACHESIZE NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 80: /* db_options ::= db_options COMP NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_COMP, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_COMP, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 81: /* db_options ::= db_options DURATION NK_INTEGER */ case 82: /* db_options ::= db_options DURATION NK_VARIABLE */ yytestcase(yyruleno==82); -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 83: /* db_options ::= db_options MAXROWS NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 84: /* db_options ::= db_options MINROWS NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 85: /* db_options ::= db_options KEEP integer_list */ case 86: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==86); -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_KEEP, yymsp[0].minor.yy601); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_KEEP, yymsp[0].minor.yy376); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 87: /* db_options ::= db_options PAGES NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 88: /* db_options ::= db_options PAGESIZE NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 89: /* db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 90: /* db_options ::= db_options PRECISION NK_STRING */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 91: /* db_options ::= db_options REPLICA NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 92: /* db_options ::= db_options VGROUPS NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 93: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 94: /* db_options ::= db_options RETENTIONS retention_list */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_RETENTIONS, yymsp[0].minor.yy601); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_RETENTIONS, yymsp[0].minor.yy376); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 95: /* db_options ::= db_options SCHEMALESS NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 96: /* db_options ::= db_options WAL_LEVEL NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_WAL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 97: /* db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 98: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 99: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-3].minor.yy600, DB_OPTION_WAL_RETENTION_PERIOD, &t); + yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-3].minor.yy476, DB_OPTION_WAL_RETENTION_PERIOD, &t); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + yymsp[-3].minor.yy476 = yylhsminor.yy476; break; case 100: /* db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 101: /* db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-3].minor.yy600, DB_OPTION_WAL_RETENTION_SIZE, &t); + yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-3].minor.yy476, DB_OPTION_WAL_RETENTION_SIZE, &t); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + yymsp[-3].minor.yy476 = yylhsminor.yy476; break; case 102: /* db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 103: /* db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 104: /* db_options ::= db_options STT_TRIGGER NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 105: /* db_options ::= db_options TABLE_PREFIX NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_TABLE_PREFIX, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_TABLE_PREFIX, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 106: /* db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_TABLE_SUFFIX, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setDatabaseOption(pCxt, yymsp[-2].minor.yy476, DB_OPTION_TABLE_SUFFIX, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 107: /* alter_db_options ::= alter_db_option */ -{ yylhsminor.yy600 = createAlterDatabaseOptions(pCxt); yylhsminor.yy600 = setAlterDatabaseOption(pCxt, yylhsminor.yy600, &yymsp[0].minor.yy661); } - yymsp[0].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createAlterDatabaseOptions(pCxt); yylhsminor.yy476 = setAlterDatabaseOption(pCxt, yylhsminor.yy476, &yymsp[0].minor.yy893); } + yymsp[0].minor.yy476 = yylhsminor.yy476; break; case 108: /* alter_db_options ::= alter_db_options alter_db_option */ -{ yylhsminor.yy600 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy600, &yymsp[0].minor.yy661); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy476, &yymsp[0].minor.yy893); } + yymsp[-1].minor.yy476 = yylhsminor.yy476; break; case 109: /* alter_db_option ::= BUFFER NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy893.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy893.val = yymsp[0].minor.yy0; } break; case 110: /* alter_db_option ::= CACHEMODEL NK_STRING */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy893.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy893.val = yymsp[0].minor.yy0; } break; case 111: /* alter_db_option ::= CACHESIZE NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy893.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy893.val = yymsp[0].minor.yy0; } break; case 112: /* alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy893.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy893.val = yymsp[0].minor.yy0; } break; case 113: /* alter_db_option ::= KEEP integer_list */ case 114: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==114); -{ yymsp[-1].minor.yy661.type = DB_OPTION_KEEP; yymsp[-1].minor.yy661.pList = yymsp[0].minor.yy601; } +{ yymsp[-1].minor.yy893.type = DB_OPTION_KEEP; yymsp[-1].minor.yy893.pList = yymsp[0].minor.yy376; } break; case 115: /* alter_db_option ::= PAGES NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_PAGES; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy893.type = DB_OPTION_PAGES; yymsp[-1].minor.yy893.val = yymsp[0].minor.yy0; } break; case 116: /* alter_db_option ::= REPLICA NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy893.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy893.val = yymsp[0].minor.yy0; } break; case 117: /* alter_db_option ::= WAL_LEVEL NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_WAL; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy893.type = DB_OPTION_WAL; yymsp[-1].minor.yy893.val = yymsp[0].minor.yy0; } break; case 118: /* alter_db_option ::= STT_TRIGGER NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy893.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy893.val = yymsp[0].minor.yy0; } break; case 119: /* integer_list ::= NK_INTEGER */ -{ yylhsminor.yy601 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy601 = yylhsminor.yy601; +{ yylhsminor.yy376 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy376 = yylhsminor.yy376; break; case 120: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ - case 317: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==317); -{ yylhsminor.yy601 = addNodeToList(pCxt, yymsp[-2].minor.yy601, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy601 = yylhsminor.yy601; + case 320: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==320); +{ yylhsminor.yy376 = addNodeToList(pCxt, yymsp[-2].minor.yy376, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy376 = yylhsminor.yy376; break; case 121: /* variable_list ::= NK_VARIABLE */ -{ yylhsminor.yy601 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy601 = yylhsminor.yy601; +{ yylhsminor.yy376 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy376 = yylhsminor.yy376; break; case 122: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ -{ yylhsminor.yy601 = addNodeToList(pCxt, yymsp[-2].minor.yy601, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy601 = yylhsminor.yy601; +{ yylhsminor.yy376 = addNodeToList(pCxt, yymsp[-2].minor.yy376, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy376 = yylhsminor.yy376; break; case 123: /* retention_list ::= retention */ case 145: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==145); @@ -4707,287 +4114,289 @@ static YYACTIONTYPE yy_reduce( case 199: /* rollup_func_list ::= rollup_func_name */ yytestcase(yyruleno==199); case 204: /* col_name_list ::= col_name */ yytestcase(yyruleno==204); case 253: /* tag_list_opt ::= tag_item */ yytestcase(yyruleno==253); - case 264: /* func_list ::= func */ yytestcase(yyruleno==264); - case 345: /* literal_list ::= signed_literal */ yytestcase(yyruleno==345); - case 411: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==411); - case 417: /* when_then_list ::= when_then_expr */ yytestcase(yyruleno==417); - case 472: /* select_list ::= select_item */ yytestcase(yyruleno==472); - case 483: /* partition_list ::= partition_item */ yytestcase(yyruleno==483); - case 536: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==536); -{ yylhsminor.yy601 = createNodeList(pCxt, yymsp[0].minor.yy600); } - yymsp[0].minor.yy601 = yylhsminor.yy601; + case 265: /* func_list ::= func */ yytestcase(yyruleno==265); + case 348: /* literal_list ::= signed_literal */ yytestcase(yyruleno==348); + case 414: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==414); + case 420: /* when_then_list ::= when_then_expr */ yytestcase(yyruleno==420); + case 475: /* select_list ::= select_item */ yytestcase(yyruleno==475); + case 486: /* partition_list ::= partition_item */ yytestcase(yyruleno==486); + case 539: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==539); +{ yylhsminor.yy376 = createNodeList(pCxt, yymsp[0].minor.yy476); } + yymsp[0].minor.yy376 = yylhsminor.yy376; break; case 124: /* retention_list ::= retention_list NK_COMMA retention */ case 156: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==156); case 200: /* rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ yytestcase(yyruleno==200); case 205: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==205); case 254: /* tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ yytestcase(yyruleno==254); - case 265: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==265); - case 346: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==346); - case 412: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==412); - case 473: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==473); - case 484: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==484); - case 537: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==537); -{ yylhsminor.yy601 = addNodeToList(pCxt, yymsp[-2].minor.yy601, yymsp[0].minor.yy600); } - yymsp[-2].minor.yy601 = yylhsminor.yy601; + case 266: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==266); + case 349: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==349); + case 415: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==415); + case 476: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==476); + case 487: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==487); + case 540: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==540); +{ yylhsminor.yy376 = addNodeToList(pCxt, yymsp[-2].minor.yy376, yymsp[0].minor.yy476); } + yymsp[-2].minor.yy376 = yylhsminor.yy376; break; case 125: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ -{ yylhsminor.yy600 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 126: /* speed_opt ::= */ - case 296: /* bufsize_opt ::= */ yytestcase(yyruleno==296); -{ yymsp[1].minor.yy248 = 0; } + case 297: /* bufsize_opt ::= */ yytestcase(yyruleno==297); +{ yymsp[1].minor.yy508 = 0; } break; case 127: /* speed_opt ::= MAX_SPEED NK_INTEGER */ - case 297: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==297); -{ yymsp[-1].minor.yy248 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } + case 298: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==298); +{ yymsp[-1].minor.yy508 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } break; case 128: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ case 130: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==130); -{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy841, yymsp[-5].minor.yy600, yymsp[-3].minor.yy601, yymsp[-1].minor.yy601, yymsp[0].minor.yy600); } +{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy845, yymsp[-5].minor.yy476, yymsp[-3].minor.yy376, yymsp[-1].minor.yy376, yymsp[0].minor.yy476); } break; case 129: /* cmd ::= CREATE TABLE multi_create_clause */ -{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy601); } +{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy376); } break; case 131: /* cmd ::= DROP TABLE multi_drop_clause */ -{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy601); } +{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy376); } break; case 132: /* cmd ::= DROP STABLE exists_opt full_table_name */ -{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy841, yymsp[0].minor.yy600); } +{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy845, yymsp[0].minor.yy476); } break; case 133: /* cmd ::= ALTER TABLE alter_table_clause */ - case 319: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==319); -{ pCxt->pRootNode = yymsp[0].minor.yy600; } + case 322: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==322); +{ pCxt->pRootNode = yymsp[0].minor.yy476; } break; case 134: /* cmd ::= ALTER STABLE alter_table_clause */ -{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy600); } +{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy476); } break; case 135: /* alter_table_clause ::= full_table_name alter_table_options */ -{ yylhsminor.yy600 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy476, yymsp[0].minor.yy476); } + yymsp[-1].minor.yy476 = yylhsminor.yy476; break; case 136: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ -{ yylhsminor.yy600 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy77, yymsp[0].minor.yy888); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy476, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy701, yymsp[0].minor.yy532); } + yymsp[-4].minor.yy476 = yylhsminor.yy476; break; case 137: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ -{ yylhsminor.yy600 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy600, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy77); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy476, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy701); } + yymsp[-3].minor.yy476 = yylhsminor.yy476; break; case 138: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ -{ yylhsminor.yy600 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy77, yymsp[0].minor.yy888); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy476, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy701, yymsp[0].minor.yy532); } + yymsp[-4].minor.yy476 = yylhsminor.yy476; break; case 139: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ -{ yylhsminor.yy600 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy77, &yymsp[0].minor.yy77); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy476, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy701, &yymsp[0].minor.yy701); } + yymsp[-4].minor.yy476 = yylhsminor.yy476; break; case 140: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ -{ yylhsminor.yy600 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy77, yymsp[0].minor.yy888); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy476, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy701, yymsp[0].minor.yy532); } + yymsp[-4].minor.yy476 = yylhsminor.yy476; break; case 141: /* alter_table_clause ::= full_table_name DROP TAG column_name */ -{ yylhsminor.yy600 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy600, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy77); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy476, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy701); } + yymsp[-3].minor.yy476 = yylhsminor.yy476; break; case 142: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ -{ yylhsminor.yy600 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy77, yymsp[0].minor.yy888); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy476, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy701, yymsp[0].minor.yy532); } + yymsp[-4].minor.yy476 = yylhsminor.yy476; break; case 143: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ -{ yylhsminor.yy600 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy77, &yymsp[0].minor.yy77); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy476, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy701, &yymsp[0].minor.yy701); } + yymsp[-4].minor.yy476 = yylhsminor.yy476; break; case 144: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ -{ yylhsminor.yy600 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy600, &yymsp[-2].minor.yy77, yymsp[0].minor.yy600); } - yymsp[-5].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy476, &yymsp[-2].minor.yy701, yymsp[0].minor.yy476); } + yymsp[-5].minor.yy476 = yylhsminor.yy476; break; case 146: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ case 149: /* multi_drop_clause ::= multi_drop_clause drop_table_clause */ yytestcase(yyruleno==149); - case 418: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==418); -{ yylhsminor.yy601 = addNodeToList(pCxt, yymsp[-1].minor.yy601, yymsp[0].minor.yy600); } - yymsp[-1].minor.yy601 = yylhsminor.yy601; + case 421: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==421); +{ yylhsminor.yy376 = addNodeToList(pCxt, yymsp[-1].minor.yy376, yymsp[0].minor.yy476); } + yymsp[-1].minor.yy376 = yylhsminor.yy376; break; case 147: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ -{ yylhsminor.yy600 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy841, yymsp[-8].minor.yy600, yymsp[-6].minor.yy600, yymsp[-5].minor.yy601, yymsp[-2].minor.yy601, yymsp[0].minor.yy600); } - yymsp[-9].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy845, yymsp[-8].minor.yy476, yymsp[-6].minor.yy476, yymsp[-5].minor.yy376, yymsp[-2].minor.yy376, yymsp[0].minor.yy476); } + yymsp[-9].minor.yy476 = yylhsminor.yy476; break; case 150: /* drop_table_clause ::= exists_opt full_table_name */ -{ yylhsminor.yy600 = createDropTableClause(pCxt, yymsp[-1].minor.yy841, yymsp[0].minor.yy600); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createDropTableClause(pCxt, yymsp[-1].minor.yy845, yymsp[0].minor.yy476); } + yymsp[-1].minor.yy476 = yylhsminor.yy476; break; case 151: /* specific_cols_opt ::= */ case 182: /* tags_def_opt ::= */ yytestcase(yyruleno==182); case 252: /* tag_list_opt ::= */ yytestcase(yyruleno==252); - case 481: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==481); - case 504: /* group_by_clause_opt ::= */ yytestcase(yyruleno==504); - case 523: /* order_by_clause_opt ::= */ yytestcase(yyruleno==523); -{ yymsp[1].minor.yy601 = NULL; } + case 301: /* col_list_opt ::= */ yytestcase(yyruleno==301); + case 484: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==484); + case 507: /* group_by_clause_opt ::= */ yytestcase(yyruleno==507); + case 526: /* order_by_clause_opt ::= */ yytestcase(yyruleno==526); +{ yymsp[1].minor.yy376 = NULL; } break; case 152: /* specific_cols_opt ::= NK_LP col_name_list NK_RP */ -{ yymsp[-2].minor.yy601 = yymsp[-1].minor.yy601; } + case 302: /* col_list_opt ::= NK_LP col_name_list NK_RP */ yytestcase(yyruleno==302); +{ yymsp[-2].minor.yy376 = yymsp[-1].minor.yy376; } break; case 153: /* full_table_name ::= table_name */ -{ yylhsminor.yy600 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy77, NULL); } - yymsp[0].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy701, NULL); } + yymsp[0].minor.yy476 = yylhsminor.yy476; break; case 154: /* full_table_name ::= db_name NK_DOT table_name */ -{ yylhsminor.yy600 = createRealTableNode(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy77, NULL); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createRealTableNode(pCxt, &yymsp[-2].minor.yy701, &yymsp[0].minor.yy701, NULL); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 157: /* column_def ::= column_name type_name */ -{ yylhsminor.yy600 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy77, yymsp[0].minor.yy888, NULL); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy701, yymsp[0].minor.yy532, NULL); } + yymsp[-1].minor.yy476 = yylhsminor.yy476; break; case 158: /* column_def ::= column_name type_name COMMENT NK_STRING */ -{ yylhsminor.yy600 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy77, yymsp[-2].minor.yy888, &yymsp[0].minor.yy0); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy701, yymsp[-2].minor.yy532, &yymsp[0].minor.yy0); } + yymsp[-3].minor.yy476 = yylhsminor.yy476; break; case 159: /* type_name ::= BOOL */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_BOOL); } +{ yymsp[0].minor.yy532 = createDataType(TSDB_DATA_TYPE_BOOL); } break; case 160: /* type_name ::= TINYINT */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_TINYINT); } +{ yymsp[0].minor.yy532 = createDataType(TSDB_DATA_TYPE_TINYINT); } break; case 161: /* type_name ::= SMALLINT */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_SMALLINT); } +{ yymsp[0].minor.yy532 = createDataType(TSDB_DATA_TYPE_SMALLINT); } break; case 162: /* type_name ::= INT */ case 163: /* type_name ::= INTEGER */ yytestcase(yyruleno==163); -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_INT); } +{ yymsp[0].minor.yy532 = createDataType(TSDB_DATA_TYPE_INT); } break; case 164: /* type_name ::= BIGINT */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_BIGINT); } +{ yymsp[0].minor.yy532 = createDataType(TSDB_DATA_TYPE_BIGINT); } break; case 165: /* type_name ::= FLOAT */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_FLOAT); } +{ yymsp[0].minor.yy532 = createDataType(TSDB_DATA_TYPE_FLOAT); } break; case 166: /* type_name ::= DOUBLE */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_DOUBLE); } +{ yymsp[0].minor.yy532 = createDataType(TSDB_DATA_TYPE_DOUBLE); } break; case 167: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy888 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy532 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } break; case 168: /* type_name ::= TIMESTAMP */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } +{ yymsp[0].minor.yy532 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } break; case 169: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy888 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy532 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } break; case 170: /* type_name ::= TINYINT UNSIGNED */ -{ yymsp[-1].minor.yy888 = createDataType(TSDB_DATA_TYPE_UTINYINT); } +{ yymsp[-1].minor.yy532 = createDataType(TSDB_DATA_TYPE_UTINYINT); } break; case 171: /* type_name ::= SMALLINT UNSIGNED */ -{ yymsp[-1].minor.yy888 = createDataType(TSDB_DATA_TYPE_USMALLINT); } +{ yymsp[-1].minor.yy532 = createDataType(TSDB_DATA_TYPE_USMALLINT); } break; case 172: /* type_name ::= INT UNSIGNED */ -{ yymsp[-1].minor.yy888 = createDataType(TSDB_DATA_TYPE_UINT); } +{ yymsp[-1].minor.yy532 = createDataType(TSDB_DATA_TYPE_UINT); } break; case 173: /* type_name ::= BIGINT UNSIGNED */ -{ yymsp[-1].minor.yy888 = createDataType(TSDB_DATA_TYPE_UBIGINT); } +{ yymsp[-1].minor.yy532 = createDataType(TSDB_DATA_TYPE_UBIGINT); } break; case 174: /* type_name ::= JSON */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_JSON); } +{ yymsp[0].minor.yy532 = createDataType(TSDB_DATA_TYPE_JSON); } break; case 175: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy888 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy532 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } break; case 176: /* type_name ::= MEDIUMBLOB */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } +{ yymsp[0].minor.yy532 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } break; case 177: /* type_name ::= BLOB */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_BLOB); } +{ yymsp[0].minor.yy532 = createDataType(TSDB_DATA_TYPE_BLOB); } break; case 178: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy888 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy532 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } break; case 179: /* type_name ::= DECIMAL */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +{ yymsp[0].minor.yy532 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 180: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy888 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +{ yymsp[-3].minor.yy532 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 181: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ -{ yymsp[-5].minor.yy888 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +{ yymsp[-5].minor.yy532 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 183: /* tags_def_opt ::= tags_def */ - case 410: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==410); -{ yylhsminor.yy601 = yymsp[0].minor.yy601; } - yymsp[0].minor.yy601 = yylhsminor.yy601; + case 413: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==413); +{ yylhsminor.yy376 = yymsp[0].minor.yy376; } + yymsp[0].minor.yy376 = yylhsminor.yy376; break; case 184: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ -{ yymsp[-3].minor.yy601 = yymsp[-1].minor.yy601; } +{ yymsp[-3].minor.yy376 = yymsp[-1].minor.yy376; } break; case 185: /* table_options ::= */ -{ yymsp[1].minor.yy600 = createDefaultTableOptions(pCxt); } +{ yymsp[1].minor.yy476 = createDefaultTableOptions(pCxt); } break; case 186: /* table_options ::= table_options COMMENT NK_STRING */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setTableOption(pCxt, yymsp[-2].minor.yy476, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 187: /* table_options ::= table_options MAX_DELAY duration_list */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy601); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setTableOption(pCxt, yymsp[-2].minor.yy476, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy376); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 188: /* table_options ::= table_options WATERMARK duration_list */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy601); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setTableOption(pCxt, yymsp[-2].minor.yy476, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy376); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 189: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-4].minor.yy600, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy601); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setTableOption(pCxt, yymsp[-4].minor.yy476, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy376); } + yymsp[-4].minor.yy476 = yylhsminor.yy476; break; case 190: /* table_options ::= table_options TTL NK_INTEGER */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setTableOption(pCxt, yymsp[-2].minor.yy476, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 191: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-4].minor.yy600, TABLE_OPTION_SMA, yymsp[-1].minor.yy601); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setTableOption(pCxt, yymsp[-4].minor.yy476, TABLE_OPTION_SMA, yymsp[-1].minor.yy376); } + yymsp[-4].minor.yy476 = yylhsminor.yy476; break; case 192: /* table_options ::= table_options DELETE_MARK duration_list */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy601); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setTableOption(pCxt, yymsp[-2].minor.yy476, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy376); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 193: /* alter_table_options ::= alter_table_option */ -{ yylhsminor.yy600 = createAlterTableOptions(pCxt); yylhsminor.yy600 = setTableOption(pCxt, yylhsminor.yy600, yymsp[0].minor.yy661.type, &yymsp[0].minor.yy661.val); } - yymsp[0].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createAlterTableOptions(pCxt); yylhsminor.yy476 = setTableOption(pCxt, yylhsminor.yy476, yymsp[0].minor.yy893.type, &yymsp[0].minor.yy893.val); } + yymsp[0].minor.yy476 = yylhsminor.yy476; break; case 194: /* alter_table_options ::= alter_table_options alter_table_option */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-1].minor.yy600, yymsp[0].minor.yy661.type, &yymsp[0].minor.yy661.val); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setTableOption(pCxt, yymsp[-1].minor.yy476, yymsp[0].minor.yy893.type, &yymsp[0].minor.yy893.val); } + yymsp[-1].minor.yy476 = yylhsminor.yy476; break; case 195: /* alter_table_option ::= COMMENT NK_STRING */ -{ yymsp[-1].minor.yy661.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy893.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy893.val = yymsp[0].minor.yy0; } break; case 196: /* alter_table_option ::= TTL NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy893.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy893.val = yymsp[0].minor.yy0; } break; case 197: /* duration_list ::= duration_literal */ - case 374: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==374); -{ yylhsminor.yy601 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); } - yymsp[0].minor.yy601 = yylhsminor.yy601; + case 377: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==377); +{ yylhsminor.yy376 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy476)); } + yymsp[0].minor.yy376 = yylhsminor.yy376; break; case 198: /* duration_list ::= duration_list NK_COMMA duration_literal */ - case 375: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==375); -{ yylhsminor.yy601 = addNodeToList(pCxt, yymsp[-2].minor.yy601, releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); } - yymsp[-2].minor.yy601 = yylhsminor.yy601; + case 378: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==378); +{ yylhsminor.yy376 = addNodeToList(pCxt, yymsp[-2].minor.yy376, releaseRawExprNode(pCxt, yymsp[0].minor.yy476)); } + yymsp[-2].minor.yy376 = yylhsminor.yy376; break; case 201: /* rollup_func_name ::= function_name */ -{ yylhsminor.yy600 = createFunctionNode(pCxt, &yymsp[0].minor.yy77, NULL); } - yymsp[0].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createFunctionNode(pCxt, &yymsp[0].minor.yy701, NULL); } + yymsp[0].minor.yy476 = yylhsminor.yy476; break; case 202: /* rollup_func_name ::= FIRST */ case 203: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==203); case 256: /* tag_item ::= QTAGS */ yytestcase(yyruleno==256); -{ yylhsminor.yy600 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } - yymsp[0].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } + yymsp[0].minor.yy476 = yylhsminor.yy476; break; case 206: /* col_name ::= column_name */ case 257: /* tag_item ::= column_name */ yytestcase(yyruleno==257); -{ yylhsminor.yy600 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy77); } - yymsp[0].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy701); } + yymsp[0].minor.yy476 = yylhsminor.yy476; break; case 207: /* cmd ::= SHOW DNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT); } @@ -5002,13 +4411,13 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT); } break; case 211: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy600, yymsp[0].minor.yy600, OP_TYPE_LIKE); } +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy476, yymsp[0].minor.yy476, OP_TYPE_LIKE); } break; case 212: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy600, yymsp[0].minor.yy600, OP_TYPE_LIKE); } +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy476, yymsp[0].minor.yy476, OP_TYPE_LIKE); } break; case 213: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy600, NULL, OP_TYPE_LIKE); } +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy476, NULL, OP_TYPE_LIKE); } break; case 214: /* cmd ::= SHOW MNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT); } @@ -5020,7 +4429,7 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT); } break; case 217: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy600, yymsp[-1].minor.yy600, OP_TYPE_EQUAL); } +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy476, yymsp[-1].minor.yy476, OP_TYPE_EQUAL); } break; case 218: /* cmd ::= SHOW STREAMS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT); } @@ -5039,13 +4448,13 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCES_STMT); } break; case 224: /* cmd ::= SHOW CREATE DATABASE db_name */ -{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy77); } +{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy701); } break; case 225: /* cmd ::= SHOW CREATE TABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy600); } +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy476); } break; case 226: /* cmd ::= SHOW CREATE STABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy600); } +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy476); } break; case 227: /* cmd ::= SHOW QUERIES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT); } @@ -5064,7 +4473,7 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT); } break; case 233: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy600); } +{ pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy476); } break; case 234: /* cmd ::= SHOW BNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_BNODES_STMT); } @@ -5079,7 +4488,7 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TRANSACTIONS_STMT); } break; case 238: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ -{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy600); } +{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy476); } break; case 239: /* cmd ::= SHOW CONSUMERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONSUMERS_STMT); } @@ -5088,10 +4497,10 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT); } break; case 241: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy600, yymsp[-1].minor.yy600, OP_TYPE_EQUAL); } +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy476, yymsp[-1].minor.yy476, OP_TYPE_EQUAL); } break; case 242: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy600, yymsp[0].minor.yy600, yymsp[-3].minor.yy601); } +{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy476, yymsp[0].minor.yy476, yymsp[-3].minor.yy376); } break; case 243: /* cmd ::= SHOW VNODES NK_INTEGER */ { pCxt->pRootNode = createShowVnodesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0), NULL); } @@ -5101,745 +4510,748 @@ static YYACTIONTYPE yy_reduce( break; case 245: /* db_name_cond_opt ::= */ case 250: /* from_db_opt ::= */ yytestcase(yyruleno==250); -{ yymsp[1].minor.yy600 = createDefaultDatabaseCondValue(pCxt); } +{ yymsp[1].minor.yy476 = createDefaultDatabaseCondValue(pCxt); } break; case 246: /* db_name_cond_opt ::= db_name NK_DOT */ -{ yylhsminor.yy600 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy77); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy701); } + yymsp[-1].minor.yy476 = yylhsminor.yy476; break; case 247: /* like_pattern_opt ::= */ - case 307: /* subtable_opt ::= */ yytestcase(yyruleno==307); - case 420: /* case_when_else_opt ::= */ yytestcase(yyruleno==420); - case 450: /* from_clause_opt ::= */ yytestcase(yyruleno==450); - case 479: /* where_clause_opt ::= */ yytestcase(yyruleno==479); - case 488: /* twindow_clause_opt ::= */ yytestcase(yyruleno==488); - case 494: /* sliding_opt ::= */ yytestcase(yyruleno==494); - case 496: /* fill_opt ::= */ yytestcase(yyruleno==496); - case 508: /* having_clause_opt ::= */ yytestcase(yyruleno==508); - case 510: /* range_opt ::= */ yytestcase(yyruleno==510); - case 512: /* every_opt ::= */ yytestcase(yyruleno==512); - case 525: /* slimit_clause_opt ::= */ yytestcase(yyruleno==525); - case 529: /* limit_clause_opt ::= */ yytestcase(yyruleno==529); -{ yymsp[1].minor.yy600 = NULL; } + case 310: /* subtable_opt ::= */ yytestcase(yyruleno==310); + case 423: /* case_when_else_opt ::= */ yytestcase(yyruleno==423); + case 453: /* from_clause_opt ::= */ yytestcase(yyruleno==453); + case 482: /* where_clause_opt ::= */ yytestcase(yyruleno==482); + case 491: /* twindow_clause_opt ::= */ yytestcase(yyruleno==491); + case 497: /* sliding_opt ::= */ yytestcase(yyruleno==497); + case 499: /* fill_opt ::= */ yytestcase(yyruleno==499); + case 511: /* having_clause_opt ::= */ yytestcase(yyruleno==511); + case 513: /* range_opt ::= */ yytestcase(yyruleno==513); + case 515: /* every_opt ::= */ yytestcase(yyruleno==515); + case 528: /* slimit_clause_opt ::= */ yytestcase(yyruleno==528); + case 532: /* limit_clause_opt ::= */ yytestcase(yyruleno==532); +{ yymsp[1].minor.yy476 = NULL; } break; case 248: /* like_pattern_opt ::= LIKE NK_STRING */ -{ yymsp[-1].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy476 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } break; case 249: /* table_name_cond ::= table_name */ -{ yylhsminor.yy600 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy77); } - yymsp[0].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy701); } + yymsp[0].minor.yy476 = yylhsminor.yy476; break; case 251: /* from_db_opt ::= FROM db_name */ -{ yymsp[-1].minor.yy600 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy77); } +{ yymsp[-1].minor.yy476 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy701); } break; case 255: /* tag_item ::= TBNAME */ -{ yylhsminor.yy600 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } + yymsp[0].minor.yy476 = yylhsminor.yy476; break; case 258: /* tag_item ::= column_name column_alias */ -{ yylhsminor.yy600 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy77), &yymsp[0].minor.yy77); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy701), &yymsp[0].minor.yy701); } + yymsp[-1].minor.yy476 = yylhsminor.yy476; break; case 259: /* tag_item ::= column_name AS column_alias */ -{ yylhsminor.yy600 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy77), &yymsp[0].minor.yy77); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; +{ yylhsminor.yy476 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy701), &yymsp[0].minor.yy701); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; case 260: /* cmd ::= CREATE SMA INDEX not_exists_opt full_table_name ON full_table_name index_options */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy841, yymsp[-3].minor.yy600, yymsp[-1].minor.yy600, NULL, yymsp[0].minor.yy600); } +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy845, yymsp[-3].minor.yy476, yymsp[-1].minor.yy476, NULL, yymsp[0].minor.yy476); } + break; + case 261: /* cmd ::= CREATE INDEX not_exists_opt full_table_name ON full_table_name NK_LP col_name_list NK_RP */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_NORMAL, yymsp[-6].minor.yy845, yymsp[-5].minor.yy476, yymsp[-3].minor.yy476, yymsp[-1].minor.yy376, NULL); } break; - case 261: /* cmd ::= DROP INDEX exists_opt full_table_name */ -{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy841, yymsp[0].minor.yy600); } + case 262: /* cmd ::= DROP INDEX exists_opt full_table_name */ +{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy845, yymsp[0].minor.yy476); } break; - case 262: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ -{ yymsp[-9].minor.yy600 = createIndexOption(pCxt, yymsp[-7].minor.yy601, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), NULL, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } + case 263: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ +{ yymsp[-9].minor.yy476 = createIndexOption(pCxt, yymsp[-7].minor.yy376, releaseRawExprNode(pCxt, yymsp[-3].minor.yy476), NULL, yymsp[-1].minor.yy476, yymsp[0].minor.yy476); } break; - case 263: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ -{ yymsp[-11].minor.yy600 = createIndexOption(pCxt, yymsp[-9].minor.yy601, releaseRawExprNode(pCxt, yymsp[-5].minor.yy600), releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } + case 264: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ +{ yymsp[-11].minor.yy476 = createIndexOption(pCxt, yymsp[-9].minor.yy376, releaseRawExprNode(pCxt, yymsp[-5].minor.yy476), releaseRawExprNode(pCxt, yymsp[-3].minor.yy476), yymsp[-1].minor.yy476, yymsp[0].minor.yy476); } break; - case 266: /* func ::= sma_func_name NK_LP expression_list NK_RP */ -{ yylhsminor.yy600 = createFunctionNode(pCxt, &yymsp[-3].minor.yy77, yymsp[-1].minor.yy601); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + case 267: /* func ::= sma_func_name NK_LP expression_list NK_RP */ +{ yylhsminor.yy476 = createFunctionNode(pCxt, &yymsp[-3].minor.yy701, yymsp[-1].minor.yy376); } + yymsp[-3].minor.yy476 = yylhsminor.yy476; break; - case 272: /* sma_stream_opt ::= */ - case 300: /* stream_options ::= */ yytestcase(yyruleno==300); -{ yymsp[1].minor.yy600 = createStreamOptions(pCxt); } + case 273: /* sma_stream_opt ::= */ + case 303: /* stream_options ::= */ yytestcase(yyruleno==303); +{ yymsp[1].minor.yy476 = createStreamOptions(pCxt); } break; - case 273: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ - case 304: /* stream_options ::= stream_options WATERMARK duration_literal */ yytestcase(yyruleno==304); -{ ((SStreamOptions*)yymsp[-2].minor.yy600)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = yymsp[-2].minor.yy600; } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 274: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ + case 307: /* stream_options ::= stream_options WATERMARK duration_literal */ yytestcase(yyruleno==307); +{ ((SStreamOptions*)yymsp[-2].minor.yy476)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy476); yylhsminor.yy476 = yymsp[-2].minor.yy476; } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 274: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy600)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = yymsp[-2].minor.yy600; } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 275: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy476)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy476); yylhsminor.yy476 = yymsp[-2].minor.yy476; } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 275: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy600)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = yymsp[-2].minor.yy600; } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 276: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy476)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy476); yylhsminor.yy476 = yymsp[-2].minor.yy476; } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 276: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ -{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy841, &yymsp[-2].minor.yy77, yymsp[0].minor.yy600); } + case 277: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ +{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy845, &yymsp[-2].minor.yy701, yymsp[0].minor.yy476); } break; - case 277: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy841, &yymsp[-3].minor.yy77, &yymsp[0].minor.yy77, false); } + case 278: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy845, &yymsp[-3].minor.yy701, &yymsp[0].minor.yy701, false); } break; - case 278: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy841, &yymsp[-5].minor.yy77, &yymsp[0].minor.yy77, true); } + case 279: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy845, &yymsp[-5].minor.yy701, &yymsp[0].minor.yy701, true); } break; - case 279: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-4].minor.yy841, &yymsp[-3].minor.yy77, yymsp[0].minor.yy600, false); } + case 280: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-4].minor.yy845, &yymsp[-3].minor.yy701, yymsp[0].minor.yy476, false); } break; - case 280: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-6].minor.yy841, &yymsp[-5].minor.yy77, yymsp[0].minor.yy600, true); } + case 281: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-6].minor.yy845, &yymsp[-5].minor.yy701, yymsp[0].minor.yy476, true); } break; - case 281: /* cmd ::= DROP TOPIC exists_opt topic_name */ -{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy841, &yymsp[0].minor.yy77); } + case 282: /* cmd ::= DROP TOPIC exists_opt topic_name */ +{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy845, &yymsp[0].minor.yy701); } break; - case 282: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ -{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy841, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy77); } + case 283: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ +{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy845, &yymsp[-2].minor.yy701, &yymsp[0].minor.yy701); } break; - case 283: /* cmd ::= DESC full_table_name */ - case 284: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==284); -{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy600); } + case 284: /* cmd ::= DESC full_table_name */ + case 285: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==285); +{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy476); } break; - case 285: /* cmd ::= RESET QUERY CACHE */ + case 286: /* cmd ::= RESET QUERY CACHE */ { pCxt->pRootNode = createResetQueryCacheStmt(pCxt); } break; - case 286: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ -{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy841, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } + case 287: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ +{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy845, yymsp[-1].minor.yy476, yymsp[0].minor.yy476); } break; - case 289: /* explain_options ::= */ -{ yymsp[1].minor.yy600 = createDefaultExplainOptions(pCxt); } + case 290: /* explain_options ::= */ +{ yymsp[1].minor.yy476 = createDefaultExplainOptions(pCxt); } break; - case 290: /* explain_options ::= explain_options VERBOSE NK_BOOL */ -{ yylhsminor.yy600 = setExplainVerbose(pCxt, yymsp[-2].minor.yy600, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 291: /* explain_options ::= explain_options VERBOSE NK_BOOL */ +{ yylhsminor.yy476 = setExplainVerbose(pCxt, yymsp[-2].minor.yy476, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 291: /* explain_options ::= explain_options RATIO NK_FLOAT */ -{ yylhsminor.yy600 = setExplainRatio(pCxt, yymsp[-2].minor.yy600, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 292: /* explain_options ::= explain_options RATIO NK_FLOAT */ +{ yylhsminor.yy476 = setExplainRatio(pCxt, yymsp[-2].minor.yy476, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 292: /* cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ -{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy841, yymsp[-8].minor.yy841, &yymsp[-5].minor.yy77, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy888, yymsp[0].minor.yy248); } + case 293: /* cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ +{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy845, yymsp[-8].minor.yy845, &yymsp[-5].minor.yy701, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy532, yymsp[0].minor.yy508); } break; - case 293: /* cmd ::= DROP FUNCTION exists_opt function_name */ -{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy841, &yymsp[0].minor.yy77); } + case 294: /* cmd ::= DROP FUNCTION exists_opt function_name */ +{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy845, &yymsp[0].minor.yy701); } break; - case 298: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name tags_def_opt subtable_opt AS query_or_subquery */ -{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-8].minor.yy841, &yymsp[-7].minor.yy77, yymsp[-4].minor.yy600, yymsp[-6].minor.yy600, yymsp[-3].minor.yy601, yymsp[-2].minor.yy600, yymsp[0].minor.yy600); } + case 299: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tags_def_opt subtable_opt AS query_or_subquery */ +{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-9].minor.yy845, &yymsp[-8].minor.yy701, yymsp[-5].minor.yy476, yymsp[-7].minor.yy476, yymsp[-3].minor.yy376, yymsp[-2].minor.yy476, yymsp[0].minor.yy476, yymsp[-4].minor.yy376); } break; - case 299: /* cmd ::= DROP STREAM exists_opt stream_name */ -{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy841, &yymsp[0].minor.yy77); } + case 300: /* cmd ::= DROP STREAM exists_opt stream_name */ +{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy845, &yymsp[0].minor.yy701); } break; - case 301: /* stream_options ::= stream_options TRIGGER AT_ONCE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy600)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy600 = yymsp[-2].minor.yy600; } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 304: /* stream_options ::= stream_options TRIGGER AT_ONCE */ +{ ((SStreamOptions*)yymsp[-2].minor.yy476)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy476 = yymsp[-2].minor.yy476; } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 302: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy600)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy600 = yymsp[-2].minor.yy600; } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 305: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ +{ ((SStreamOptions*)yymsp[-2].minor.yy476)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy476 = yymsp[-2].minor.yy476; } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 303: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ -{ ((SStreamOptions*)yymsp[-3].minor.yy600)->triggerType = STREAM_TRIGGER_MAX_DELAY; ((SStreamOptions*)yymsp[-3].minor.yy600)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = yymsp[-3].minor.yy600; } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + case 306: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ +{ ((SStreamOptions*)yymsp[-3].minor.yy476)->triggerType = STREAM_TRIGGER_MAX_DELAY; ((SStreamOptions*)yymsp[-3].minor.yy476)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy476); yylhsminor.yy476 = yymsp[-3].minor.yy476; } + yymsp[-3].minor.yy476 = yylhsminor.yy476; break; - case 305: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ -{ ((SStreamOptions*)yymsp[-3].minor.yy600)->ignoreExpired = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy600 = yymsp[-3].minor.yy600; } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + case 308: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ +{ ((SStreamOptions*)yymsp[-3].minor.yy476)->ignoreExpired = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy476 = yymsp[-3].minor.yy476; } + yymsp[-3].minor.yy476 = yylhsminor.yy476; break; - case 306: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ -{ ((SStreamOptions*)yymsp[-2].minor.yy600)->fillHistory = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy600 = yymsp[-2].minor.yy600; } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 309: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ +{ ((SStreamOptions*)yymsp[-2].minor.yy476)->fillHistory = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy476 = yymsp[-2].minor.yy476; } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 308: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ - case 495: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==495); - case 513: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==513); -{ yymsp[-3].minor.yy600 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy600); } + case 311: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ + case 498: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==498); + case 516: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==516); +{ yymsp[-3].minor.yy476 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy476); } break; - case 309: /* cmd ::= KILL CONNECTION NK_INTEGER */ + case 312: /* cmd ::= KILL CONNECTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &yymsp[0].minor.yy0); } break; - case 310: /* cmd ::= KILL QUERY NK_STRING */ + case 313: /* cmd ::= KILL QUERY NK_STRING */ { pCxt->pRootNode = createKillQueryStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 311: /* cmd ::= KILL TRANSACTION NK_INTEGER */ + case 314: /* cmd ::= KILL TRANSACTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_TRANSACTION_STMT, &yymsp[0].minor.yy0); } break; - case 312: /* cmd ::= BALANCE VGROUP */ + case 315: /* cmd ::= BALANCE VGROUP */ { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); } break; - case 313: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + case 316: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 314: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ -{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy601); } + case 317: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ +{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy376); } break; - case 315: /* cmd ::= SPLIT VGROUP NK_INTEGER */ + case 318: /* cmd ::= SPLIT VGROUP NK_INTEGER */ { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 316: /* dnode_list ::= DNODE NK_INTEGER */ -{ yymsp[-1].minor.yy601 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - break; - case 318: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ -{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } - break; - case 320: /* cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ -{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-4].minor.yy600, yymsp[-2].minor.yy601, yymsp[0].minor.yy600); } - break; - case 321: /* cmd ::= INSERT INTO full_table_name query_or_subquery */ -{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-1].minor.yy600, NULL, yymsp[0].minor.yy600); } - break; - case 322: /* literal ::= NK_INTEGER */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 323: /* literal ::= NK_FLOAT */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 324: /* literal ::= NK_STRING */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 325: /* literal ::= NK_BOOL */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 326: /* literal ::= TIMESTAMP NK_STRING */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; - break; - case 327: /* literal ::= duration_literal */ - case 337: /* signed_literal ::= signed */ yytestcase(yyruleno==337); - case 357: /* expr_or_subquery ::= expression */ yytestcase(yyruleno==357); - case 358: /* expression ::= literal */ yytestcase(yyruleno==358); - case 359: /* expression ::= pseudo_column */ yytestcase(yyruleno==359); - case 360: /* expression ::= column_reference */ yytestcase(yyruleno==360); - case 361: /* expression ::= function_expression */ yytestcase(yyruleno==361); - case 362: /* expression ::= case_when_expression */ yytestcase(yyruleno==362); - case 393: /* function_expression ::= literal_func */ yytestcase(yyruleno==393); - case 442: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==442); - case 446: /* boolean_primary ::= predicate */ yytestcase(yyruleno==446); - case 448: /* common_expression ::= expr_or_subquery */ yytestcase(yyruleno==448); - case 449: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==449); - case 452: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==452); - case 454: /* table_reference ::= table_primary */ yytestcase(yyruleno==454); - case 455: /* table_reference ::= joined_table */ yytestcase(yyruleno==455); - case 459: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==459); - case 515: /* query_simple ::= query_specification */ yytestcase(yyruleno==515); - case 516: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==516); - case 519: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==519); - case 521: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==521); -{ yylhsminor.yy600 = yymsp[0].minor.yy600; } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 328: /* literal ::= NULL */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 329: /* literal ::= NK_QUESTION */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 330: /* duration_literal ::= NK_VARIABLE */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 331: /* signed ::= NK_INTEGER */ -{ yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 332: /* signed ::= NK_PLUS NK_INTEGER */ -{ yymsp[-1].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } - break; - case 333: /* signed ::= NK_MINUS NK_INTEGER */ + case 319: /* dnode_list ::= DNODE NK_INTEGER */ +{ yymsp[-1].minor.yy376 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + break; + case 321: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ +{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy476, yymsp[0].minor.yy476); } + break; + case 323: /* cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ +{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-4].minor.yy476, yymsp[-2].minor.yy376, yymsp[0].minor.yy476); } + break; + case 324: /* cmd ::= INSERT INTO full_table_name query_or_subquery */ +{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-1].minor.yy476, NULL, yymsp[0].minor.yy476); } + break; + case 325: /* literal ::= NK_INTEGER */ +{ yylhsminor.yy476 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy476 = yylhsminor.yy476; + break; + case 326: /* literal ::= NK_FLOAT */ +{ yylhsminor.yy476 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy476 = yylhsminor.yy476; + break; + case 327: /* literal ::= NK_STRING */ +{ yylhsminor.yy476 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy476 = yylhsminor.yy476; + break; + case 328: /* literal ::= NK_BOOL */ +{ yylhsminor.yy476 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy476 = yylhsminor.yy476; + break; + case 329: /* literal ::= TIMESTAMP NK_STRING */ +{ yylhsminor.yy476 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy476 = yylhsminor.yy476; + break; + case 330: /* literal ::= duration_literal */ + case 340: /* signed_literal ::= signed */ yytestcase(yyruleno==340); + case 360: /* expr_or_subquery ::= expression */ yytestcase(yyruleno==360); + case 361: /* expression ::= literal */ yytestcase(yyruleno==361); + case 362: /* expression ::= pseudo_column */ yytestcase(yyruleno==362); + case 363: /* expression ::= column_reference */ yytestcase(yyruleno==363); + case 364: /* expression ::= function_expression */ yytestcase(yyruleno==364); + case 365: /* expression ::= case_when_expression */ yytestcase(yyruleno==365); + case 396: /* function_expression ::= literal_func */ yytestcase(yyruleno==396); + case 445: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==445); + case 449: /* boolean_primary ::= predicate */ yytestcase(yyruleno==449); + case 451: /* common_expression ::= expr_or_subquery */ yytestcase(yyruleno==451); + case 452: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==452); + case 455: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==455); + case 457: /* table_reference ::= table_primary */ yytestcase(yyruleno==457); + case 458: /* table_reference ::= joined_table */ yytestcase(yyruleno==458); + case 462: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==462); + case 518: /* query_simple ::= query_specification */ yytestcase(yyruleno==518); + case 519: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==519); + case 522: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==522); + case 524: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==524); +{ yylhsminor.yy476 = yymsp[0].minor.yy476; } + yymsp[0].minor.yy476 = yylhsminor.yy476; + break; + case 331: /* literal ::= NULL */ +{ yylhsminor.yy476 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy476 = yylhsminor.yy476; + break; + case 332: /* literal ::= NK_QUESTION */ +{ yylhsminor.yy476 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy476 = yylhsminor.yy476; + break; + case 333: /* duration_literal ::= NK_VARIABLE */ +{ yylhsminor.yy476 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy476 = yylhsminor.yy476; + break; + case 334: /* signed ::= NK_INTEGER */ +{ yylhsminor.yy476 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy476 = yylhsminor.yy476; + break; + case 335: /* signed ::= NK_PLUS NK_INTEGER */ +{ yymsp[-1].minor.yy476 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } + break; + case 336: /* signed ::= NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); + yylhsminor.yy476 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + yymsp[-1].minor.yy476 = yylhsminor.yy476; break; - case 334: /* signed ::= NK_FLOAT */ -{ yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; + case 337: /* signed ::= NK_FLOAT */ +{ yylhsminor.yy476 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy476 = yylhsminor.yy476; break; - case 335: /* signed ::= NK_PLUS NK_FLOAT */ -{ yymsp[-1].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + case 338: /* signed ::= NK_PLUS NK_FLOAT */ +{ yymsp[-1].minor.yy476 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } break; - case 336: /* signed ::= NK_MINUS NK_FLOAT */ + case 339: /* signed ::= NK_MINUS NK_FLOAT */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); + yylhsminor.yy476 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; - break; - case 338: /* signed_literal ::= NK_STRING */ -{ yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 339: /* signed_literal ::= NK_BOOL */ -{ yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 340: /* signed_literal ::= TIMESTAMP NK_STRING */ -{ yymsp[-1].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } - break; - case 341: /* signed_literal ::= duration_literal */ - case 343: /* signed_literal ::= literal_func */ yytestcase(yyruleno==343); - case 413: /* star_func_para ::= expr_or_subquery */ yytestcase(yyruleno==413); - case 475: /* select_item ::= common_expression */ yytestcase(yyruleno==475); - case 485: /* partition_item ::= expr_or_subquery */ yytestcase(yyruleno==485); - case 520: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==520); - case 522: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==522); - case 535: /* search_condition ::= common_expression */ yytestcase(yyruleno==535); -{ yylhsminor.yy600 = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 342: /* signed_literal ::= NULL */ -{ yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 344: /* signed_literal ::= NK_QUESTION */ -{ yylhsminor.yy600 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 363: /* expression ::= NK_LP expression NK_RP */ - case 447: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==447); - case 534: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==534); -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 364: /* expression ::= NK_PLUS expr_or_subquery */ + yymsp[-1].minor.yy476 = yylhsminor.yy476; + break; + case 341: /* signed_literal ::= NK_STRING */ +{ yylhsminor.yy476 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy476 = yylhsminor.yy476; + break; + case 342: /* signed_literal ::= NK_BOOL */ +{ yylhsminor.yy476 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy476 = yylhsminor.yy476; + break; + case 343: /* signed_literal ::= TIMESTAMP NK_STRING */ +{ yymsp[-1].minor.yy476 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } + break; + case 344: /* signed_literal ::= duration_literal */ + case 346: /* signed_literal ::= literal_func */ yytestcase(yyruleno==346); + case 416: /* star_func_para ::= expr_or_subquery */ yytestcase(yyruleno==416); + case 478: /* select_item ::= common_expression */ yytestcase(yyruleno==478); + case 488: /* partition_item ::= expr_or_subquery */ yytestcase(yyruleno==488); + case 523: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==523); + case 525: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==525); + case 538: /* search_condition ::= common_expression */ yytestcase(yyruleno==538); +{ yylhsminor.yy476 = releaseRawExprNode(pCxt, yymsp[0].minor.yy476); } + yymsp[0].minor.yy476 = yylhsminor.yy476; + break; + case 345: /* signed_literal ::= NULL */ +{ yylhsminor.yy476 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy476 = yylhsminor.yy476; + break; + case 347: /* signed_literal ::= NK_QUESTION */ +{ yylhsminor.yy476 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy476 = yylhsminor.yy476; + break; + case 366: /* expression ::= NK_LP expression NK_RP */ + case 450: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==450); + case 537: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==537); +{ yylhsminor.yy476 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy476)); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; + break; + case 367: /* expression ::= NK_PLUS expr_or_subquery */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy476); + yylhsminor.yy476 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy476)); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + yymsp[-1].minor.yy476 = yylhsminor.yy476; break; - case 365: /* expression ::= NK_MINUS expr_or_subquery */ + case 368: /* expression ::= NK_MINUS expr_or_subquery */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy600), NULL)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy476); + yylhsminor.yy476 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy476), NULL)); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + yymsp[-1].minor.yy476 = yylhsminor.yy476; break; - case 366: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ + case 369: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy476); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy476); + yylhsminor.yy476 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy476), releaseRawExprNode(pCxt, yymsp[0].minor.yy476))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 367: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ + case 370: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy476); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy476); + yylhsminor.yy476 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy476), releaseRawExprNode(pCxt, yymsp[0].minor.yy476))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 368: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ + case 371: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy476); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy476); + yylhsminor.yy476 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy476), releaseRawExprNode(pCxt, yymsp[0].minor.yy476))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 369: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ + case 372: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy476); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy476); + yylhsminor.yy476 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy476), releaseRawExprNode(pCxt, yymsp[0].minor.yy476))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 370: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ + case 373: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy476); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy476); + yylhsminor.yy476 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy476), releaseRawExprNode(pCxt, yymsp[0].minor.yy476))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 371: /* expression ::= column_reference NK_ARROW NK_STRING */ + case 374: /* expression ::= column_reference NK_ARROW NK_STRING */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy476); + yylhsminor.yy476 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy476), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 372: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ + case 375: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy476); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy476); + yylhsminor.yy476 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy476), releaseRawExprNode(pCxt, yymsp[0].minor.yy476))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 373: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ + case 376: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy476); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy476); + yylhsminor.yy476 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy476), releaseRawExprNode(pCxt, yymsp[0].minor.yy476))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 376: /* column_reference ::= column_name */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy77, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy77)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 377: /* column_reference ::= table_name NK_DOT column_name */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy77, createColumnNode(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy77)); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 378: /* pseudo_column ::= ROWTS */ - case 379: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==379); - case 381: /* pseudo_column ::= QSTART */ yytestcase(yyruleno==381); - case 382: /* pseudo_column ::= QEND */ yytestcase(yyruleno==382); - case 383: /* pseudo_column ::= QDURATION */ yytestcase(yyruleno==383); - case 384: /* pseudo_column ::= WSTART */ yytestcase(yyruleno==384); - case 385: /* pseudo_column ::= WEND */ yytestcase(yyruleno==385); - case 386: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==386); - case 387: /* pseudo_column ::= IROWTS */ yytestcase(yyruleno==387); - case 388: /* pseudo_column ::= ISFILLED */ yytestcase(yyruleno==388); - case 389: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==389); - case 395: /* literal_func ::= NOW */ yytestcase(yyruleno==395); -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 380: /* pseudo_column ::= table_name NK_DOT TBNAME */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy77)))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 390: /* function_expression ::= function_name NK_LP expression_list NK_RP */ - case 391: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==391); -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy77, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy77, yymsp[-1].minor.yy601)); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; - break; - case 392: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), yymsp[-1].minor.yy888)); } - yymsp[-5].minor.yy600 = yylhsminor.yy600; - break; - case 394: /* literal_func ::= noarg_func NK_LP NK_RP */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy77, NULL)); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 409: /* star_func_para_list ::= NK_STAR */ -{ yylhsminor.yy601 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy601 = yylhsminor.yy601; - break; - case 414: /* star_func_para ::= table_name NK_DOT NK_STAR */ - case 478: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==478); -{ yylhsminor.yy600 = createColumnNode(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 415: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy601, yymsp[-1].minor.yy600)); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; - break; - case 416: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), yymsp[-2].minor.yy601, yymsp[-1].minor.yy600)); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; - break; - case 419: /* when_then_expr ::= WHEN common_expression THEN common_expression */ -{ yymsp[-3].minor.yy600 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); } - break; - case 421: /* case_when_else_opt ::= ELSE common_expression */ -{ yymsp[-1].minor.yy600 = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); } - break; - case 422: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ - case 427: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==427); + yymsp[-2].minor.yy476 = yylhsminor.yy476; + break; + case 379: /* column_reference ::= column_name */ +{ yylhsminor.yy476 = createRawExprNode(pCxt, &yymsp[0].minor.yy701, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy701)); } + yymsp[0].minor.yy476 = yylhsminor.yy476; + break; + case 380: /* column_reference ::= table_name NK_DOT column_name */ +{ yylhsminor.yy476 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy701, &yymsp[0].minor.yy701, createColumnNode(pCxt, &yymsp[-2].minor.yy701, &yymsp[0].minor.yy701)); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; + break; + case 381: /* pseudo_column ::= ROWTS */ + case 382: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==382); + case 384: /* pseudo_column ::= QSTART */ yytestcase(yyruleno==384); + case 385: /* pseudo_column ::= QEND */ yytestcase(yyruleno==385); + case 386: /* pseudo_column ::= QDURATION */ yytestcase(yyruleno==386); + case 387: /* pseudo_column ::= WSTART */ yytestcase(yyruleno==387); + case 388: /* pseudo_column ::= WEND */ yytestcase(yyruleno==388); + case 389: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==389); + case 390: /* pseudo_column ::= IROWTS */ yytestcase(yyruleno==390); + case 391: /* pseudo_column ::= ISFILLED */ yytestcase(yyruleno==391); + case 392: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==392); + case 398: /* literal_func ::= NOW */ yytestcase(yyruleno==398); +{ yylhsminor.yy476 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } + yymsp[0].minor.yy476 = yylhsminor.yy476; + break; + case 383: /* pseudo_column ::= table_name NK_DOT TBNAME */ +{ yylhsminor.yy476 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy701, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy701)))); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; + break; + case 393: /* function_expression ::= function_name NK_LP expression_list NK_RP */ + case 394: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==394); +{ yylhsminor.yy476 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy701, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy701, yymsp[-1].minor.yy376)); } + yymsp[-3].minor.yy476 = yylhsminor.yy476; + break; + case 395: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ +{ yylhsminor.yy476 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy476), yymsp[-1].minor.yy532)); } + yymsp[-5].minor.yy476 = yylhsminor.yy476; + break; + case 397: /* literal_func ::= noarg_func NK_LP NK_RP */ +{ yylhsminor.yy476 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy701, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy701, NULL)); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; + break; + case 412: /* star_func_para_list ::= NK_STAR */ +{ yylhsminor.yy376 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy376 = yylhsminor.yy376; + break; + case 417: /* star_func_para ::= table_name NK_DOT NK_STAR */ + case 481: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==481); +{ yylhsminor.yy476 = createColumnNode(pCxt, &yymsp[-2].minor.yy701, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; + break; + case 418: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ +{ yylhsminor.yy476 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy376, yymsp[-1].minor.yy476)); } + yymsp[-3].minor.yy476 = yylhsminor.yy476; + break; + case 419: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ +{ yylhsminor.yy476 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy476), yymsp[-2].minor.yy376, yymsp[-1].minor.yy476)); } + yymsp[-4].minor.yy476 = yylhsminor.yy476; + break; + case 422: /* when_then_expr ::= WHEN common_expression THEN common_expression */ +{ yymsp[-3].minor.yy476 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy476), releaseRawExprNode(pCxt, yymsp[0].minor.yy476)); } + break; + case 424: /* case_when_else_opt ::= ELSE common_expression */ +{ yymsp[-1].minor.yy476 = releaseRawExprNode(pCxt, yymsp[0].minor.yy476); } + break; + case 425: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ + case 430: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==430); { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy666, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy476); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy476); + yylhsminor.yy476 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy128, releaseRawExprNode(pCxt, yymsp[-2].minor.yy476), releaseRawExprNode(pCxt, yymsp[0].minor.yy476))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 423: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ + case 426: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy600), releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy476); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy476); + yylhsminor.yy476 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy476), releaseRawExprNode(pCxt, yymsp[-2].minor.yy476), releaseRawExprNode(pCxt, yymsp[0].minor.yy476))); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; + yymsp[-4].minor.yy476 = yylhsminor.yy476; break; - case 424: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ + case 427: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy600), releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy476); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy476); + yylhsminor.yy476 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy476), releaseRawExprNode(pCxt, yymsp[-2].minor.yy476), releaseRawExprNode(pCxt, yymsp[0].minor.yy476))); } - yymsp[-5].minor.yy600 = yylhsminor.yy600; + yymsp[-5].minor.yy476 = yylhsminor.yy476; break; - case 425: /* predicate ::= expr_or_subquery IS NULL */ + case 428: /* predicate ::= expr_or_subquery IS NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy476); + yylhsminor.yy476 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy476), NULL)); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 426: /* predicate ::= expr_or_subquery IS NOT NULL */ + case 429: /* predicate ::= expr_or_subquery IS NOT NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy476); + yylhsminor.yy476 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy476), NULL)); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + yymsp[-3].minor.yy476 = yylhsminor.yy476; break; - case 428: /* compare_op ::= NK_LT */ -{ yymsp[0].minor.yy666 = OP_TYPE_LOWER_THAN; } + case 431: /* compare_op ::= NK_LT */ +{ yymsp[0].minor.yy128 = OP_TYPE_LOWER_THAN; } break; - case 429: /* compare_op ::= NK_GT */ -{ yymsp[0].minor.yy666 = OP_TYPE_GREATER_THAN; } + case 432: /* compare_op ::= NK_GT */ +{ yymsp[0].minor.yy128 = OP_TYPE_GREATER_THAN; } break; - case 430: /* compare_op ::= NK_LE */ -{ yymsp[0].minor.yy666 = OP_TYPE_LOWER_EQUAL; } + case 433: /* compare_op ::= NK_LE */ +{ yymsp[0].minor.yy128 = OP_TYPE_LOWER_EQUAL; } break; - case 431: /* compare_op ::= NK_GE */ -{ yymsp[0].minor.yy666 = OP_TYPE_GREATER_EQUAL; } + case 434: /* compare_op ::= NK_GE */ +{ yymsp[0].minor.yy128 = OP_TYPE_GREATER_EQUAL; } break; - case 432: /* compare_op ::= NK_NE */ -{ yymsp[0].minor.yy666 = OP_TYPE_NOT_EQUAL; } + case 435: /* compare_op ::= NK_NE */ +{ yymsp[0].minor.yy128 = OP_TYPE_NOT_EQUAL; } break; - case 433: /* compare_op ::= NK_EQ */ -{ yymsp[0].minor.yy666 = OP_TYPE_EQUAL; } + case 436: /* compare_op ::= NK_EQ */ +{ yymsp[0].minor.yy128 = OP_TYPE_EQUAL; } break; - case 434: /* compare_op ::= LIKE */ -{ yymsp[0].minor.yy666 = OP_TYPE_LIKE; } + case 437: /* compare_op ::= LIKE */ +{ yymsp[0].minor.yy128 = OP_TYPE_LIKE; } break; - case 435: /* compare_op ::= NOT LIKE */ -{ yymsp[-1].minor.yy666 = OP_TYPE_NOT_LIKE; } + case 438: /* compare_op ::= NOT LIKE */ +{ yymsp[-1].minor.yy128 = OP_TYPE_NOT_LIKE; } break; - case 436: /* compare_op ::= MATCH */ -{ yymsp[0].minor.yy666 = OP_TYPE_MATCH; } + case 439: /* compare_op ::= MATCH */ +{ yymsp[0].minor.yy128 = OP_TYPE_MATCH; } break; - case 437: /* compare_op ::= NMATCH */ -{ yymsp[0].minor.yy666 = OP_TYPE_NMATCH; } + case 440: /* compare_op ::= NMATCH */ +{ yymsp[0].minor.yy128 = OP_TYPE_NMATCH; } break; - case 438: /* compare_op ::= CONTAINS */ -{ yymsp[0].minor.yy666 = OP_TYPE_JSON_CONTAINS; } + case 441: /* compare_op ::= CONTAINS */ +{ yymsp[0].minor.yy128 = OP_TYPE_JSON_CONTAINS; } break; - case 439: /* in_op ::= IN */ -{ yymsp[0].minor.yy666 = OP_TYPE_IN; } + case 442: /* in_op ::= IN */ +{ yymsp[0].minor.yy128 = OP_TYPE_IN; } break; - case 440: /* in_op ::= NOT IN */ -{ yymsp[-1].minor.yy666 = OP_TYPE_NOT_IN; } + case 443: /* in_op ::= NOT IN */ +{ yymsp[-1].minor.yy128 = OP_TYPE_NOT_IN; } break; - case 441: /* in_predicate_value ::= NK_LP literal_list NK_RP */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy601)); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 444: /* in_predicate_value ::= NK_LP literal_list NK_RP */ +{ yylhsminor.yy476 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy376)); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 443: /* boolean_value_expression ::= NOT boolean_primary */ + case 446: /* boolean_value_expression ::= NOT boolean_primary */ { - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy600), NULL)); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy476); + yylhsminor.yy476 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy476), NULL)); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + yymsp[-1].minor.yy476 = yylhsminor.yy476; break; - case 444: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + case 447: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy476); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy476); + yylhsminor.yy476 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy476), releaseRawExprNode(pCxt, yymsp[0].minor.yy476))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 445: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + case 448: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy476); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy476); + yylhsminor.yy476 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy476), releaseRawExprNode(pCxt, yymsp[0].minor.yy476))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 451: /* from_clause_opt ::= FROM table_reference_list */ - case 480: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==480); - case 509: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==509); -{ yymsp[-1].minor.yy600 = yymsp[0].minor.yy600; } + case 454: /* from_clause_opt ::= FROM table_reference_list */ + case 483: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==483); + case 512: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==512); +{ yymsp[-1].minor.yy476 = yymsp[0].minor.yy476; } break; - case 453: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -{ yylhsminor.yy600 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy600, yymsp[0].minor.yy600, NULL); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 456: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ yylhsminor.yy476 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy476, yymsp[0].minor.yy476, NULL); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 456: /* table_primary ::= table_name alias_opt */ -{ yylhsminor.yy600 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy77, &yymsp[0].minor.yy77); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + case 459: /* table_primary ::= table_name alias_opt */ +{ yylhsminor.yy476 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy701, &yymsp[0].minor.yy701); } + yymsp[-1].minor.yy476 = yylhsminor.yy476; break; - case 457: /* table_primary ::= db_name NK_DOT table_name alias_opt */ -{ yylhsminor.yy600 = createRealTableNode(pCxt, &yymsp[-3].minor.yy77, &yymsp[-1].minor.yy77, &yymsp[0].minor.yy77); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + case 460: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ yylhsminor.yy476 = createRealTableNode(pCxt, &yymsp[-3].minor.yy701, &yymsp[-1].minor.yy701, &yymsp[0].minor.yy701); } + yymsp[-3].minor.yy476 = yylhsminor.yy476; break; - case 458: /* table_primary ::= subquery alias_opt */ -{ yylhsminor.yy600 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600), &yymsp[0].minor.yy77); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + case 461: /* table_primary ::= subquery alias_opt */ +{ yylhsminor.yy476 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy476), &yymsp[0].minor.yy701); } + yymsp[-1].minor.yy476 = yylhsminor.yy476; break; - case 460: /* alias_opt ::= */ -{ yymsp[1].minor.yy77 = nil_token; } + case 463: /* alias_opt ::= */ +{ yymsp[1].minor.yy701 = nil_token; } break; - case 462: /* alias_opt ::= AS table_alias */ -{ yymsp[-1].minor.yy77 = yymsp[0].minor.yy77; } + case 465: /* alias_opt ::= AS table_alias */ +{ yymsp[-1].minor.yy701 = yymsp[0].minor.yy701; } break; - case 463: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - case 464: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==464); -{ yymsp[-2].minor.yy600 = yymsp[-1].minor.yy600; } + case 466: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + case 467: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==467); +{ yymsp[-2].minor.yy476 = yymsp[-1].minor.yy476; } break; - case 465: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -{ yylhsminor.yy600 = createJoinTableNode(pCxt, yymsp[-4].minor.yy560, yymsp[-5].minor.yy600, yymsp[-2].minor.yy600, yymsp[0].minor.yy600); } - yymsp[-5].minor.yy600 = yylhsminor.yy600; + case 468: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ yylhsminor.yy476 = createJoinTableNode(pCxt, yymsp[-4].minor.yy288, yymsp[-5].minor.yy476, yymsp[-2].minor.yy476, yymsp[0].minor.yy476); } + yymsp[-5].minor.yy476 = yylhsminor.yy476; break; - case 466: /* join_type ::= */ -{ yymsp[1].minor.yy560 = JOIN_TYPE_INNER; } + case 469: /* join_type ::= */ +{ yymsp[1].minor.yy288 = JOIN_TYPE_INNER; } break; - case 467: /* join_type ::= INNER */ -{ yymsp[0].minor.yy560 = JOIN_TYPE_INNER; } + case 470: /* join_type ::= INNER */ +{ yymsp[0].minor.yy288 = JOIN_TYPE_INNER; } break; - case 468: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 471: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { - yymsp[-11].minor.yy600 = createSelectStmt(pCxt, yymsp[-10].minor.yy841, yymsp[-9].minor.yy601, yymsp[-8].minor.yy600); - yymsp[-11].minor.yy600 = addWhereClause(pCxt, yymsp[-11].minor.yy600, yymsp[-7].minor.yy600); - yymsp[-11].minor.yy600 = addPartitionByClause(pCxt, yymsp[-11].minor.yy600, yymsp[-6].minor.yy601); - yymsp[-11].minor.yy600 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy600, yymsp[-2].minor.yy600); - yymsp[-11].minor.yy600 = addGroupByClause(pCxt, yymsp[-11].minor.yy600, yymsp[-1].minor.yy601); - yymsp[-11].minor.yy600 = addHavingClause(pCxt, yymsp[-11].minor.yy600, yymsp[0].minor.yy600); - yymsp[-11].minor.yy600 = addRangeClause(pCxt, yymsp[-11].minor.yy600, yymsp[-5].minor.yy600); - yymsp[-11].minor.yy600 = addEveryClause(pCxt, yymsp[-11].minor.yy600, yymsp[-4].minor.yy600); - yymsp[-11].minor.yy600 = addFillClause(pCxt, yymsp[-11].minor.yy600, yymsp[-3].minor.yy600); + yymsp[-11].minor.yy476 = createSelectStmt(pCxt, yymsp[-10].minor.yy845, yymsp[-9].minor.yy376, yymsp[-8].minor.yy476); + yymsp[-11].minor.yy476 = addWhereClause(pCxt, yymsp[-11].minor.yy476, yymsp[-7].minor.yy476); + yymsp[-11].minor.yy476 = addPartitionByClause(pCxt, yymsp[-11].minor.yy476, yymsp[-6].minor.yy376); + yymsp[-11].minor.yy476 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy476, yymsp[-2].minor.yy476); + yymsp[-11].minor.yy476 = addGroupByClause(pCxt, yymsp[-11].minor.yy476, yymsp[-1].minor.yy376); + yymsp[-11].minor.yy476 = addHavingClause(pCxt, yymsp[-11].minor.yy476, yymsp[0].minor.yy476); + yymsp[-11].minor.yy476 = addRangeClause(pCxt, yymsp[-11].minor.yy476, yymsp[-5].minor.yy476); + yymsp[-11].minor.yy476 = addEveryClause(pCxt, yymsp[-11].minor.yy476, yymsp[-4].minor.yy476); + yymsp[-11].minor.yy476 = addFillClause(pCxt, yymsp[-11].minor.yy476, yymsp[-3].minor.yy476); } break; - case 471: /* set_quantifier_opt ::= ALL */ -{ yymsp[0].minor.yy841 = false; } + case 474: /* set_quantifier_opt ::= ALL */ +{ yymsp[0].minor.yy845 = false; } break; - case 474: /* select_item ::= NK_STAR */ -{ yylhsminor.yy600 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; + case 477: /* select_item ::= NK_STAR */ +{ yylhsminor.yy476 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy476 = yylhsminor.yy476; break; - case 476: /* select_item ::= common_expression column_alias */ - case 486: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==486); -{ yylhsminor.yy600 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600), &yymsp[0].minor.yy77); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + case 479: /* select_item ::= common_expression column_alias */ + case 489: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==489); +{ yylhsminor.yy476 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy476), &yymsp[0].minor.yy701); } + yymsp[-1].minor.yy476 = yylhsminor.yy476; break; - case 477: /* select_item ::= common_expression AS column_alias */ - case 487: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==487); -{ yylhsminor.yy600 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), &yymsp[0].minor.yy77); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 480: /* select_item ::= common_expression AS column_alias */ + case 490: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==490); +{ yylhsminor.yy476 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy476), &yymsp[0].minor.yy701); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 482: /* partition_by_clause_opt ::= PARTITION BY partition_list */ - case 505: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==505); - case 524: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==524); -{ yymsp[-2].minor.yy601 = yymsp[0].minor.yy601; } + case 485: /* partition_by_clause_opt ::= PARTITION BY partition_list */ + case 508: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==508); + case 527: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==527); +{ yymsp[-2].minor.yy376 = yymsp[0].minor.yy376; } break; - case 489: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ -{ yymsp[-5].minor.yy600 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); } + case 492: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ +{ yymsp[-5].minor.yy476 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy476), releaseRawExprNode(pCxt, yymsp[-1].minor.yy476)); } break; - case 490: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ -{ yymsp[-3].minor.yy600 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); } + case 493: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ +{ yymsp[-3].minor.yy476 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy476)); } break; - case 491: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-5].minor.yy600 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), NULL, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } + case 494: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-5].minor.yy476 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy476), NULL, yymsp[-1].minor.yy476, yymsp[0].minor.yy476); } break; - case 492: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-7].minor.yy600 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy600), releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } + case 495: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-7].minor.yy476 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy476), releaseRawExprNode(pCxt, yymsp[-3].minor.yy476), yymsp[-1].minor.yy476, yymsp[0].minor.yy476); } break; - case 493: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ -{ yymsp[-6].minor.yy600 = createEventWindowNode(pCxt, yymsp[-3].minor.yy600, yymsp[0].minor.yy600); } + case 496: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ +{ yymsp[-6].minor.yy476 = createEventWindowNode(pCxt, yymsp[-3].minor.yy476, yymsp[0].minor.yy476); } break; - case 497: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -{ yymsp[-3].minor.yy600 = createFillNode(pCxt, yymsp[-1].minor.yy798, NULL); } + case 500: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ yymsp[-3].minor.yy476 = createFillNode(pCxt, yymsp[-1].minor.yy690, NULL); } break; - case 498: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ -{ yymsp[-5].minor.yy600 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy601)); } + case 501: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ +{ yymsp[-5].minor.yy476 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy376)); } break; - case 499: /* fill_mode ::= NONE */ -{ yymsp[0].minor.yy798 = FILL_MODE_NONE; } + case 502: /* fill_mode ::= NONE */ +{ yymsp[0].minor.yy690 = FILL_MODE_NONE; } break; - case 500: /* fill_mode ::= PREV */ -{ yymsp[0].minor.yy798 = FILL_MODE_PREV; } + case 503: /* fill_mode ::= PREV */ +{ yymsp[0].minor.yy690 = FILL_MODE_PREV; } break; - case 501: /* fill_mode ::= NULL */ -{ yymsp[0].minor.yy798 = FILL_MODE_NULL; } + case 504: /* fill_mode ::= NULL */ +{ yymsp[0].minor.yy690 = FILL_MODE_NULL; } break; - case 502: /* fill_mode ::= LINEAR */ -{ yymsp[0].minor.yy798 = FILL_MODE_LINEAR; } + case 505: /* fill_mode ::= LINEAR */ +{ yymsp[0].minor.yy690 = FILL_MODE_LINEAR; } break; - case 503: /* fill_mode ::= NEXT */ -{ yymsp[0].minor.yy798 = FILL_MODE_NEXT; } + case 506: /* fill_mode ::= NEXT */ +{ yymsp[0].minor.yy690 = FILL_MODE_NEXT; } break; - case 506: /* group_by_list ::= expr_or_subquery */ -{ yylhsminor.yy601 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); } - yymsp[0].minor.yy601 = yylhsminor.yy601; + case 509: /* group_by_list ::= expr_or_subquery */ +{ yylhsminor.yy376 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy476))); } + yymsp[0].minor.yy376 = yylhsminor.yy376; break; - case 507: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ -{ yylhsminor.yy601 = addNodeToList(pCxt, yymsp[-2].minor.yy601, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); } - yymsp[-2].minor.yy601 = yylhsminor.yy601; + case 510: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ +{ yylhsminor.yy376 = addNodeToList(pCxt, yymsp[-2].minor.yy376, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy476))); } + yymsp[-2].minor.yy376 = yylhsminor.yy376; break; - case 511: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ -{ yymsp[-5].minor.yy600 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); } + case 514: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ +{ yymsp[-5].minor.yy476 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy476), releaseRawExprNode(pCxt, yymsp[-1].minor.yy476)); } break; - case 514: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 517: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ { - yylhsminor.yy600 = addOrderByClause(pCxt, yymsp[-3].minor.yy600, yymsp[-2].minor.yy601); - yylhsminor.yy600 = addSlimitClause(pCxt, yylhsminor.yy600, yymsp[-1].minor.yy600); - yylhsminor.yy600 = addLimitClause(pCxt, yylhsminor.yy600, yymsp[0].minor.yy600); + yylhsminor.yy476 = addOrderByClause(pCxt, yymsp[-3].minor.yy476, yymsp[-2].minor.yy376); + yylhsminor.yy476 = addSlimitClause(pCxt, yylhsminor.yy476, yymsp[-1].minor.yy476); + yylhsminor.yy476 = addLimitClause(pCxt, yylhsminor.yy476, yymsp[0].minor.yy476); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + yymsp[-3].minor.yy476 = yylhsminor.yy476; break; - case 517: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ -{ yylhsminor.yy600 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy600, yymsp[0].minor.yy600); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + case 520: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ +{ yylhsminor.yy476 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy476, yymsp[0].minor.yy476); } + yymsp[-3].minor.yy476 = yylhsminor.yy476; break; - case 518: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ -{ yylhsminor.yy600 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy600, yymsp[0].minor.yy600); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 521: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ +{ yylhsminor.yy476 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy476, yymsp[0].minor.yy476); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 526: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 530: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==530); -{ yymsp[-1].minor.yy600 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + case 529: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 533: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==533); +{ yymsp[-1].minor.yy476 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 527: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 531: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==531); -{ yymsp[-3].minor.yy600 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 530: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 534: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==534); +{ yymsp[-3].minor.yy476 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 528: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 532: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==532); -{ yymsp[-3].minor.yy600 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 531: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 535: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==535); +{ yymsp[-3].minor.yy476 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 533: /* subquery ::= NK_LP query_expression NK_RP */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy600); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 536: /* subquery ::= NK_LP query_expression NK_RP */ +{ yylhsminor.yy476 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy476); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 538: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ -{ yylhsminor.yy600 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), yymsp[-1].minor.yy32, yymsp[0].minor.yy385); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 541: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ +{ yylhsminor.yy476 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy476), yymsp[-1].minor.yy554, yymsp[0].minor.yy697); } + yymsp[-2].minor.yy476 = yylhsminor.yy476; break; - case 539: /* ordering_specification_opt ::= */ -{ yymsp[1].minor.yy32 = ORDER_ASC; } + case 542: /* ordering_specification_opt ::= */ +{ yymsp[1].minor.yy554 = ORDER_ASC; } break; - case 540: /* ordering_specification_opt ::= ASC */ -{ yymsp[0].minor.yy32 = ORDER_ASC; } + case 543: /* ordering_specification_opt ::= ASC */ +{ yymsp[0].minor.yy554 = ORDER_ASC; } break; - case 541: /* ordering_specification_opt ::= DESC */ -{ yymsp[0].minor.yy32 = ORDER_DESC; } + case 544: /* ordering_specification_opt ::= DESC */ +{ yymsp[0].minor.yy554 = ORDER_DESC; } break; - case 542: /* null_ordering_opt ::= */ -{ yymsp[1].minor.yy385 = NULL_ORDER_DEFAULT; } + case 545: /* null_ordering_opt ::= */ +{ yymsp[1].minor.yy697 = NULL_ORDER_DEFAULT; } break; - case 543: /* null_ordering_opt ::= NULLS FIRST */ -{ yymsp[-1].minor.yy385 = NULL_ORDER_FIRST; } + case 546: /* null_ordering_opt ::= NULLS FIRST */ +{ yymsp[-1].minor.yy697 = NULL_ORDER_FIRST; } break; - case 544: /* null_ordering_opt ::= NULLS LAST */ -{ yymsp[-1].minor.yy385 = NULL_ORDER_LAST; } + case 547: /* null_ordering_opt ::= NULLS LAST */ +{ yymsp[-1].minor.yy697 = NULL_ORDER_LAST; } break; default: break; /********** End reduce actions ************************************************/ }; - assert( yyruleno* dst) const { STableMeta* src = getTableSchemaMeta(db, tbname); if (nullptr == src) { - return TSDB_CODE_TSC_INVALID_TABLE_NAME; + return TSDB_CODE_PAR_TABLE_NOT_EXIST; } int32_t len = sizeof(STableMeta) + sizeof(SSchema) * (src->tableInfo.numOfTags + src->tableInfo.numOfColumns); dst->reset((STableMeta*)taosMemoryCalloc(1, len)); diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index 8bdea07150ab3d6d521d4332183819c1144cd3b5..8286ae72e19b767fdddf6a882e370261ebfc4ccb 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -755,14 +755,23 @@ TEST_F(ParserInitialCTest, createStream) { }; auto setCreateStreamReq = [&](const char* pStream, const char* pSrcDb, const char* pSql, const char* pDstStb, - int8_t igExists = 0, int8_t triggerType = STREAM_TRIGGER_AT_ONCE, int64_t maxDelay = 0, - int64_t watermark = 0, int8_t igExpired = STREAM_DEFAULT_IGNORE_EXPIRED, - int8_t fillHistory = STREAM_DEFAULT_FILL_HISTORY) { + int8_t createStb = STREAM_CREATE_STABLE_TRUE, int8_t igExists = 0) { snprintf(expect.name, sizeof(expect.name), "0.%s", pStream); snprintf(expect.sourceDB, sizeof(expect.sourceDB), "0.%s", pSrcDb); snprintf(expect.targetStbFullName, sizeof(expect.targetStbFullName), "0.test.%s", pDstStb); expect.igExists = igExists; expect.sql = strdup(pSql); + expect.createStb = createStb; + expect.triggerType = STREAM_TRIGGER_AT_ONCE; + expect.maxDelay = 0; + expect.watermark = 0; + expect.fillHistory = STREAM_DEFAULT_FILL_HISTORY; + expect.igExpired = STREAM_DEFAULT_IGNORE_EXPIRED; + }; + + auto setStreamOptions = [&](int8_t triggerType = STREAM_TRIGGER_AT_ONCE, int64_t maxDelay = 0, int64_t watermark = 0, + int8_t igExpired = STREAM_DEFAULT_IGNORE_EXPIRED, + int8_t fillHistory = STREAM_DEFAULT_FILL_HISTORY) { expect.triggerType = triggerType; expect.maxDelay = maxDelay; expect.watermark = watermark; @@ -813,19 +822,22 @@ TEST_F(ParserInitialCTest, createStream) { ASSERT_EQ(pField->flags, pExpectField->flags); } } + ASSERT_EQ(req.checkpointFreq, expect.checkpointFreq); + ASSERT_EQ(req.createStb, expect.createStb); tFreeSCMCreateStreamReq(&req); }); - setCreateStreamReq("s1", "test", "create stream s1 into st1 as select count(*) from t1 interval(10s)", "st1"); - run("CREATE STREAM s1 INTO st1 AS SELECT COUNT(*) FROM t1 INTERVAL(10S)"); + setCreateStreamReq("s1", "test", "create stream s1 into st3 as select count(*) from t1 interval(10s)", "st3"); + run("CREATE STREAM s1 INTO st3 AS SELECT COUNT(*) FROM t1 INTERVAL(10S)"); clearCreateStreamReq(); setCreateStreamReq( "s1", "test", - "create stream if not exists s1 trigger max_delay 20s watermark 10s ignore expired 0 fill_history 1 into st1 " + "create stream if not exists s1 trigger max_delay 20s watermark 10s ignore expired 0 fill_history 1 into st3 " "as select count(*) from t1 interval(10s)", - "st1", 1, STREAM_TRIGGER_MAX_DELAY, 20 * MILLISECOND_PER_SECOND, 10 * MILLISECOND_PER_SECOND, 0, 1); - run("CREATE STREAM IF NOT EXISTS s1 TRIGGER MAX_DELAY 20s WATERMARK 10s IGNORE EXPIRED 0 FILL_HISTORY 1 INTO st1 AS " + "st3", 1, 1); + setStreamOptions(STREAM_TRIGGER_MAX_DELAY, 20 * MILLISECOND_PER_SECOND, 10 * MILLISECOND_PER_SECOND, 0, 1); + run("CREATE STREAM IF NOT EXISTS s1 TRIGGER MAX_DELAY 20s WATERMARK 10s IGNORE EXPIRED 0 FILL_HISTORY 1 INTO st3 AS " "SELECT COUNT(*) " "FROM t1 INTERVAL(10S)"); clearCreateStreamReq(); @@ -839,6 +851,11 @@ TEST_F(ParserInitialCTest, createStream) { run("CREATE STREAM s1 INTO st3 TAGS(tname VARCHAR(10), id INT) SUBTABLE(CONCAT('new-', tname)) " "AS SELECT _WSTART wstart, COUNT(*) cnt FROM st1 PARTITION BY TBNAME tname, tag1 id INTERVAL(10S)"); clearCreateStreamReq(); + + setCreateStreamReq("s1", "test", "create stream s1 into st1 as select max(c1), c2 from t1 interval(10s)", "st1", + STREAM_CREATE_STABLE_FALSE); + run("CREATE STREAM s1 INTO st1 AS SELECT MAX(c1), c2 FROM t1 INTERVAL(10S)"); + clearCreateStreamReq(); } TEST_F(ParserInitialCTest, createStreamSemanticCheck) { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index e1687fc3a5e3136fb34893c2af6994c4c4893586..83341d200af10a8df800b16c08818ccd2be681bf 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1284,18 +1284,14 @@ static int32_t smaIndexOptFindSmaFunc(SNode* pQueryFunc, SNodeList* pSmaFuncs) { return -1; } -static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, SNodeList** pOutput, - int32_t* pWStrartIndex) { +static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, + SNodeList** pOutput) { SNodeList* pCols = NULL; SNode* pFunc = NULL; int32_t code = TSDB_CODE_SUCCESS; int32_t index = 0; int32_t smaFuncIndex = -1; - *pWStrartIndex = -1; FOREACH(pFunc, pFuncs) { - if (FUNCTION_TYPE_WSTART == ((SFunctionNode*)pFunc)->funcType) { - *pWStrartIndex = index; - } smaFuncIndex = smaIndexOptFindSmaFunc(pFunc, pSmaFuncs); if (smaFuncIndex < 0) { break; @@ -1317,8 +1313,7 @@ static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNo return code; } -static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList** pCols, - int32_t* pWStrartIndex) { +static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList** pCols) { SWindowLogicNode* pWindow = (SWindowLogicNode*)pScan->node.pParent; if (!smaIndexOptEqualInterval(pScan, pWindow, pIndex)) { return TSDB_CODE_SUCCESS; @@ -1326,14 +1321,14 @@ static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo SNodeList* pSmaFuncs = NULL; int32_t code = nodesStringToList(pIndex->expr, &pSmaFuncs); if (TSDB_CODE_SUCCESS == code) { - code = smaIndexOptCreateSmaCols(pWindow->pFuncs, pIndex->dstTbUid, pSmaFuncs, pCols, pWStrartIndex); + code = smaIndexOptCreateSmaCols(pWindow->pFuncs, pIndex->dstTbUid, pSmaFuncs, pCols); } nodesDestroyList(pSmaFuncs); return code; } static int32_t smaIndexOptApplyIndex(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, - SNodeList* pSmaCols, int32_t wstrartIndex) { + SNodeList* pSmaCols) { SLogicNode* pSmaScan = NULL; int32_t code = smaIndexOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); if (TSDB_CODE_SUCCESS == code) { @@ -1350,10 +1345,9 @@ static int32_t smaIndexOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogi for (int32_t i = 0; i < nindexes; ++i) { STableIndexInfo* pIndex = taosArrayGet(pScan->pSmaIndexes, i); SNodeList* pSmaCols = NULL; - int32_t wstrartIndex = -1; - code = smaIndexOptCouldApplyIndex(pScan, pIndex, &pSmaCols, &wstrartIndex); + code = smaIndexOptCouldApplyIndex(pScan, pIndex, &pSmaCols); if (TSDB_CODE_SUCCESS == code && NULL != pSmaCols) { - code = smaIndexOptApplyIndex(pLogicSubplan, pScan, pIndex, pSmaCols, wstrartIndex); + code = smaIndexOptApplyIndex(pLogicSubplan, pScan, pIndex, pSmaCols); taosArrayDestroyEx(pScan->pSmaIndexes, smaIndexOptDestroySmaIndex); pScan->pSmaIndexes = NULL; pCxt->optimized = true; diff --git a/source/libs/planner/test/planOtherTest.cpp b/source/libs/planner/test/planOtherTest.cpp index 4741d241b5ac8b5541b6f92d08dbe1dd26151099..3a12d62340f1744db5d351d7494e2e181b512d76 100644 --- a/source/libs/planner/test/planOtherTest.cpp +++ b/source/libs/planner/test/planOtherTest.cpp @@ -30,7 +30,7 @@ TEST_F(PlanOtherTest, createTopic) { TEST_F(PlanOtherTest, createStream) { useDb("root", "test"); - run("create stream if not exists s1 trigger window_close watermark 10s into st1 as select count(*) from t1 " + run("create stream if not exists s1 trigger window_close watermark 10s into st3 as select count(*) from t1 " "interval(10s)"); run("CREATE STREAM s1 INTO st3 TAGS(tname VARCHAR(10), id INT) SUBTABLE(CONCAT('new-', tname)) " @@ -43,9 +43,9 @@ TEST_F(PlanOtherTest, createStream) { TEST_F(PlanOtherTest, createStreamUseSTable) { useDb("root", "test"); - run("CREATE STREAM IF NOT EXISTS s1 into st1 as SELECT COUNT(*) FROM st1 INTERVAL(10s)"); + run("CREATE STREAM IF NOT EXISTS s1 into st3 as SELECT COUNT(*) FROM st1 INTERVAL(10s)"); - run("CREATE STREAM IF NOT EXISTS s1 into st1 as SELECT COUNT(*) FROM st1 PARTITION BY TBNAME INTERVAL(10s)"); + run("CREATE STREAM IF NOT EXISTS s1 into st3 as SELECT COUNT(*) FROM st1 PARTITION BY TBNAME INTERVAL(10s)"); } TEST_F(PlanOtherTest, createSmaIndex) { diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index c74882cf23dac901eab09b779afd56cbfb55dcaa..d8cd5f4e751ede58190a4458eb0e348e63210b00 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -116,8 +116,6 @@ int32_t cleanupTaskQueue() { } static void execHelper(struct SSchedMsg* pSchedMsg) { - assert(pSchedMsg != NULL && pSchedMsg->ahandle != NULL); - __async_exec_fn_t execFn = (__async_exec_fn_t)pSchedMsg->ahandle; int32_t code = execFn(pSchedMsg->thandle); if (code != 0 && pSchedMsg->msg != NULL) { @@ -126,8 +124,6 @@ static void execHelper(struct SSchedMsg* pSchedMsg) { } int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) { - assert(execFn != NULL); - SSchedMsg schedMsg = {0}; schedMsg.fp = execHelper; schedMsg.ahandle = execFn; @@ -138,7 +134,10 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) } void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { - assert(pMsgBody != NULL); + if (NULL == pMsgBody) { + return; + } + taosMemoryFreeClear(pMsgBody->target.dbFName); taosMemoryFreeClear(pMsgBody->msgInfo.pData); if (pMsgBody->paramFreeFp) { @@ -243,7 +242,8 @@ void destroyQueryExecRes(SExecResult* pRes) { break; } case TDMT_VND_SUBMIT: { - tFreeSSubmitRsp((SSubmitRsp*)pRes->res); + tDestroySSubmitRsp2((SSubmitRsp2*)pRes->res, TSDB_MSG_FLG_DECODE); + taosMemoryFreeClear(pRes->res); break; } case TDMT_SCH_QUERY: @@ -393,7 +393,7 @@ char* parseTagDatatoJson(void* p) { } else if (pTagVal->nData == 0) { value = cJSON_CreateString(""); } else { - ASSERT(0); + goto end; } cJSON_AddItemToObject(json, tagJsonKey, value); @@ -412,7 +412,7 @@ char* parseTagDatatoJson(void* p) { } cJSON_AddItemToObject(json, tagJsonKey, value); } else { - ASSERT(0); + goto end; } } string = cJSON_PrintUnformatted(json); @@ -447,7 +447,6 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { return TSDB_CODE_SUCCESS; } - void freeVgInfo(SDBVgInfo* vgInfo) { if (NULL == vgInfo) { return; @@ -459,7 +458,6 @@ void freeVgInfo(SDBVgInfo* vgInfo) { taosMemoryFreeClear(vgInfo); } - int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { if (NULL == pSrc) { *pDst = NULL; @@ -498,3 +496,53 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { return TSDB_CODE_SUCCESS; } + +int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst) { + if (NULL == pSrc) { + *pDst = NULL; + return TSDB_CODE_SUCCESS; + } + + *pDst = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == *pDst) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + (*pDst)->flags = pSrc->flags; + if (pSrc->name) { + (*pDst)->name = strdup(pSrc->name); + } + (*pDst)->uid = pSrc->uid; + (*pDst)->ctime = pSrc->ctime; + (*pDst)->ttl = pSrc->ttl; + (*pDst)->commentLen = pSrc->commentLen; + if (pSrc->comment) { + (*pDst)->comment = strdup(pSrc->comment); + } + (*pDst)->type = pSrc->type; + + if (pSrc->type == TSDB_CHILD_TABLE) { + if (pSrc->ctb.stbName) { + (*pDst)->ctb.stbName = strdup(pSrc->ctb.stbName); + } + (*pDst)->ctb.tagNum = pSrc->ctb.tagNum; + (*pDst)->ctb.suid = pSrc->ctb.suid; + if (pSrc->ctb.tagName) { + (*pDst)->ctb.tagName = taosArrayDup(pSrc->ctb.tagName, NULL); + } + STag* pTag = (STag*)pSrc->ctb.pTag; + if (pTag) { + (*pDst)->ctb.pTag = taosMemoryMalloc(pTag->len); + memcpy((*pDst)->ctb.pTag, pTag, pTag->len); + } + } else { + (*pDst)->ntb.schemaRow.nCols = pSrc->ntb.schemaRow.nCols; + (*pDst)->ntb.schemaRow.version = pSrc->ntb.schemaRow.nCols; + if (pSrc->ntb.schemaRow.nCols > 0 && pSrc->ntb.schemaRow.pSchema) { + (*pDst)->ntb.schemaRow.pSchema = taosMemoryMalloc(pSrc->ntb.schemaRow.nCols * sizeof(SSchema)); + memcpy((*pDst)->ntb.schemaRow.pSchema, pSrc->ntb.schemaRow.pSchema, pSrc->ntb.schemaRow.nCols * sizeof(SSchema)); + } + } + + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index eb6091d60583ecbd07fbb202870d97dce4577d91..cad01d88c384ca34a555be90fecca6a0d37e4bae 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -316,34 +316,34 @@ typedef struct SQWorkerMgmt { #define QW_LOCK(type, _lock) \ do { \ if (QW_READ == (type)) { \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before read lock"); \ QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRLockLatch(_lock); \ QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) > 0); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value after read lock"); \ } else { \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before write lock"); \ QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWLockLatch(_lock); \ QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value after write lock"); \ } \ } while (0) #define QW_UNLOCK(type, _lock) \ do { \ if (QW_READ == (type)) { \ - assert(atomic_load_32((_lock)) > 0); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value before read unlock"); \ QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRUnLockLatch(_lock); \ QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after read unlock"); \ } else { \ - assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value before write unlock"); \ QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWUnLockLatch(_lock); \ QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after write unlock"); \ } \ } while (0) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 81f73b1226227d79a1c11cbaa1d162bd462c6bf1..e9d9a129a0d6bc31f736b5bea88e7de813f185ac 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -147,7 +147,6 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { size_t numOfResBlock = taosArrayGetSize(pResList); for (int32_t j = 0; j < numOfResBlock; ++j) { SSDataBlock *pRes = taosArrayGetP(pResList, j); - ASSERT(pRes->info.rows > 0); SInputData inputData = {.pData = pRes}; code = dsPutDataBlock(sinkHandle, &inputData, &qcontinue); diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 48df7e36a30737324342bb06bdbbfced8b73a5cd..2a5eeecd36f03f26519768c822343b8cd7323b22 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -375,7 +375,7 @@ extern SSchedulerMgmt schMgmt; #define SCH_JOB_NEED_WAIT(_job) (!SCH_IS_QUERY_JOB(_job)) #define SCH_JOB_NEED_DROP(_job) (SCH_IS_QUERY_JOB(_job)) #define SCH_IS_EXPLAIN_JOB(_job) (EXPLAIN_MODE_ANALYZE == (_job)->attr.explainMode) -#define SCH_NETWORK_ERR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) +#define SCH_NETWORK_ERR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || (_code) == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED) #define SCH_MERGE_TASK_NETWORK_ERR(_task, _code, _len) \ (SCH_NETWORK_ERR(_code) && (((_len) > 0) || (!SCH_IS_DATA_BIND_TASK(_task)) || (_task)->redirectCtx.inRedirect)) #define SCH_REDIRECT_MSGTYPE(_msgType) \ @@ -478,34 +478,34 @@ extern SSchedulerMgmt schMgmt; #define SCH_LOCK(type, _lock) \ do { \ if (SCH_READ == (type)) { \ - assert(atomic_load_32(_lock) >= 0); \ + ASSERTS(atomic_load_32(_lock) >= 0, "invalid lock value before read lock"); \ SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRLockLatch(_lock); \ SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32(_lock) > 0); \ + ASSERTS(atomic_load_32(_lock) > 0, "invalid lock value after read lock"); \ } else { \ - assert(atomic_load_32(_lock) >= 0); \ + ASSERTS(atomic_load_32(_lock) >= 0, "invalid lock value before write lock"); \ SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWLockLatch(_lock); \ SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32(_lock) == TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32(_lock) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value after write lock"); \ } \ } while (0) #define SCH_UNLOCK(type, _lock) \ do { \ if (SCH_READ == (type)) { \ - assert(atomic_load_32((_lock)) > 0); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value before read unlock"); \ SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRUnLockLatch(_lock); \ SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after read unlock"); \ } else { \ - assert(atomic_load_32((_lock)) & TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32((_lock)) & TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value before write unlock"); \ SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWUnLockLatch(_lock); \ SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after write unlock"); \ } \ } while (0) diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index d422f0e88f104749dc92804b46d38aca6f89b40b..6a8f81f8c78a0146396371d79f4c8ed95e73b1fb 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -668,6 +668,7 @@ void schFreeJobImpl(void *job) { taosMemoryFreeClear(pJob->userRes.execRes); taosMemoryFreeClear(pJob->fetchRes); taosMemoryFreeClear(pJob->sql); + tsem_destroy(&pJob->rspSem); taosMemoryFree(pJob); int32_t jobNum = atomic_sub_fetch_32(&schMgmt.jobNum, 1); @@ -748,7 +749,10 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - tsem_init(&pJob->rspSem, 0, 0); + if (tsem_init(&pJob->rspSem, 0, 0)) { + SCH_JOB_ELOG("tsem_init failed, errno:%d", errno); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } pJob->refId = taosAddRef(schMgmt.jobRef, pJob); if (pJob->refId < 0) { diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index b6de9383d7a6e59ac7453227ead12fb4ba095edc..5ceb8228b03bdd0b4811b7834b24fafe8b5e6a30 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -261,46 +261,51 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa if (msg) { SDecoder coder = {0}; - SSubmitRsp *rsp = taosMemoryMalloc(sizeof(*rsp)); + SSubmitRsp2 *rsp = taosMemoryMalloc(sizeof(*rsp)); tDecoderInit(&coder, msg, msgSize); - code = tDecodeSSubmitRsp(&coder, rsp); + code = tDecodeSSubmitRsp2(&coder, rsp); + tDecoderClear(&coder); if (code) { - SCH_TASK_ELOG("decode submitRsp failed, code:%d", code); - tFreeSSubmitRsp(rsp); + SCH_TASK_ELOG("tDecodeSSubmitRsp2 failed, code:%d", code); + tDestroySSubmitRsp2(rsp, TSDB_MSG_FLG_DECODE); + taosMemoryFree(rsp); SCH_ERR_JRET(code); } - if (rsp->nBlocks > 0) { - for (int32_t i = 0; i < rsp->nBlocks; ++i) { - SSubmitBlkRsp *blk = rsp->pBlocks + i; - if (TSDB_CODE_SUCCESS != blk->code) { - code = blk->code; - tFreeSSubmitRsp(rsp); - SCH_ERR_JRET(code); + atomic_add_fetch_64(&pJob->resNumOfRows, rsp->affectedRows); + + int32_t createTbRspNum = taosArrayGetSize(rsp->aCreateTbRsp); + SCH_TASK_DLOG("submit succeed, affectedRows:%d, createTbRspNum:%d", rsp->affectedRows, createTbRspNum); + + if (rsp->aCreateTbRsp && taosArrayGetSize(rsp->aCreateTbRsp) > 0) { + SCH_LOCK(SCH_WRITE, &pJob->resLock); + if (pJob->execRes.res) { + SSubmitRsp2 *sum = pJob->execRes.res; + sum->affectedRows += rsp->affectedRows; + if (sum->aCreateTbRsp) { + taosArrayAddAll(sum->aCreateTbRsp, rsp->aCreateTbRsp); + taosArrayDestroy(rsp->aCreateTbRsp); + } else { + TSWAP(sum->aCreateTbRsp, rsp->aCreateTbRsp); } + taosMemoryFree(rsp); + } else { + pJob->execRes.res = rsp; + pJob->execRes.msgType = TDMT_VND_SUBMIT; } - } - - atomic_add_fetch_64(&pJob->resNumOfRows, rsp->affectedRows); - SCH_TASK_DLOG("submit succeed, affectedRows:%d, blocks:%d", rsp->affectedRows, rsp->nBlocks); - - SCH_LOCK(SCH_WRITE, &pJob->resLock); - if (pJob->execRes.res) { - SSubmitRsp *sum = pJob->execRes.res; - sum->affectedRows += rsp->affectedRows; - sum->nBlocks += rsp->nBlocks; - if (rsp->nBlocks > 0 && rsp->pBlocks) { - sum->pBlocks = taosMemoryRealloc(sum->pBlocks, sum->nBlocks * sizeof(*sum->pBlocks)); - memcpy(sum->pBlocks + sum->nBlocks - rsp->nBlocks, rsp->pBlocks, rsp->nBlocks * sizeof(*sum->pBlocks)); + pJob->execRes.numOfBytes += pTask->msgLen; + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + } else { + SCH_LOCK(SCH_WRITE, &pJob->resLock); + pJob->execRes.numOfBytes += pTask->msgLen; + if (NULL == pJob->execRes.res) { + TSWAP(pJob->execRes.res, rsp); + pJob->execRes.msgType = TDMT_VND_SUBMIT; } - taosMemoryFree(rsp->pBlocks); + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + tDestroySSubmitRsp2(rsp, TSDB_MSG_FLG_DECODE); taosMemoryFree(rsp); - } else { - pJob->execRes.res = rsp; - pJob->execRes.msgType = TDMT_VND_SUBMIT; } - pJob->execRes.numOfBytes += pTask->msgLen; - SCH_UNLOCK(SCH_WRITE, &pJob->resLock); } taosMemoryFreeClear(msg); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 5b542dd54b1273dc89cec261f3f0b04bd9d35b14..60729c4d0e5a292990eae57bccbdd3cc4c783ab9 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -187,6 +187,23 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1; } +int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock) { + if (pTask->outputType == TASK_OUTPUT__TABLE) { + pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pBlock->blocks); + taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes); + taosFreeQitem(pBlock); + } else if (pTask->outputType == TASK_OUTPUT__SMA) { + pTask->smaSink.smaSink(pTask->smaSink.vnode, pTask->smaSink.smaId, pBlock->blocks); + taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes); + taosFreeQitem(pBlock); + } else { + ASSERT(pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH); + taosWriteQitem(pTask->outputQueue->queue, pBlock); + streamDispatch(pTask); + } + return 0; +} + int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp, bool exec) { qDebug("task %d receive dispatch req from node %d task %d", pTask->taskId, pReq->upstreamNodeId, pReq->upstreamTaskId); @@ -199,9 +216,9 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S return -1; } - if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { - streamDispatch(pTask); - } + /*if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {*/ + /*streamDispatch(pTask);*/ + /*}*/ } else { streamSchedExec(pTask); } @@ -237,9 +254,9 @@ int32_t streamProcessRunReq(SStreamTask* pTask) { return -1; } - if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { - streamDispatch(pTask); - } + /*if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {*/ + /*streamDispatch(pTask);*/ + /*}*/ return 0; } diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 6cc684dddf88c6d635f3299e4bd6f81ed811fb9e..3f2feb9d286a1cad87c45fe24bc0b5132b873a7f 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -66,12 +66,13 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock return 0; } -SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq) { - SStreamDataSubmit* pDataSubmit = (SStreamDataSubmit*)taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM, 0); +SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit) { + SStreamDataSubmit2* pDataSubmit = (SStreamDataSubmit2*)taosAllocateQitem(sizeof(SStreamDataSubmit2), DEF_QITEM, 0); + if (pDataSubmit == NULL) return NULL; pDataSubmit->dataRef = (int32_t*)taosMemoryMalloc(sizeof(int32_t)); if (pDataSubmit->dataRef == NULL) goto FAIL; - pDataSubmit->data = pReq; + pDataSubmit->submit = submit; *pDataSubmit->dataRef = 1; pDataSubmit->type = STREAM_INPUT__DATA_SUBMIT; return pDataSubmit; @@ -80,47 +81,49 @@ FAIL: return NULL; } -SStreamMergedSubmit* streamMergedSubmitNew() { - SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)taosAllocateQitem(sizeof(SStreamMergedSubmit), DEF_QITEM, 0); +SStreamMergedSubmit2* streamMergedSubmitNew() { + SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)taosAllocateQitem(sizeof(SStreamMergedSubmit2), DEF_QITEM, 0); + if (pMerged == NULL) return NULL; - pMerged->reqs = taosArrayInit(0, sizeof(void*)); + pMerged->submits = taosArrayInit(0, sizeof(SPackedData)); pMerged->dataRefs = taosArrayInit(0, sizeof(void*)); - if (pMerged->dataRefs == NULL || pMerged->reqs == NULL) goto FAIL; + if (pMerged->dataRefs == NULL || pMerged->submits == NULL) goto FAIL; pMerged->type = STREAM_INPUT__MERGED_SUBMIT; return pMerged; FAIL: - if (pMerged->reqs) taosArrayDestroy(pMerged->reqs); + if (pMerged->submits) taosArrayDestroy(pMerged->submits); if (pMerged->dataRefs) taosArrayDestroy(pMerged->dataRefs); taosFreeQitem(pMerged); return NULL; } -int32_t streamMergeSubmit(SStreamMergedSubmit* pMerged, SStreamDataSubmit* pSubmit) { +int32_t streamMergeSubmit(SStreamMergedSubmit2* pMerged, SStreamDataSubmit2* pSubmit) { taosArrayPush(pMerged->dataRefs, &pSubmit->dataRef); - taosArrayPush(pMerged->reqs, &pSubmit->data); + taosArrayPush(pMerged->submits, &pSubmit->submit); pMerged->ver = pSubmit->ver; return 0; } -static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit* pDataSubmit) { +static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit2* pDataSubmit) { atomic_add_fetch_32(pDataSubmit->dataRef, 1); } -SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit) { - SStreamDataSubmit* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM, 0); +SStreamDataSubmit2* streamSubmitRefClone(SStreamDataSubmit2* pSubmit) { + SStreamDataSubmit2* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit2), DEF_QITEM, 0); + if (pSubmitClone == NULL) { return NULL; } streamDataSubmitRefInc(pSubmit); - memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit)); + memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit2)); return pSubmitClone; } -void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit) { +void streamDataSubmitRefDec(SStreamDataSubmit2* pDataSubmit) { int32_t ref = atomic_sub_fetch_32(pDataSubmit->dataRef, 1); ASSERT(ref >= 0); if (ref == 0) { - taosMemoryFree(pDataSubmit->data); + taosMemoryFree(pDataSubmit->submit.msgStr); taosMemoryFree(pDataSubmit->dataRef); } } @@ -135,16 +138,16 @@ SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* taosFreeQitem(elem); return dst; } else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && elem->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)dst; - SStreamDataSubmit* pBlockSrc = (SStreamDataSubmit*)elem; + SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)dst; + SStreamDataSubmit2* pBlockSrc = (SStreamDataSubmit2*)elem; streamMergeSubmit(pMerged, pBlockSrc); taosFreeQitem(elem); return dst; } else if (dst->type == STREAM_INPUT__DATA_SUBMIT && elem->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamMergedSubmit* pMerged = streamMergedSubmitNew(); + SStreamMergedSubmit2* pMerged = streamMergedSubmitNew(); ASSERT(pMerged); - streamMergeSubmit(pMerged, (SStreamDataSubmit*)dst); - streamMergeSubmit(pMerged, (SStreamDataSubmit*)elem); + streamMergeSubmit(pMerged, (SStreamDataSubmit2*)dst); + streamMergeSubmit(pMerged, (SStreamDataSubmit2*)elem); taosFreeQitem(dst); taosFreeQitem(elem); return (SStreamQueueItem*)pMerged; @@ -162,22 +165,22 @@ void streamFreeQitem(SStreamQueueItem* data) { taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)blockDataFreeRes); taosFreeQitem(data); } else if (type == STREAM_INPUT__DATA_SUBMIT) { - streamDataSubmitRefDec((SStreamDataSubmit*)data); + streamDataSubmitRefDec((SStreamDataSubmit2*)data); taosFreeQitem(data); } else if (type == STREAM_INPUT__MERGED_SUBMIT) { - SStreamMergedSubmit* pMerge = (SStreamMergedSubmit*)data; - int32_t sz = taosArrayGetSize(pMerge->reqs); + SStreamMergedSubmit2* pMerge = (SStreamMergedSubmit2*)data; + int32_t sz = taosArrayGetSize(pMerge->submits); for (int32_t i = 0; i < sz; i++) { int32_t* pRef = taosArrayGetP(pMerge->dataRefs, i); int32_t ref = atomic_sub_fetch_32(pRef, 1); ASSERT(ref >= 0); if (ref == 0) { - void* dataStr = taosArrayGetP(pMerge->reqs, i); - taosMemoryFree(dataStr); + SPackedData* pSubmit = (SPackedData*)taosArrayGet(pMerge->submits, i); + taosMemoryFree(pSubmit->msgStr); taosMemoryFree(pRef); } } - taosArrayDestroy(pMerge->reqs); + taosArrayDestroy(pMerge->submits); taosArrayDestroy(pMerge->dataRefs); taosFreeQitem(pMerge); } else if (type == STREAM_INPUT__REF_DATA_BLOCK) { diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 4e0b0630bc1f08969ebec2cf4a2ff674cb1824ff..f2b1db19e8eb3a15f9edf04d1bbbac3a6c750e0c 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -487,8 +487,6 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat taosMemoryFree(pReqs); } return code; - } else { - ASSERT(0); } return 0; } @@ -514,7 +512,6 @@ int32_t streamDispatch(SStreamTask* pTask) { int32_t code = 0; if (streamDispatchAllBlocks(pTask, pBlock) < 0) { - ASSERT(0); code = -1; streamQueueProcessFail(pTask->outputQueue); atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index b5bceb9b94fb8e819e0774a477286fd8b14e4a24..62ba74bd23b0a3b7817376e13629b13953c6226a 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -26,17 +26,18 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); - const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)data; - qDebug("task %d %p set submit input %p %p %d 1", pTask->taskId, pTask, pSubmit, pSubmit->data, *pSubmit->dataRef); - qSetMultiStreamInput(exec, pSubmit->data, 1, STREAM_INPUT__DATA_SUBMIT); + const SStreamDataSubmit2* pSubmit = (const SStreamDataSubmit2*)data; + qDebug("task %d %p set submit input %p %p %d %" PRId64, pTask->taskId, pTask, pSubmit, pSubmit->submit.msgStr, + pSubmit->submit.msgLen, pSubmit->submit.ver); + qSetMultiStreamInput(exec, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { const SStreamDataBlock* pBlock = (const SStreamDataBlock*)data; SArray* blocks = pBlock->blocks; qDebug("task %d %p set ssdata input", pTask->taskId, pTask); qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { - const SStreamMergedSubmit* pMerged = (const SStreamMergedSubmit*)data; - SArray* blocks = pMerged->reqs; + const SStreamMergedSubmit2* pMerged = (const SStreamMergedSubmit2*)data; + SArray* blocks = pMerged->submits; qDebug("task %d %p set submit input (merged), batch num: %d", pTask->taskId, pTask, (int32_t)blocks->size); qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__MERGED_SUBMIT); } else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) { @@ -48,6 +49,10 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* // exec while (1) { + if (pTask->taskStatus == TASK_STATUS__DROPPING) { + return 0; + } + SSDataBlock* output = NULL; uint64_t ts = 0; if ((code = qExecTask(exec, &output, &ts)) < 0) { @@ -112,7 +117,11 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { ASSERT(0); } if (output == NULL) { - finished = true; + if (qStreamRecoverScanFinished(exec)) { + finished = true; + } else { + qSetStreamOpOpen(exec); + } break; } @@ -250,11 +259,11 @@ int32_t streamExecForAll(SStreamTask* pTask) { qRes->blocks = pRes; if (((SStreamQueueItem*)input)->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)input; + SStreamDataSubmit2* pSubmit = (SStreamDataSubmit2*)input; qRes->childId = pTask->selfChildId; qRes->sourceVer = pSubmit->ver; } else if (((SStreamQueueItem*)input)->type == STREAM_INPUT__MERGED_SUBMIT) { - SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)input; + SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)input; qRes->childId = pTask->selfChildId; qRes->sourceVer = pMerged->ver; } diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 6889a870d19b2ec01f2f77ab57d8e70866599053..52777fd83401454483955eababaffe3123243383 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -143,6 +143,7 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* ASSERT(left >= 0); if (left == 0) { taosArrayDestroy(pTask->checkReqIds); + pTask->checkReqIds = NULL; streamTaskLaunchRecover(pTask, version); } } else if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index cf388da92ca8bec662fb5ddfbd6ada79510bfcd2..e26c8a5c5b128d6087c7c687a17cd3c5a2fa0b3e 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -663,8 +663,7 @@ int32_t streamStateSessionClear(SStreamState* pState) { void* buf = NULL; int32_t size = 0; int32_t code = streamStateSessionGetKVByCur(pCur, &delKey, &buf, &size); - if (code == 0) { - ASSERT(size > 0); + if (code == 0 && size > 0) { memset(buf, 0, size); streamStateSessionPut(pState, &delKey, buf, size); } else { diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index f7252ed8a036152b2ae9fb46861fdb8ac6754d2d..e9aba0bc3941d501ba25a5ea28aaf02e11d42cbd 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -175,6 +175,8 @@ void tFreeSStreamTask(SStreamTask* pTask) { } if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { taosArrayDestroy(pTask->shuffleDispatcher.dbInfo.pVgroupInfos); + taosArrayDestroy(pTask->checkReqIds); + pTask->checkReqIds = NULL; } if (pTask->pState) streamStateClose(pTask->pState); diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 6af60af43d0af5c526054e1f99494d6068e817cf..b5227152dfe0012bb34446a8d486f46af0400c55 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -238,7 +238,7 @@ int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode); int32_t syncNodeStartElectTimer(SSyncNode* pSyncNode, int32_t ms); int32_t syncNodeStopElectTimer(SSyncNode* pSyncNode); int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms); -int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode); +void syncNodeResetElectTimer(SSyncNode* pSyncNode); int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode); int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode); int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode); diff --git a/source/libs/sync/inc/syncPipeline.h b/source/libs/sync/inc/syncPipeline.h index 8c7edf85ffa9db4e3ab833c9171f1f388a87fdf4..a0a0691694776f2830d4f80ec2e96ec8bf782921 100644 --- a/source/libs/sync/inc/syncPipeline.h +++ b/source/libs/sync/inc/syncPipeline.h @@ -61,7 +61,7 @@ typedef struct SSyncLogBuffer { // SSyncLogRepMgr SSyncLogReplMgr* syncLogReplMgrCreate(); void syncLogReplMgrDestroy(SSyncLogReplMgr* pMgr); -int32_t syncLogReplMgrReset(SSyncLogReplMgr* pMgr); +void syncLogReplMgrReset(SSyncLogReplMgr* pMgr); int32_t syncNodeLogReplMgrInit(SSyncNode* pNode); void syncNodeLogReplMgrDestroy(SSyncNode* pNode); @@ -109,6 +109,8 @@ SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode, int32_t syncLogBufferValidate(SSyncLogBuffer* pBuf); int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex toIndex); +int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry, + int32_t applyCode); #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 2b6e14a457b1b6a649bd7686bb7238157598314e..974a8f968e99759c6a2124226129f853c2aff3b2 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -56,7 +56,7 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI void snapshotSenderDestroy(SSyncSnapshotSender *pSender); bool snapshotSenderIsStart(SSyncSnapshotSender *pSender); int32_t snapshotSenderStart(SSyncSnapshotSender *pSender); -int32_t snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish); +void snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish); int32_t snapshotSend(SSyncSnapshotSender *pSender); int32_t snapshotReSend(SSyncSnapshotSender *pSender); @@ -79,8 +79,8 @@ typedef struct SSyncSnapshotReceiver { SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId); void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver); -int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg); -int32_t snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); +void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg); +void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index f1aa9312c627cb48a0d12610c7c42cd2f865b12a..1a481a7e1452b852c44d655a33c3fdb06b79fc1a 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -200,12 +200,15 @@ int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg) { code = syncNodeOnLocalCmd(pSyncNode, pMsg); break; default: - sError("vgId:%d, failed to process msg:%p since invalid type:%s", pSyncNode->vgId, pMsg, - TMSG_INFO(pMsg->msgType)); + terrno = TSDB_CODE_MSG_NOT_PROCESSED; code = -1; } syncNodeRelease(pSyncNode); + if (code != 0) { + sDebug("vgId:%d, failed to process sync msg:%p type:%s since 0x%x", pSyncNode->vgId, pMsg, TMSG_INFO(pMsg->msgType), + terrno); + } return code; } @@ -228,8 +231,7 @@ int32_t syncSendTimeoutRsp(int64_t rid, int64_t seq) { syncNodeRelease(pNode); if (ret == 1) { - sInfo("send timeout response, seq:%" PRId64 " handle:%p ahandle:%p", seq, rpcMsg.info.handle, - rpcMsg.info.ahandle); + sInfo("send timeout response, seq:%" PRId64 " handle:%p ahandle:%p", seq, rpcMsg.info.handle, rpcMsg.info.ahandle); rpcSendResponse(&rpcMsg); return 0; } else { @@ -1084,13 +1086,17 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { // snapshot senders for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { SSyncSnapshotSender* pSender = snapshotSenderCreate(pSyncNode, i); - // ASSERT(pSender != NULL); - (pSyncNode->senders)[i] = pSender; - sSDebug(pSender, "snapshot sender create new while open, data:%p", pSender); + if (pSender == NULL) return NULL; + + pSyncNode->senders[i] = pSender; + sSDebug(pSender, "snapshot sender create while open sync node, data:%p", pSender); } // snapshot receivers pSyncNode->pNewNodeReceiver = snapshotReceiverCreate(pSyncNode, EMPTY_RAFT_ID); + if (pSyncNode->pNewNodeReceiver == NULL) return NULL; + sRDebug(pSyncNode->pNewNodeReceiver, "snapshot receiver create while open sync node, data:%p", + pSyncNode->pNewNodeReceiver); // is config changing pSyncNode->changing = false; @@ -1131,10 +1137,8 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->hbrSlowNum = 0; pSyncNode->tmrRoutineNum = 0; - sNInfo(pSyncNode, "sync open, node:%p", pSyncNode); - sTrace("vgId:%d, tsElectInterval:%d, tsHeartbeatInterval:%d, tsHeartbeatTimeout:%d", pSyncNode->vgId, tsElectInterval, - tsHeartbeatInterval, tsHeartbeatTimeout); - + sNInfo(pSyncNode, "sync open, node:%p electInterval:%d heartbeatInterval:%d heartbeatTimeout:%d", pSyncNode, + tsElectInterval, tsHeartbeatInterval, tsHeartbeatTimeout); return pSyncNode; _error: @@ -1251,6 +1255,8 @@ void syncNodePreClose(SSyncNode* pSyncNode) { snapshotReceiverForceStop(pSyncNode->pNewNodeReceiver); } + sDebug("vgId:%d, snapshot receiver destroy while preclose sync node, data:%p", pSyncNode->vgId, + pSyncNode->pNewNodeReceiver); snapshotReceiverDestroy(pSyncNode->pNewNodeReceiver); pSyncNode->pNewNodeReceiver = NULL; } @@ -1295,15 +1301,15 @@ void syncNodeClose(SSyncNode* pSyncNode) { syncNodeStopHeartbeatTimer(pSyncNode); for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { - if ((pSyncNode->senders)[i] != NULL) { - sSTrace((pSyncNode->senders)[i], "snapshot sender destroy while close, data:%p", (pSyncNode->senders)[i]); + if (pSyncNode->senders[i] != NULL) { + sDebug("vgId:%d, snapshot sender destroy while close, data:%p", pSyncNode->vgId, pSyncNode->senders[i]); - if (snapshotSenderIsStart((pSyncNode->senders)[i])) { - snapshotSenderStop((pSyncNode->senders)[i], false); + if (snapshotSenderIsStart(pSyncNode->senders[i])) { + snapshotSenderStop(pSyncNode->senders[i], false); } - snapshotSenderDestroy((pSyncNode->senders)[i]); - (pSyncNode->senders)[i] = NULL; + snapshotSenderDestroy(pSyncNode->senders[i]); + pSyncNode->senders[i] = NULL; } } @@ -1312,6 +1318,7 @@ void syncNodeClose(SSyncNode* pSyncNode) { snapshotReceiverForceStop(pSyncNode->pNewNodeReceiver); } + sDebug("vgId:%d, snapshot receiver destroy while close, data:%p", pSyncNode->vgId, pSyncNode->pNewNodeReceiver); snapshotReceiverDestroy(pSyncNode->pNewNodeReceiver); pSyncNode->pNewNodeReceiver = NULL; } @@ -1382,8 +1389,7 @@ int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms) { return ret; } -int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode) { - int32_t ret = 0; +void syncNodeResetElectTimer(SSyncNode* pSyncNode) { int32_t electMS; if (pSyncNode->pRaftCfg->isStandBy) { @@ -1391,11 +1397,11 @@ int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode) { } else { electMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine); } - ret = syncNodeRestartElectTimer(pSyncNode, electMS); + + (void)syncNodeRestartElectTimer(pSyncNode, electMS); sNTrace(pSyncNode, "reset elect timer, min:%d, max:%d, ms:%d", pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine, electMS); - return ret; } static int32_t syncNodeDoStartHeartbeatTimer(SSyncNode* pSyncNode) { @@ -1455,23 +1461,20 @@ int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode) { return 0; } -// utils -------------- int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { SEpSet epSet; syncUtilRaftId2EpSet(destRaftId, &epSet); + if (pSyncNode->syncSendMSg != NULL) { - // htonl syncUtilMsgHtoN(pMsg->pCont); - pMsg->info.noResp = 1; - pSyncNode->syncSendMSg(&epSet, pMsg); + return pSyncNode->syncSendMSg(&epSet, pMsg); } else { sError("vgId:%d, sync send msg by id error, fp-send-msg is null", pSyncNode->vgId); rpcFreeCont(pMsg->pCont); + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; return -1; } - - return 0; } int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg) { @@ -1586,7 +1589,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde memcpy(oldReplicasId, pSyncNode->replicasId, sizeof(oldReplicasId)); SSyncSnapshotSender* oldSenders[TSDB_MAX_REPLICA]; for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { - oldSenders[i] = (pSyncNode->senders)[i]; + oldSenders[i] = pSyncNode->senders[i]; sSTrace(oldSenders[i], "snapshot sender save old"); } @@ -1625,7 +1628,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // clear new for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { - (pSyncNode->senders)[i] = NULL; + pSyncNode->senders[i] = NULL; } // reset new @@ -1640,16 +1643,16 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde sNTrace(pSyncNode, "snapshot sender reset for: %" PRId64 ", newIndex:%d, %s:%d, %p", (pSyncNode->replicasId)[i].addr, i, host, port, oldSenders[j]); - (pSyncNode->senders)[i] = oldSenders[j]; + pSyncNode->senders[i] = oldSenders[j]; oldSenders[j] = NULL; reset = true; // reset replicaIndex - int32_t oldreplicaIndex = (pSyncNode->senders)[i]->replicaIndex; - (pSyncNode->senders)[i]->replicaIndex = i; + int32_t oldreplicaIndex = pSyncNode->senders[i]->replicaIndex; + pSyncNode->senders[i]->replicaIndex = i; sNTrace(pSyncNode, "snapshot sender udpate replicaIndex from %d to %d, %s:%d, %p, reset:%d", oldreplicaIndex, - i, host, port, (pSyncNode->senders)[i], reset); + i, host, port, pSyncNode->senders[i], reset); break; } @@ -1658,18 +1661,23 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // create new for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { - if ((pSyncNode->senders)[i] == NULL) { - (pSyncNode->senders)[i] = snapshotSenderCreate(pSyncNode, i); - sSTrace((pSyncNode->senders)[i], "snapshot sender create new while reconfig, data:%p", (pSyncNode->senders)[i]); + if (pSyncNode->senders[i] == NULL) { + pSyncNode->senders[i] = snapshotSenderCreate(pSyncNode, i); + if (pSyncNode->senders[i] == NULL) { + // will be created later while send snapshot + sSError(pSyncNode->senders[i], "snapshot sender create failed while reconfig"); + } else { + sSDebug(pSyncNode->senders[i], "snapshot sender create while reconfig, data:%p", pSyncNode->senders[i]); + } } else { - sSTrace((pSyncNode->senders)[i], "snapshot sender already exist, data:%p", (pSyncNode->senders)[i]); + sSDebug(pSyncNode->senders[i], "snapshot sender already exist, data:%p", pSyncNode->senders[i]); } } // free old for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { if (oldSenders[i] != NULL) { - sNTrace(pSyncNode, "snapshot sender destroy old, data:%p replica-index:%d", oldSenders[i], i); + sSDebug(oldSenders[i], "snapshot sender destroy old, data:%p replica-index:%d", oldSenders[i], i); snapshotSenderDestroy(oldSenders[i]); oldSenders[i] = NULL; } @@ -1844,8 +1852,8 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { SSyncSnapshotSender* pMySender = syncNodeGetSnapshotSender(pSyncNode, &(pSyncNode->myRaftId)); if (pMySender != NULL) { for (int32_t i = 0; i < pSyncNode->pMatchIndex->replicaNum; ++i) { - if ((pSyncNode->senders)[i]->privateTerm > pMySender->privateTerm) { - pMySender->privateTerm = (pSyncNode->senders)[i]->privateTerm; + if (pSyncNode->senders[i]->privateTerm > pMySender->privateTerm) { + pMySender->privateTerm = pSyncNode->senders[i]->privateTerm; } } (pMySender->privateTerm) += 100; @@ -2376,9 +2384,20 @@ int32_t syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHand } int32_t syncNodeAppend(SSyncNode* ths, SSyncRaftEntry* pEntry) { + if (pEntry->dataLen < sizeof(SMsgHead)) { + sError("vgId:%d, cannot append an invalid client request with no msg head. type:%s, dataLen:%d", ths->vgId, + TMSG_INFO(pEntry->originalRpcType), pEntry->dataLen); + syncEntryDestroy(pEntry); + return -1; + } + // append to log buffer if (syncLogBufferAppend(ths->pLogBuf, ths, pEntry) < 0) { - sError("vgId:%d, failed to enqueue sync log buffer. index:%" PRId64 "", ths->vgId, pEntry->index); + sError("vgId:%d, failed to enqueue sync log buffer, index:%" PRId64, ths->vgId, pEntry->index); + terrno = TSDB_CODE_SYN_BUFFER_FULL; + (void)syncLogFsmExecute(ths, ths->pFsm, ths->state, ths->pRaftStore->currentTerm, pEntry, + TSDB_CODE_SYN_BUFFER_FULL); + syncEntryDestroy(pEntry); return -1; } @@ -2671,16 +2690,24 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIn pEntry = syncEntryBuildFromRpcMsg(pMsg, term, index); } + if (pEntry == NULL) { + sError("vgId:%d, failed to process client request since %s.", ths->vgId, terrstr()); + return -1; + } + if (ths->state == TAOS_SYNC_STATE_LEADER) { if (pRetIndex) { (*pRetIndex) = index; } int32_t code = syncNodeAppend(ths, pEntry); - if (code < 0 && ths->vgId != 1 && vnodeIsMsgBlock(pEntry->originalRpcType)) { - ASSERTS(false, "failed to append blocking msg"); + if (code < 0) { + sNError(ths, "failed to append blocking msg"); } return code; + } else { + syncEntryDestroy(pEntry); + pEntry = NULL; } return -1; diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index f438856ace8091fad89f7e75c754d1c50e6089c4..ee649c268c67d6022ed75bbd81fb10a3994b9feb 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -26,6 +26,11 @@ #include "syncSnapshot.h" #include "syncUtil.h" +static bool syncIsMsgBlock(tmsg_t type) { + return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || + (type == TDMT_VND_UPDATE_TAG_VAL) || (type == TDMT_VND_ALTER_CONFIRM); +} + int64_t syncLogBufferGetEndIndex(SSyncLogBuffer* pBuf) { taosThreadMutexLock(&pBuf->mutex); int64_t index = pBuf->endIndex; @@ -40,7 +45,7 @@ int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt if (index - pBuf->startIndex >= pBuf->size) { sError("vgId:%d, failed to append due to sync log buffer full. index:%" PRId64 "", pNode->vgId, index); - goto _out; + goto _err; } ASSERT(index == pBuf->endIndex); @@ -61,9 +66,8 @@ int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt taosThreadMutexUnlock(&pBuf->mutex); return 0; -_out: +_err: syncLogBufferValidate(pBuf); - syncEntryDestroy(pEntry); taosThreadMutexUnlock(&pBuf->mutex); return -1; } @@ -112,7 +116,7 @@ SyncTerm syncLogReplMgrGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, S return prevLogTerm; } - sError("vgId:%d, failed to get log term since %s. index: %" PRId64 "", pNode->vgId, terrstr(), prevIndex); + sInfo("vgId:%d, failed to get log term since %s. index:%" PRId64, pNode->vgId, terrstr(), prevIndex); terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; return -1; } @@ -441,26 +445,25 @@ _out: return matchIndex; } -int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry) { - ASSERTS(pFsm->FpCommitCb != NULL, "No commit cb registered for the FSM"); - +int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry, + int32_t applyCode) { if ((pNode->replicaNum == 1) && pNode->restoreFinish && pNode->vgId != 1) { return 0; } - if (pNode->vgId != 1 && vnodeIsMsgBlock(pEntry->originalRpcType)) { - sTrace("vgId:%d, blocking msg ready to execute. index:%" PRId64 ", term: %" PRId64 ", type: %s", pNode->vgId, - pEntry->index, pEntry->term, TMSG_INFO(pEntry->originalRpcType)); + if (pNode->vgId != 1 && syncIsMsgBlock(pEntry->originalRpcType)) { + sTrace("vgId:%d, blocking msg ready to execute, index:%" PRId64 ", term:%" PRId64 ", type:%s code:0x%x", + pNode->vgId, pEntry->index, pEntry->term, TMSG_INFO(pEntry->originalRpcType), applyCode); } - SRpcMsg rpcMsg = {0}; + SRpcMsg rpcMsg = {.code = applyCode}; syncEntry2OriginalRpc(pEntry, &rpcMsg); SFsmCbMeta cbMeta = {0}; cbMeta.index = pEntry->index; cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(pNode, pEntry->index); cbMeta.isWeak = pEntry->isWeak; - cbMeta.code = 0; + cbMeta.code = applyCode; cbMeta.state = role; cbMeta.seqNum = pEntry->seqNum; cbMeta.term = pEntry->term; @@ -469,7 +472,6 @@ int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, Syn (void)syncRespMgrGetAndDel(pNode->pSyncRespMgr, cbMeta.seqNum, &rpcMsg.info); int32_t code = pFsm->FpCommitCb(pFsm, &rpcMsg, &cbMeta); - ASSERT(rpcMsg.pCont == NULL); return code; } @@ -520,7 +522,7 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm pEntry->term, TMSG_INFO(pEntry->originalRpcType)); } - if (syncLogFsmExecute(pNode, pFsm, role, term, pEntry) != 0) { + if (syncLogFsmExecute(pNode, pFsm, role, term, pEntry, 0) != 0) { sError("vgId:%d, failed to execute sync log entry. index:%" PRId64 ", term:%" PRId64 ", role: %d, current term: %" PRId64, vgId, pEntry->index, pEntry->term, role, term); @@ -566,7 +568,9 @@ _out: return ret; } -int32_t syncLogReplMgrReset(SSyncLogReplMgr* pMgr) { +void syncLogReplMgrReset(SSyncLogReplMgr* pMgr) { + if (pMgr == NULL) return; + ASSERT(pMgr->startIndex >= 0); for (SyncIndex index = pMgr->startIndex; index < pMgr->endIndex; index++) { memset(&pMgr->states[index % pMgr->size], 0, sizeof(pMgr->states[0])); @@ -576,7 +580,6 @@ int32_t syncLogReplMgrReset(SSyncLogReplMgr* pMgr) { pMgr->endIndex = 0; pMgr->restored = false; pMgr->retryBackoff = 0; - return 0; } int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 018ac5bb7da3764d2b54b6dfc7995ffe9d9a47d8..3f9f397ef5726b2d29449080529dff6a4d9c2ddf 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -115,8 +115,8 @@ static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncI const char* sysErrStr = strerror(errno); sNError(pData->pSyncNode, - "wal restore from snapshot error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", - snapshotIndex, err, err, errStr, sysErr, sysErrStr); + "wal restore from snapshot error, index:%" PRId64 ", err:0x%x, msg:%s, syserr:%d, sysmsg:%s", snapshotIndex, + err, errStr, sysErr, sysErrStr); return -1; } @@ -212,8 +212,8 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr int32_t sysErr = errno; const char* sysErrStr = strerror(errno); - sNError(pData->pSyncNode, "wal write error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", - pEntry->index, err, err, errStr, sysErr, sysErrStr); + sNError(pData->pSyncNode, "wal write error, index:%" PRId64 ", err:0x%x, msg:%s, syserr:%d, sysmsg:%s", + pEntry->index, err, errStr, sysErr, sysErrStr); return -1; } @@ -257,11 +257,11 @@ int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncR const char* sysErrStr = strerror(errno); if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { - sNTrace(pData->pSyncNode, "wal read not exist, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", index, - err, err, errStr, sysErr, sysErrStr); + sNTrace(pData->pSyncNode, "wal read not exist, index:%" PRId64 ", err:0x%x, msg:%s, syserr:%d, sysmsg:%s", index, + err, errStr, sysErr, sysErrStr); } else { - sNTrace(pData->pSyncNode, "wal read error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", index, - err, err, errStr, sysErr, sysErrStr); + sNTrace(pData->pSyncNode, "wal read error, index:%" PRId64 ", err:0x%x, msg:%s, syserr:%d, sysmsg:%s", index, err, + errStr, sysErr, sysErrStr); } /* @@ -341,8 +341,8 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn const char* errStr = tstrerror(err); int32_t sysErr = errno; const char* sysErrStr = strerror(errno); - sError("vgId:%d, wal truncate error, from-index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", - pData->pSyncNode->vgId, fromIndex, err, err, errStr, sysErr, sysErrStr); + sError("vgId:%d, wal truncate error, from-index:%" PRId64 ", err:0x%x, msg:%s, syserr:%d, sysmsg:%s", + pData->pSyncNode->vgId, fromIndex, err, errStr, sysErr, sysErrStr); } // event log @@ -392,8 +392,8 @@ int32_t raftLogUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { const char* errStr = tstrerror(err); int32_t sysErr = errno; const char* sysErrStr = strerror(errno); - sError("vgId:%d, wal update commit index error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", - pData->pSyncNode->vgId, index, err, err, errStr, sysErr, sysErrStr); + sError("vgId:%d, wal update commit index error, index:%" PRId64 ", err:0x%x, msg:%s, syserr:%d, sysmsg:%s", + pData->pSyncNode->vgId, index, err, errStr, sysErr, sysErrStr); return -1; } return 0; diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 30324c1113a8434604a4a97b3b0e87c6460125e6..f2da2fb2ce527761c8f562dbf7b7611b428f6c3d 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -49,13 +49,11 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI pSender->pSyncNode->pFsm->FpGetSnapshotInfo(pSender->pSyncNode->pFsm, &pSender->snapshot); pSender->finish = false; - sDebug("vgId:%d, snapshot sender create", pSender->pSyncNode->vgId); return pSender; } void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { if (pSender == NULL) return; - sDebug("vgId:%d, snapshot sender destroy", pSender->pSyncNode->vgId); // free current block if (pSender->pCurrentBlock != NULL) { @@ -76,12 +74,6 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { bool snapshotSenderIsStart(SSyncSnapshotSender *pSender) { return pSender->start; } int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { - if (snapshotSenderIsStart(pSender)) { - sSError(pSender, "vgId:%d, snapshot sender is already start"); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - return -1; - } - pSender->start = true; pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; @@ -96,7 +88,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { pSender->snapshot.lastApplyTerm = SYNC_TERM_INVALID; pSender->snapshot.lastConfigIndex = SYNC_INDEX_INVALID; - memset(&(pSender->lastConfig), 0, sizeof(pSender->lastConfig)); + memset(&pSender->lastConfig, 0, sizeof(pSender->lastConfig)); pSender->sendingMS = 0; pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; pSender->startTime = taosGetTimestampMs(); @@ -112,7 +104,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { SyncSnapshotSend *pMsg = rpcMsg.pCont; pMsg->srcId = pSender->pSyncNode->myRaftId; - pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; + pMsg->destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; pMsg->beginIndex = pSender->snapshotParam.start; pMsg->lastIndex = pSender->snapshot.lastApplyIndex; @@ -123,7 +115,6 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { pMsg->seq = SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT; // event log - sSDebug(pSender, "snapshot sender start"); syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, "snapshot sender start"); // send msg @@ -135,7 +126,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { return 0; } -int32_t snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) { +void snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) { sSDebug(pSender, "snapshot sender stop, finish:%d reader:%p", finish, pSender->pReader); // update flag @@ -155,8 +146,6 @@ int32_t snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) { pSender->pCurrentBlock = NULL; pSender->blockLen = 0; } - - return 0; } // when sender receive ack, call this function to send msg from seq @@ -178,8 +167,8 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { } if (pSender->blockLen > 0) { - sSDebug(pSender, "snapshot sender continue to read, blockLen:%d seq:%d", pSender->blockLen, pSender->seq); // has read data + sSDebug(pSender, "snapshot sender continue to read, blockLen:%d seq:%d", pSender->blockLen, pSender->seq); } else { // read finish, update seq to end pSender->seq = SYNC_SNAPSHOT_SEQ_END; @@ -195,7 +184,7 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { SyncSnapshotSend *pMsg = rpcMsg.pCont; pMsg->srcId = pSender->pSyncNode->myRaftId; - pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; + pMsg->destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; pMsg->beginIndex = pSender->snapshotParam.start; pMsg->lastIndex = pSender->snapshot.lastApplyIndex; @@ -203,7 +192,6 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex; pMsg->lastConfig = pSender->lastConfig; pMsg->seq = pSender->seq; - // pMsg->privateTerm = pSender->privateTerm; if (pSender->pCurrentBlock != NULL) { memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); @@ -211,10 +199,8 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { // event log if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) { - sSDebug(pSender, "snapshot sender finish, seq:%d", pSender->seq); syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, "snapshot sender finish"); } else { - sSDebug(pSender, "snapshot sender sending, seq:%d", pSender->seq); syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, "snapshot sender sending"); } @@ -239,7 +225,7 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { SyncSnapshotSend *pMsg = rpcMsg.pCont; pMsg->srcId = pSender->pSyncNode->myRaftId; - pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; + pMsg->destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; pMsg->beginIndex = pSender->snapshotParam.start; pMsg->lastIndex = pSender->snapshot.lastApplyIndex; @@ -249,12 +235,10 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { pMsg->seq = pSender->seq; if (pSender->pCurrentBlock != NULL && pSender->blockLen > 0) { - // pMsg->privateTerm = pSender->privateTerm; memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); } // event log - sSDebug(pSender, "snapshot sender resend, seq:%d", pSender->seq); syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, "snapshot sender resend"); // send msg @@ -294,19 +278,16 @@ int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) { } if (snapshotSenderIsStart(pSender)) { - sSError(pSender, "snapshot sender already start, ignore"); + sSInfo(pSender, "snapshot sender already start, ignore"); return 0; } if (pSender->finish && taosGetTimestampMs() - pSender->endTime < SNAPSHOT_WAIT_MS) { sSInfo(pSender, "snapshot sender start too frequently, ignore"); - return 1; + return 0; } - char host[64]; - uint16_t port; - syncUtilU642Addr(pDestId->addr, host, sizeof(host), &port); - sSInfo(pSender, "snapshot sender start for peer:%s:%u", host, port); + sSInfo(pSender, "snapshot sender start"); int32_t code = snapshotSenderStart(pSender); if (code != 0) { @@ -339,13 +320,11 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId from pReceiver->snapshot.lastApplyTerm = 0; pReceiver->snapshot.lastConfigIndex = SYNC_INDEX_INVALID; - sDebug("vgId:%d, snapshot receiver create", pSyncNode->vgId); return pReceiver; } void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) { if (pReceiver == NULL) return; - sDebug("vgId:%d, snapshot receiver destroy", pReceiver->pSyncNode->vgId); // close writer if (pReceiver->pWriter != NULL) { @@ -369,7 +348,6 @@ void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) { // force close, abandon incomplete data if (pReceiver->pWriter != NULL) { - // event log int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false, &pReceiver->snapshot); if (ret != 0) { @@ -381,13 +359,7 @@ void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) { pReceiver->start = false; } -int32_t snapshotReceiverStartWriter(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg) { - if (!snapshotReceiverIsStart(pReceiver)) { - sRError(pReceiver, "snapshot receiver is not start"); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - return -1; - } - +static int32_t snapshotReceiverStartWriter(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg) { if (pReceiver->pWriter != NULL) { sRError(pReceiver, "vgId:%d, snapshot receiver writer is not null"); terrno = TSDB_CODE_SYN_INTERNAL_ERROR; @@ -417,10 +389,10 @@ int32_t snapshotReceiverStartWriter(SSyncSnapshotReceiver *pReceiver, SyncSnapsh return 0; } -int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pPreMsg) { +void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pPreMsg) { if (snapshotReceiverIsStart(pReceiver)) { sRInfo(pReceiver, "snapshot receiver has started"); - return 0; + return; } pReceiver->start = true; @@ -431,12 +403,11 @@ int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend // event log sRInfo(pReceiver, "snapshot receiver is start"); - return 0; } // just set start = false // FpSnapshotStopWrite should not be called, assert writer == NULL -int32_t snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { +void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { sRInfo(pReceiver, "snapshot receiver stop, not apply, writer:%p", pReceiver->pWriter); if (pReceiver->pWriter != NULL) { @@ -451,17 +422,10 @@ int32_t snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { } pReceiver->start = false; - return 0; } // when recv last snapshot block, apply data into snapshot static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) { - if (pMsg->seq != SYNC_SNAPSHOT_SEQ_END) { - sRError(pReceiver, "snapshot receiver seq:%d is invalid", pMsg->seq); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - return -1; - } - int32_t code = 0; if (pReceiver->pWriter != NULL) { // write data @@ -523,7 +487,7 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap static int32_t snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) { if (pMsg->seq != pReceiver->ack + 1) { sRError(pReceiver, "snapshot receiver invalid seq, ack:%d seq:%d", pReceiver->ack, pMsg->seq); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + terrno = TSDB_CODE_SYN_INVALID_SNAPSHOT_MSG; return -1; } @@ -583,6 +547,7 @@ SyncIndex syncNodeGetSnapBeginIndex(SSyncNode *ths) { static int32_t syncNodeOnSnapshotPre(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; int64_t timeNow = taosGetTimestampMs(); + int32_t code = 0; if (snapshotReceiverIsStart(pReceiver)) { // already start @@ -594,14 +559,14 @@ static int32_t syncNodeOnSnapshotPre(SSyncNode *pSyncNode, SyncSnapshotSend *pMs sRInfo(pReceiver, "snapshot receiver startTime:%" PRId64 " == msg startTime:%" PRId64 " send reply", pReceiver->startTime, pMsg->startTime); goto _SEND_REPLY; - } else { // ignore - sRInfo(pReceiver, "snapshot receiver startTime:%" PRId64 " < msg startTime:%" PRId64 " ignore", - pReceiver->startTime, pMsg->startTime); - return 0; + sRError(pReceiver, "snapshot receiver startTime:%" PRId64 " < msg startTime:%" PRId64 " ignore", + pReceiver->startTime, pMsg->startTime); + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + code = terrno; + goto _SEND_REPLY; } - } else { // start new sRInfo(pReceiver, "snapshot receiver not start yet so start new one"); @@ -612,7 +577,8 @@ _START_RECEIVER: if (timeNow - pMsg->startTime > SNAPSHOT_MAX_CLOCK_SKEW_MS) { sRError(pReceiver, "snapshot receiver time skew too much, now:%" PRId64 " msg startTime:%" PRId64, timeNow, pMsg->startTime); - return -1; + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + code = terrno; } else { // waiting for clock match while (timeNow < pMsg->startTime) { @@ -648,7 +614,7 @@ _SEND_REPLY: pRspMsg->lastTerm = pMsg->lastTerm; pRspMsg->startTime = pReceiver->startTime; pRspMsg->ack = pMsg->seq; // receiver maybe already closed - pRspMsg->code = 0; + pRspMsg->code = code; pRspMsg->snapBeginIndex = syncNodeGetSnapBeginIndex(pSyncNode); // send msg @@ -658,26 +624,36 @@ _SEND_REPLY: return -1; } - return 0; + return code; } static int32_t syncNodeOnSnapshotBegin(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // condition 1 SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; + int32_t code = TSDB_CODE_SYN_INTERNAL_ERROR; if (!snapshotReceiverIsStart(pReceiver)) { - sRError(pReceiver, "snapshot receiver not start"); - return -1; + sRError(pReceiver, "snapshot receiver begin failed since not start"); + goto _SEND_REPLY; } if (pReceiver->startTime != pMsg->startTime) { - sRError(pReceiver, "snapshot receiver startTime:%" PRId64 " not equal to msg startTime:%" PRId64, + sRError(pReceiver, "snapshot receiver begin failed since startTime:%" PRId64 " not equal to msg startTime:%" PRId64, pReceiver->startTime, pMsg->startTime); - return -1; + goto _SEND_REPLY; } // start writer - snapshotReceiverStartWriter(pReceiver, pMsg); + if (snapshotReceiverStartWriter(pReceiver, pMsg) != 0) { + sRError(pReceiver, "snapshot receiver begin failed since start writer failed"); + goto _SEND_REPLY; + } + + code = 0; +_SEND_REPLY: + if (code != 0 && terrno != 0) { + code = terrno; + } // build msg SRpcMsg rpcMsg = {0}; @@ -694,7 +670,7 @@ static int32_t syncNodeOnSnapshotBegin(SSyncNode *pSyncNode, SyncSnapshotSend *p pRspMsg->lastTerm = pMsg->lastTerm; pRspMsg->startTime = pReceiver->startTime; pRspMsg->ack = pReceiver->ack; // receiver maybe already closed - pRspMsg->code = 0; + pRspMsg->code = code; pRspMsg->snapBeginIndex = pReceiver->snapshotParam.start; // send msg @@ -704,10 +680,10 @@ static int32_t syncNodeOnSnapshotBegin(SSyncNode *pSyncNode, SyncSnapshotSend *p return -1; } - return 0; + return code; } -static int32_t syncNodeOnSnapshotTransfering(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { +static int32_t syncNodeOnSnapshotReceive(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // condition 4 // transfering SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; @@ -721,8 +697,12 @@ static int32_t syncNodeOnSnapshotTransfering(SSyncNode *pSyncNode, SyncSnapshotS timeNow = taosGetTimestampMs(); } + int32_t code = 0; if (snapshotReceiverGotData(pReceiver, pMsg) != 0) { - return -1; + code = terrno; + if (code >= SYNC_SNAPSHOT_SEQ_INVALID) { + code = TSDB_CODE_SYN_INTERNAL_ERROR; + } } // build msg @@ -740,17 +720,17 @@ static int32_t syncNodeOnSnapshotTransfering(SSyncNode *pSyncNode, SyncSnapshotS pRspMsg->lastTerm = pMsg->lastTerm; pRspMsg->startTime = pReceiver->startTime; pRspMsg->ack = pReceiver->ack; // receiver maybe already closed - pRspMsg->code = 0; + pRspMsg->code = code; pRspMsg->snapBeginIndex = pReceiver->snapshotParam.start; // send msg - syncLogSendSyncSnapshotRsp(pSyncNode, pRspMsg, "snapshot receiver receiving"); + syncLogSendSyncSnapshotRsp(pSyncNode, pRspMsg, "snapshot receiver received"); if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) { sRError(pReceiver, "snapshot receiver send resp failed since %s", terrstr()); return -1; } - return 0; + return code; } static int32_t syncNodeOnSnapshotEnd(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { @@ -787,7 +767,7 @@ static int32_t syncNodeOnSnapshotEnd(SSyncNode *pSyncNode, SyncSnapshotSend *pMs pRspMsg->lastTerm = pMsg->lastTerm; pRspMsg->startTime = pReceiver->startTime; pRspMsg->ack = pReceiver->ack; // receiver maybe already closed - pRspMsg->code = 0; + pRspMsg->code = code; pRspMsg->snapBeginIndex = pReceiver->snapshotParam.start; // send msg @@ -797,7 +777,7 @@ static int32_t syncNodeOnSnapshotEnd(SSyncNode *pSyncNode, SyncSnapshotSend *pMs return -1; } - return 0; + return code; } // receiver on message @@ -827,12 +807,14 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { // if already drop replica, do not process if (!syncNodeInRaftGroup(pSyncNode, &pMsg->srcId)) { syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "not in my config"); - return 0; + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + return -1; } if (pMsg->term < pSyncNode->pRaftStore->currentTerm) { syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "reject since small term"); - return 0; + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + return -1; } if (pMsg->term > pSyncNode->pRaftStore->currentTerm) { @@ -841,56 +823,49 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { syncNodeResetElectTimer(pSyncNode); // state, term, seq/ack + int32_t code = 0; if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { if (pMsg->term == pSyncNode->pRaftStore->currentTerm) { if (pMsg->seq == SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT) { syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq pre-snapshot"); - syncNodeOnSnapshotPre(pSyncNode, pMsg); + code = syncNodeOnSnapshotPre(pSyncNode, pMsg); } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) { syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq begin"); - syncNodeOnSnapshotBegin(pSyncNode, pMsg); + code = syncNodeOnSnapshotBegin(pSyncNode, pMsg); } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq end"); - syncNodeOnSnapshotEnd(pSyncNode, pMsg); + code = syncNodeOnSnapshotEnd(pSyncNode, pMsg); if (syncLogBufferReInit(pSyncNode->pLogBuf, pSyncNode) != 0) { sRError(pReceiver, "failed to reinit log buffer since %s", terrstr()); - return -1; + code = -1; } } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { // force close, no response syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process force stop"); snapshotReceiverForceStop(pReceiver); } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { - syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq"); - syncNodeOnSnapshotTransfering(pSyncNode, pMsg); + syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq data"); + code = syncNodeOnSnapshotReceive(pSyncNode, pMsg); } else { // error log sRError(pReceiver, "snapshot receiver recv error seq:%d, my ack:%d", pMsg->seq, pReceiver->ack); - return -1; + code = -1; } } else { // error log sRError(pReceiver, "snapshot receiver term not equal"); - return -1; + code = -1; } } else { // error log sRError(pReceiver, "snapshot receiver not follower"); - return -1; + code = -1; } - return 0; + return code; } -int32_t syncNodeOnSnapshotReplyPre(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { - // get sender - SSyncSnapshotSender *pSender = syncNodeGetSnapshotSender(pSyncNode, &(pMsg->srcId)); - if (pSender == NULL) { - sNError(pSyncNode, "prepare snapshot error since sender is null"); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - return -1; - } - +static int32_t syncNodeOnSnapshotPreRsp(SSyncNode *pSyncNode, SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) { SSnapshot snapshot = {0}; pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); @@ -912,7 +887,7 @@ int32_t syncNodeOnSnapshotReplyPre(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) pSender->snapshot = snapshot; // start reader - int32_t code = pSyncNode->pFsm->FpSnapshotStartRead(pSyncNode->pFsm, &(pSender->snapshotParam), &(pSender->pReader)); + int32_t code = pSyncNode->pFsm->FpSnapshotStartRead(pSyncNode->pFsm, &pSender->snapshotParam, &pSender->pReader); if (code != 0) { sSError(pSender, "prepare snapshot failed since %s", terrstr()); return -1; @@ -933,7 +908,7 @@ int32_t syncNodeOnSnapshotReplyPre(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) SyncSnapshotSend *pSendMsg = rpcMsg.pCont; pSendMsg->srcId = pSender->pSyncNode->myRaftId; - pSendMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; + pSendMsg->destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; pSendMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; pSendMsg->beginIndex = pSender->snapshotParam.start; pSendMsg->lastIndex = pSender->snapshot.lastApplyIndex; @@ -963,8 +938,9 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { SyncSnapshotRsp *pMsg = pRpcMsg->pCont; // if already drop replica, do not process - if (!syncNodeInRaftGroup(pSyncNode, &(pMsg->srcId))) { + if (!syncNodeInRaftGroup(pSyncNode, &pMsg->srcId)) { syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "maybe replica already dropped"); + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; return -1; } @@ -976,76 +952,96 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { return -1; } + // state, term, seq/ack + if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "snapshot sender not leader"); + sSError(pSender, "snapshot sender not leader"); + terrno = TSDB_CODE_SYN_NOT_LEADER; + goto _ERROR; + } + if (pMsg->startTime != pSender->startTime) { - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "sender:% " PRId64 " receiver:%" PRId64 " time not match"); - return -1; + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "snapshot sender and receiver time not match"); + sSError(pSender, "sender:%" PRId64 " receiver:%" PRId64 " time not match, code:0x%x", pMsg->startTime, + pSender->startTime, pMsg->code); + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + goto _ERROR; } - // state, term, seq/ack - if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { - if (pMsg->term == pSyncNode->pRaftStore->currentTerm) { - // prepare , send begin msg - if (pMsg->ack == SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT) { - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq pre-snapshot"); - syncNodeOnSnapshotReplyPre(pSyncNode, pMsg); - return 0; - } + if (pMsg->term != pSyncNode->pRaftStore->currentTerm) { + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "snapshot sender and receiver term not match"); + sSError(pSender, "snapshot sender term not equal, msg term:%" PRId64 " currentTerm:%" PRId64, pMsg->term, + pSyncNode->pRaftStore->currentTerm); + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + goto _ERROR; + } - if (pMsg->ack == SYNC_SNAPSHOT_SEQ_BEGIN) { - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq begin"); - if (snapshotSenderUpdateProgress(pSender, pMsg) != 0) { - return -1; - } + if (pMsg->code != 0) { + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "receive error code"); + sSError(pSender, "snapshot sender receive error code:0x%x and stop sender", pMsg->code); + terrno = pMsg->code; + goto _ERROR; + } - if (snapshotSend(pSender) != 0) { - return -1; - } - return 0; - } + // prepare , send begin msg + if (pMsg->ack == SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT) { + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq pre-snapshot"); + return syncNodeOnSnapshotPreRsp(pSyncNode, pSender, pMsg); + } - // receive ack is finish, close sender - if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) { - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq end"); - snapshotSenderStop(pSender, true); - SSyncLogReplMgr *pMgr = syncNodeGetLogReplMgr(pSyncNode, &pMsg->srcId); - if (pMgr) { - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "reset repl mgr"); - syncLogReplMgrReset(pMgr); - } - return 0; - } + if (pMsg->ack == SYNC_SNAPSHOT_SEQ_BEGIN) { + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq begin"); + if (snapshotSenderUpdateProgress(pSender, pMsg) != 0) { + return -1; + } - // send next msg - if (pMsg->ack == pSender->seq) { - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq"); - // update sender ack - if (snapshotSenderUpdateProgress(pSender, pMsg) != 0) { - return -1; - } - if (snapshotSend(pSender) != 0) { - return -1; - } + if (snapshotSend(pSender) != 0) { + return -1; + } + return 0; + } - } else if (pMsg->ack == pSender->seq - 1) { - // maybe resend - syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq and resend"); - snapshotReSend(pSender); + // receive ack is finish, close sender + if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) { + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq end"); + snapshotSenderStop(pSender, true); + SSyncLogReplMgr *pMgr = syncNodeGetLogReplMgr(pSyncNode, &pMsg->srcId); + syncLogReplMgrReset(pMgr); + return 0; + } - } else { - // error log - sSError(pSender, "snapshot sender recv error ack:%d, my seq:%d", pMsg->ack, pSender->seq); - return -1; - } - } else { - // error log - sSError(pSender, "snapshot sender term not equal"); + // send next msg + if (pMsg->ack == pSender->seq) { + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq data"); + // update sender ack + if (snapshotSenderUpdateProgress(pSender, pMsg) != 0) { + return -1; + } + if (snapshotSend(pSender) != 0) { + return -1; + } + } else if (pMsg->ack == pSender->seq - 1) { + // maybe resend + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq and resend"); + if (snapshotReSend(pSender) != 0) { return -1; } } else { // error log - sSError(pSender, "snapshot sender not leader"); + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "receive error ack"); + sSError(pSender, "snapshot sender receive error ack:%d, my seq:%d", pMsg->ack, pSender->seq); + snapshotSenderStop(pSender, true); + SSyncLogReplMgr *pMgr = syncNodeGetLogReplMgr(pSyncNode, &pMsg->srcId); + syncLogReplMgrReset(pMgr); return -1; } return 0; + +_ERROR: + snapshotSenderStop(pSender, true); + SSyncLogReplMgr *pMgr = syncNodeGetLogReplMgr(pSyncNode, &pMsg->srcId); + syncLogReplMgrReset(pMgr); + + return -1; } diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 5c1f264460630d12249cafa5ea20f8df5e279976..4e25d72b2ba587dae1192fa686e1d2a628f74142 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -74,7 +74,10 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg SBTree *pBt; int ret; - ASSERT(keyLen != 0); + if (keyLen == 0) { + tdbError("tdb/btree-open: key len cannot be zero."); + return -1; + } *ppBt = NULL; @@ -152,7 +155,11 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg tdbPostCommit(pPager->pEnv, txn); } - ASSERT(pgno != 0); + if (pgno == 0) { + tdbError("tdb/btree-open: pgno cannot be zero."); + tdbOsFree(pBt); + return -1; + } pBt->root = pgno; /* // TODO: pBt->root @@ -192,7 +199,7 @@ int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, in ret = tdbBtcMoveTo(&btc, pKey, kLen, &c); if (ret < 0) { tdbBtcClose(&btc); - ASSERT(0); + tdbError("tdb/btree-insert: btc move to failed with ret: %d.", ret); return -1; } @@ -202,17 +209,17 @@ int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, in if (c > 0) { btc.idx++; } else if (c == 0) { - // dup key not allowed - tdbError("unable to insert dup key. pKey: %p, kLen: %d, btc: %p, pTxn: %p", pKey, kLen, &btc, pTxn); - // ASSERT(0); + // dup key not allowed with insert + tdbBtcClose(&btc); + tdbError("tdb/btree-insert: dup key. pKey: %p, kLen: %d, btc: %p, pTxn: %p", pKey, kLen, &btc, pTxn); return -1; } } ret = tdbBtcUpsert(&btc, pKey, kLen, pVal, vLen, 1); if (ret < 0) { - ASSERT(0); tdbBtcClose(&btc); + tdbError("tdb/btree-insert: btc upsert failed with ret: %d.", ret); return -1; } @@ -233,7 +240,7 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) { ret = tdbBtcMoveTo(&btc, pKey, kLen, &c); if (ret < 0) { tdbBtcClose(&btc); - ASSERT(0); + tdbError("tdb/btree-delete: btc move to failed with ret: %d.", ret); return -1; } @@ -264,7 +271,7 @@ int tdbBtreeUpsert(SBTree *pBt, const void *pKey, int nKey, const void *pData, i // move the cursor ret = tdbBtcMoveTo(&btc, pKey, nKey, &c); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btree-upsert: btc move to failed with ret: %d.", ret); tdbBtcClose(&btc); return -1; } @@ -280,8 +287,8 @@ int tdbBtreeUpsert(SBTree *pBt, const void *pKey, int nKey, const void *pData, i ret = tdbBtcUpsert(&btc, pKey, nKey, pData, nData, c); if (ret < 0) { - ASSERT(0); tdbBtcClose(&btc); + tdbError("tdb/btree-upsert: btc upsert failed with ret: %d.", ret); return -1; } @@ -309,7 +316,8 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL ret = tdbBtcMoveTo(&btc, pKey, kLen, &cret); if (ret < 0) { tdbBtcClose(&btc); - ASSERT(0); + tdbError("tdb/btree-pget: btc move to failed with ret: %d.", ret); + return -1; } if (btc.idx < 0 || cret) { @@ -325,7 +333,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL pTKey = tdbRealloc(*ppKey, cd.kLen); if (pTKey == NULL) { tdbBtcClose(&btc); - ASSERT(0); + tdbError("tdb/btree-pget: realloc pTKey failed."); return -1; } *ppKey = pTKey; @@ -337,7 +345,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL pTVal = tdbRealloc(*ppVal, cd.vLen); if (pTVal == NULL) { tdbBtcClose(&btc); - ASSERT(0); + tdbError("tdb/btree-pget: realloc pTVal failed."); return -1; } *ppVal = pTVal; @@ -350,7 +358,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL } if (TDB_CELLDECODER_FREE_VAL(&cd)) { - tdbDebug("tdb btc/pget/2 decoder: %p pVal free: %p", &cd, cd.pVal); + tdbTrace("tdb btc/pget/2 decoder: %p pVal free: %p", &cd, cd.pVal); tdbFree(cd.pVal); } @@ -366,7 +374,9 @@ static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2 int mlen; int cret; - ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); + if (ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL)) { + // -1 is less than + } mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; cret = memcmp(pKey1, pKey2, mlen); @@ -381,36 +391,7 @@ static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2 } return cret; } -/* -static int tdbBtreeOpenImpl(SBTree *pBt) { - // Try to get the root page of the an existing btree - SPgno pgno; - SPage *pPage; - int ret; - - { - // 1. TODO: Search the main DB to check if the DB exists - ret = tdbPagerOpenDB(pBt->pPager, &pgno, true, pBt); - ASSERT(ret == 0); - } - if (pgno != 0) { - pBt->root = pgno; - return 0; - } - - // Try to create a new database - ret = tdbPagerAllocPage(pBt->pPager, &pgno); - if (ret < 0) { - ASSERT(0); - return -1; - } - - ASSERT(pgno != 0); - pBt->root = pgno; - return 0; -} -*/ int tdbBtreeInitPage(SPage *pPage, void *arg, int init) { SBTree *pBt; u8 flags; @@ -546,11 +527,15 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx nOlds = 3; } for (int i = 0; i < nOlds; i++) { - ASSERT(sIdx + i <= nCells); + if (ASSERT(sIdx + i <= nCells)) { + return -1; + } SPgno pgno; if (sIdx + i == nCells) { - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pParent)); + if (ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pParent))) { + return -1; + } pgno = ((SIntHdr *)(pParent->pData))->pgno; } else { pCell = tdbPageGetCell(pParent, sIdx + i); @@ -560,7 +545,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ret = tdbPagerFetchPage(pBt->pPager, &pgno, pOlds + i, tdbBtreeInitPage, &((SBtreeInitPageArg){.pBt = pBt, .flags = 0}), pTxn); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btree-balance: fetch page failed with ret: %d.", ret); return -1; } @@ -679,7 +664,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx szRCell = tdbBtreeCellSize(pPage, pCell, 0, NULL, NULL); } - ASSERT(infoNews[iNew - 1].cnt > 0); + if (ASSERT(infoNews[iNew - 1].cnt > 0)) { + return -1; + } if (infoNews[iNew].size + szRCell >= infoNews[iNew - 1].size - szRCell) { break; @@ -722,7 +709,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx iarg.flags = flags; ret = tdbPagerFetchPage(pBt->pPager, &pgno, pNews + iNew, tdbBtreeInitPage, &iarg, pTxn); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btree-balance: fetch page failed with ret: %d.", ret); + return -1; } ret = tdbPagerWrite(pBt->pPager, pNews[iNew]); @@ -766,8 +754,12 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx pCell = tdbPageGetCell(pPage, oIdx); szCell = tdbBtreeCellSize(pPage, pCell, 0, NULL, NULL); - ASSERT(nNewCells <= infoNews[iNew].cnt); - ASSERT(iNew < nNews); + if (ASSERT(nNewCells <= infoNews[iNew].cnt)) { + return -1; + } + if (ASSERT(iNew < nNews)) { + return -1; + } if (nNewCells < infoNews[iNew].cnt) { tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); @@ -806,14 +798,20 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } } } else { - ASSERT(childNotLeaf); - ASSERT(iNew < nNews - 1); + if (ASSERT(childNotLeaf)) { + return -1; + } + if (ASSERT(iNew < nNews - 1)) { + return -1; + } // set current new page right-most child ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; // insert to parent as divider cell - ASSERT(iNew < nNews - 1); + if (ASSERT(iNew < nNews - 1)) { + return -1; + } ((SPgno *)pCell)[0] = TDB_PAGE_PGNO(pNews[iNew]); tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); @@ -828,7 +826,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } if (childNotLeaf) { - ASSERT(TDB_PAGE_TOTAL_CELLS(pNews[nNews - 1]) == infoNews[nNews - 1].cnt); + if (ASSERT(TDB_PAGE_TOTAL_CELLS(pNews[nNews - 1]) == infoNews[nNews - 1].cnt)) { + return -1; + } ((SIntHdr *)(pNews[nNews - 1]->pData))->pgno = rPgno; SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; @@ -1018,7 +1018,9 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const nLeft -= kLen; // pack partial val to local if any space left if (nLocal > nHeader + kLen + sizeof(SPgno)) { - ASSERT(pVal != NULL && vLen != 0); + if (ASSERT(pVal != NULL && vLen != 0)) { + return -1; + } memcpy(pCell + nHeader + kLen, pVal, nLocal - nHeader - kLen - sizeof(SPgno)); nLeft -= nLocal - nHeader - kLen - sizeof(SPgno); } @@ -1180,9 +1182,15 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo int nPayload; int ret; - ASSERT(pPage->kLen == TDB_VARIANT_LEN || pPage->kLen == kLen); - ASSERT(pPage->vLen == TDB_VARIANT_LEN || pPage->vLen == vLen); - ASSERT(pKey != NULL && kLen > 0); + if (ASSERT(pPage->kLen == TDB_VARIANT_LEN || pPage->kLen == kLen)) { + return -1; + } + if (ASSERT(pPage->vLen == TDB_VARIANT_LEN || pPage->vLen == vLen)) { + return -1; + } + if (ASSERT(pKey != NULL && kLen > 0)) { + return -1; + } nPayload = 0; nHeader = 0; @@ -1191,7 +1199,10 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo // 1. Encode Header part /* Encode SPgno if interior page */ if (!leaf) { - ASSERT(pPage->vLen == sizeof(SPgno)); + if (pPage->vLen != sizeof(SPgno)) { + tdbError("tdb/btree-encode-cell: invalid cell."); + return -1; + } ((SPgno *)(pCell + nHeader))[0] = ((SPgno *)pVal)[0]; nHeader = nHeader + sizeof(SPgno); @@ -1216,8 +1227,8 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo ret = tdbBtreeEncodePayload(pPage, pCell, nHeader, pKey, kLen, pVal, vLen, &nPayload, pTxn, pBt); if (ret < 0) { // TODO - ASSERT(0); - return 0; + tdbError("tdb/btree-encode-cell: encode payload failed with ret: %d.", ret); + return -1; } *szCell = nHeader + nPayload; @@ -1234,7 +1245,10 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, int vLen = pDecoder->vLen; if (pDecoder->pVal) { - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pPage)); + if (TDB_BTREE_PAGE_IS_LEAF(pPage)) { + tdbError("tdb/btree-decode-payload: leaf page with non-null pVal."); + return -1; + } nPayload = pDecoder->kLen; } else { nPayload = pDecoder->kLen + pDecoder->vLen; @@ -1435,7 +1449,10 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD // 1. Decode header part if (!leaf) { - ASSERT(pPage->vLen == sizeof(SPgno)); + if (pPage->vLen != sizeof(SPgno)) { + tdbError("tdb/btree-decode-cell: invalid cell."); + return -1; + } pDecoder->pgno = ((SPgno *)(pCell + nHeader))[0]; pDecoder->pVal = (u8 *)(&(pDecoder->pgno)); @@ -1449,7 +1466,10 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD } if (pPage->vLen == TDB_VARIANT_LEN) { - ASSERT(leaf); + if (!leaf) { + tdbError("tdb/btree-decode-cell: not a leaf page."); + return -1; + } nHeader += tdbGetVarInt(pCell + nHeader, &(pDecoder->vLen)); } else { pDecoder->vLen = pPage->vLen; @@ -1481,7 +1501,10 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN * } if (pPage->vLen == TDB_VARIANT_LEN) { - ASSERT(leaf); + if (!leaf) { + tdbError("tdb/btree-cell-size: not a leaf page."); + return -1; + } nHeader += tdbGetVarInt(pCell + nHeader, &vLen); } else if (leaf) { vLen = pPage->vLen; @@ -1577,29 +1600,42 @@ int tdbBtcMoveToFirst(SBTC *pBtc) { ret = tdbPagerFetchPage(pPager, &pBt->root, &(pBtc->pPage), tdbBtreeInitPage, &((SBtreeInitPageArg){.pBt = pBt, .flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF}), pBtc->pTxn); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-tofirst: fetch page failed with ret: %d.", ret); return -1; } - ASSERT(TDB_BTREE_PAGE_IS_ROOT(pBtc->pPage)); + if (!TDB_BTREE_PAGE_IS_ROOT(pBtc->pPage)) { + tdbError("tdb/btc-move-tofirst: not a root page"); + return -1; + } pBtc->iPage = 0; if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) > 0) { pBtc->idx = 0; } else { // no any data, point to an invalid position - ASSERT(TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + if (!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + tdbError("tdb/btc-move-to-first: not a leaf page."); + return -1; + } + pBtc->idx = -1; return 0; } } else { - ASSERT(0); + // TODO + tdbError("tdb/btc-move-to-first: move from a dirty cursor."); + return -1; #if 0 // move from a position int iPage = 0; for (; iPage < pBtc->iPage; iPage++) { - ASSERT(pBtc->idxStack[iPage] >= 0); + if (pBtc->idxStack[iPage] < 0) { + tdbError("tdb/btc-move-to-first: invalid idx: %d.", pBtc->idxStack[iPage]); + return -1; + } + if (pBtc->idxStack[iPage]) break; } @@ -1621,7 +1657,7 @@ int tdbBtcMoveToFirst(SBTC *pBtc) { ret = tdbBtcMoveDownward(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-tofirst: btc move downward failed with ret: %d.", ret); return -1; } @@ -1646,7 +1682,7 @@ int tdbBtcMoveToLast(SBTC *pBtc) { ret = tdbPagerFetchPage(pPager, &pBt->root, &(pBtc->pPage), tdbBtreeInitPage, &((SBtreeInitPageArg){.pBt = pBt, .flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF}), pBtc->pTxn); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-tolast: fetch page failed with ret: %d.", ret); return -1; } @@ -1656,18 +1692,28 @@ int tdbBtcMoveToLast(SBTC *pBtc) { pBtc->idx = TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage) ? nCells - 1 : nCells; } else { // no data at all, point to an invalid position - ASSERT(TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + if (!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + tdbError("tdb/btc-move-to-last: not a leaf page."); + return -1; + } + pBtc->idx = -1; return 0; } } else { - ASSERT(0); + // TODO + tdbError("tdb/btc-move-to-last: move from a dirty cursor."); + return -1; #if 0 int iPage = 0; // downward search for (; iPage < pBtc->iPage; iPage++) { - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pBtc->pgStack[iPage])); + if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pgStack[iPage])) { + tdbError("tdb/btc-move-to-last: leaf page in cursor stack."); + return -1; + } + nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pgStack[iPage]); if (pBtc->idxStack[iPage] != nCells) break; } @@ -1694,7 +1740,7 @@ int tdbBtcMoveToLast(SBTC *pBtc) { ret = tdbBtcMoveDownward(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-tolast: btc move downward failed with ret: %d.", ret); return -1; } @@ -1752,7 +1798,7 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { ret = tdbBtcMoveToNext(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btree-next: btc move to next failed with ret: %d.", ret); return -1; } @@ -1798,7 +1844,7 @@ int tdbBtreePrev(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { ret = tdbBtcMoveToPrev(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btree-prev: btc move to prev failed with ret: %d.", ret); return -1; } @@ -1810,7 +1856,10 @@ int tdbBtcMoveToNext(SBTC *pBtc) { int ret; SCell *pCell; - ASSERT(TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + if (!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + tdbError("tdb/btc-move-to-next: not a leaf page."); + return -1; + } if (pBtc->idx < 0) return -1; @@ -1829,7 +1878,10 @@ int tdbBtcMoveToNext(SBTC *pBtc) { tdbBtcMoveUpward(pBtc); pBtc->idx++; - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + tdbError("tdb/btree-decode-cell: should not be a leaf page here."); + return -1; + } if (pBtc->idx <= TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { break; } @@ -1841,7 +1893,7 @@ int tdbBtcMoveToNext(SBTC *pBtc) { ret = tdbBtcMoveDownward(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-tonext: btc move downward failed with ret: %d.", ret); return -1; } @@ -1893,8 +1945,15 @@ static int tdbBtcMoveDownward(SBTC *pBtc) { SPgno pgno; SCell *pCell; - ASSERT(pBtc->idx >= 0); - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + if (pBtc->idx < 0) { + tdbError("tdb/btc-move-downward: invalid idx: %d.", pBtc->idx); + return -1; + } + + if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + tdbError("tdb/btc-move-downward: should not be a leaf page here."); + return -1; + } if (pBtc->idx < TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); @@ -1903,7 +1962,10 @@ static int tdbBtcMoveDownward(SBTC *pBtc) { pgno = ((SIntHdr *)pBtc->pPage->pData)->pgno; } - ASSERT(pgno); + if (!pgno) { + tdbError("tdb/btc-move-downward: invalid pgno."); + return -1; + } pBtc->pgStack[pBtc->iPage] = pBtc->pPage; pBtc->idxStack[pBtc->iPage] = pBtc->idx; @@ -1914,7 +1976,7 @@ static int tdbBtcMoveDownward(SBTC *pBtc) { ret = tdbPagerFetchPage(pBtc->pBt->pPager, &pgno, &pBtc->pPage, tdbBtreeInitPage, &((SBtreeInitPageArg){.pBt = pBtc->pBt, .flags = 0}), pBtc->pTxn); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-downward: fetch page failed with ret: %d.", ret); return -1; } @@ -1969,7 +2031,10 @@ int tdbBtcDelete(SBTC *pBtc) { int nKey; int ret; - ASSERT(idx >= 0 && idx < nCells); + if (idx < 0 || idx >= nCells) { + tdbError("tdb/btc-delete: idx: %d out of range[%d, %d).", idx, 0, nCells); + return -1; + } // drop the cell on the leaf ret = tdbPagerWrite(pPager, pBtc->pPage); @@ -2007,7 +2072,7 @@ int tdbBtcDelete(SBTC *pBtc) { ret = tdbPageUpdateCell(pPage, idx, pCell, szCell, pBtc->pTxn, pBtc->pBt); if (ret < 0) { tdbOsFree(pCell); - ASSERT(0); + tdbError("tdb/btc-delete: page update cell failed with ret: %d.", ret); return -1; } tdbOsFree(pCell); @@ -2018,11 +2083,14 @@ int tdbBtcDelete(SBTC *pBtc) { } } else { // delete the leaf page and do balance - ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0); + if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) != 0) { + tdbError("tdb/btc-delete: page to be deleted should be empty."); + return -1; + } ret = tdbBtreeBalance(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-delete: btree balance failed with ret: %d.", ret); return -1; } } @@ -2039,13 +2107,16 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int void *pBuf; int ret; - ASSERT(pBtc->idx >= 0); + if (pBtc->idx < 0) { + tdbError("tdb/btc-upsert: invalid idx: %d.", pBtc->idx); + return -1; + } // alloc space szBuf = kLen + nData + 14; pBuf = tdbRealloc(pBtc->pBt->pBuf, pBtc->pBt->pageSize > szBuf ? szBuf : pBtc->pBt->pageSize); if (pBuf == NULL) { - ASSERT(0); + tdbError("tdb/btc-upsert: realloc pBuf failed."); return -1; } pBtc->pBt->pBuf = pBuf; @@ -2054,7 +2125,7 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int // encode cell ret = tdbBtreeEncodeCell(pBtc->pPage, pKey, kLen, pData, nData, pCell, &szCell, pBtc->pTxn, pBtc->pBt); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-upsert: btree encode cell failed with ret: %d.", ret); return -1; } @@ -2067,16 +2138,22 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int // insert or update if (insert) { - ASSERT(pBtc->idx <= nCells); + if (pBtc->idx > nCells) { + tdbError("tdb/btc-upsert: invalid idx: %d, nCells: %d.", pBtc->idx, nCells); + return -1; + } ret = tdbPageInsertCell(pBtc->pPage, pBtc->idx, pCell, szCell, 0); } else { - ASSERT(pBtc->idx < nCells); + if (pBtc->idx >= nCells) { + tdbError("tdb/btc-upsert: invalid idx: %d, nCells: %d.", pBtc->idx, nCells); + return -1; + } ret = tdbPageUpdateCell(pBtc->pPage, pBtc->idx, pCell, szCell, pBtc->pTxn, pBtc->pBt); } if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-upsert: page insert/update cell failed with ret: %d.", ret); return -1; } @@ -2084,7 +2161,7 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int if (pBtc->pPage->nOverflow > 0) { ret = tdbBtreeBalance(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-upsert: btree balance failed with ret: %d.", ret); return -1; } } @@ -2109,8 +2186,8 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { &((SBtreeInitPageArg){.pBt = pBt, .flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF}), pBtc->pTxn); if (ret < 0) { // TODO - ASSERT(0); - return 0; + tdbError("tdb/btc-move-to: fetch page failed with ret: %d.", ret); + return -1; } pBtc->iPage = 0; @@ -2118,7 +2195,9 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { // for empty tree, just return with an invalid position if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0) return 0; } else { - ASSERT(0); + // TODO + tdbError("tdb/btc-move-to: move from a dirty cursor."); + return -1; #if 0 SPage *pPage; int idx; @@ -2130,7 +2209,10 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { idx = pBtc->idxStack[iPage]; nCells = TDB_PAGE_TOTAL_CELLS(pPage); - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pPage)); + if (TDB_BTREE_PAGE_IS_LEAF(pPage)) { + tdbError("tdb/btc-move-to: leaf page in cursor stack."); + return -1; + } // check if key <= current position if (idx < nCells) { @@ -2233,7 +2315,10 @@ int tdbBtcClose(SBTC *pBtc) { if (pBtc->iPage < 0) return 0; for (;;) { - ASSERT(pBtc->pPage); + if (NULL == pBtc->pPage) { + tdbError("tdb/btc-close: null ptr pPage."); + return -1; + } tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage, pBtc->pTxn); diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index c79279c658a746e83a40eaed6fd6cb8cbe6e31d7..952c49db73a2c8986d67c8e327175ad00e778ae6 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -247,7 +247,10 @@ void tdbEnvRemovePager(TDB *pDb, SPager *pPager) { // remove from the list for (ppPager = &pDb->pgrList; *ppPager && (*ppPager != pPager); ppPager = &((*ppPager)->pNext)) { } - ASSERT(*ppPager == pPager); + if (*ppPager != pPager) { + tdbError("tdb/db: invalid pPager: %p, *ppPager: %p", pPager, *ppPager); + return; + } *ppPager = pPager->pNext; // remove from hash @@ -255,7 +258,10 @@ void tdbEnvRemovePager(TDB *pDb, SPager *pPager) { ppPager = &pDb->pgrHash[hash % pDb->nPgrHash]; for (; *ppPager && *ppPager != pPager; ppPager = &((*ppPager)->pHashNext)) { } - ASSERT(*ppPager == pPager); + if (*ppPager != pPager) { + tdbError("tdb/db: invalid pPager: %p, *ppPager: %p", pPager, *ppPager); + return; + } *ppPager = pPager->pNext; // decrease the counter diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 4896568c7f5ce45ce816f16d3eec0436cf51dfc5..262f3d27e648cfebb4f081c725f443756cb6a06d 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -236,10 +236,10 @@ void tdbPCacheInvalidatePage(SPCache *pCache, SPager *pPager, SPgno pgno) { void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) { i32 nRef; - ASSERT(pTxn); - - // nRef = tdbUnrefPage(pPage); - // ASSERT(nRef >= 0); + if (!pTxn) { + tdbError("tdb/pcache: null ptr pTxn, release failed."); + return; + } tdbPCacheLock(pCache); nRef = tdbUnrefPage(pPage); @@ -275,7 +275,10 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) SPage *pPage = NULL; SPage *pPageH = NULL; - ASSERT(pTxn); + if (!pTxn) { + tdbError("tdb/pcache: null ptr pTxn, fetch impl failed."); + return NULL; + } // 1. Search the hash table pPage = pCache->pgHash[tdbPCachePageHash(pPgid) % pCache->nHash]; @@ -315,8 +318,8 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) if (!pPage && pTxn->xMalloc != NULL) { ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg); if (ret < 0 || pPage == NULL) { - // TODO - ASSERT(0); + tdbError("tdb/pcache: ret: %" PRId32 " pPage: %p, page create failed.", ret, pPage); + // TODO: recycle other backup pages return NULL; } @@ -370,7 +373,11 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) static void tdbPCachePinPage(SPCache *pCache, SPage *pPage) { if (pPage->pLruNext != NULL) { - ASSERT(tdbGetPageRef(pPage) == 0); + int32_t nRef = tdbGetPageRef(pPage); + if (nRef != 0) { + tdbError("tdb/pcache: pin page's ref not zero: %" PRId32, nRef); + return; + } pPage->pLruPrev->pLruNext = pPage->pLruNext; pPage->pLruNext->pLruPrev = pPage->pLruPrev; @@ -383,13 +390,23 @@ static void tdbPCachePinPage(SPCache *pCache, SPage *pPage) { } static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) { - i32 nRef; - - ASSERT(pPage->isLocal); - ASSERT(!pPage->isDirty); - ASSERT(tdbGetPageRef(pPage) == 0); - - ASSERT(pPage->pLruNext == NULL); + i32 nRef = tdbGetPageRef(pPage); + if (nRef != 0) { + tdbError("tdb/pcache: unpin page's ref not zero: %" PRId32, nRef); + return; + } + if (!pPage->isLocal) { + tdbError("tdb/pcache: unpin page's not local: %" PRIu8, pPage->isLocal); + return; + } + if (pPage->isDirty) { + tdbError("tdb/pcache: unpin page's dirty: %" PRIu8, pPage->isDirty); + return; + } + if (NULL != pPage->pLruNext) { + tdbError("tdb/pcache: unpin page's pLruNext not null."); + return; + } tdbTrace("pCache:%p unpin page %p/%d, nPages:%d, pgno:%d, ", pCache, pPage, pPage->id, pCache->nPages, TDB_PAGE_PGNO(pPage)); diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index 50dc8e0a6506abf40cd85b33acdd0e50f737168b..f19c3f28c05e4ec586328320228c59966ad1e3ba 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -43,9 +43,15 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) u8 *ptr; int size; - ASSERT(xMalloc); + if (!xMalloc) { + tdbError("tdb/page-create: null xMalloc."); + return -1; + } - ASSERT(TDB_IS_PGSIZE_VLD(pageSize)); + if (!TDB_IS_PGSIZE_VLD(pageSize)) { + tdbError("tdb/page-create: invalid pageSize: %d.", pageSize); + return -1; + } *ppPage = NULL; size = pageSize + sizeof(*pPage); @@ -69,16 +75,24 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) *ppPage = pPage; - tdbTrace("page/create: %p/%d %p", pPage, pPage->id, xMalloc); + tdbTrace("tdb/page-create: %p/%d %p", pPage, pPage->id, xMalloc); return 0; } int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) { u8 *ptr; - tdbTrace("page/destroy: %p/%d %p", pPage, pPage->id, xFree); - ASSERT(!pPage->isDirty); - ASSERT(xFree); + tdbTrace("tdb/page-destroy: %p/%d %p", pPage, pPage->id, xFree); + + if (pPage->isDirty) { + tdbError("tdb/page-destroy: dirty page: %" PRIu8 ".", pPage->isDirty); + return -1; + } + + if (!xFree) { + tdbError("tdb/page-destroy: null xFree."); + return -1; + } for (int iOvfl = 0; iOvfl < pPage->nOverflow; iOvfl++) { tdbTrace("tdbPage/destroy/free ovfl cell: %p/%p", pPage->apOvfl[iOvfl], pPage); @@ -105,7 +119,10 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell pPage->nOverflow = 0; pPage->xCellSize = xCellSize; - ASSERT((u8 *)pPage->pPageFtr == pPage->pFreeEnd); + if ((u8 *)pPage->pPageFtr != pPage->pFreeEnd) { + tdbError("tdb/page-zero: invalid page, pFreeEnd: %p, pPageFtr: %p", pPage->pFreeEnd, pPage->pPageFtr); + return; + } } void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)) { @@ -121,8 +138,15 @@ void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell pPage->nOverflow = 0; pPage->xCellSize = xCellSize; - ASSERT(pPage->pFreeEnd >= pPage->pFreeStart); - ASSERT(pPage->pFreeEnd - pPage->pFreeStart <= TDB_PAGE_NFREE(pPage)); + if (pPage->pFreeEnd < pPage->pFreeStart) { + tdbError("tdb/page-init: invalid page, pFreeEnd: %p, pFreeStart: %p", pPage->pFreeEnd, pPage->pFreeStart); + return; + } + if (pPage->pFreeEnd - pPage->pFreeStart > TDB_PAGE_NFREE(pPage)) { + tdbError("tdb/page-init: invalid page, pFreeEnd: %p, pFreeStart: %p, NFREE: %d", pPage->pFreeEnd, pPage->pFreeStart, + TDB_PAGE_NFREE(pPage)); + return; + } } int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl) { @@ -132,7 +156,11 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl int lidx; // local idx SCell *pNewCell; - ASSERT(szCell <= TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)); + if (szCell > TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)) { + tdbError("tdb/page-insert-cell: invalid page, szCell: %d, max free: %lu", szCell, + TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)); + return -1; + } nFree = TDB_PAGE_NFREE(pPage); nCells = TDB_PAGE_NCELLS(pPage); @@ -176,7 +204,11 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl TDB_PAGE_CELL_OFFSET_AT_SET(pPage, lidx, pNewCell - pPage->pData); TDB_PAGE_NCELLS_SET(pPage, nCells + 1); - ASSERT(pPage->pFreeStart == pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)); + if (pPage->pFreeStart != pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)) { + tdbError("tdb/page-insert-cell: invalid page, pFreeStart: %p, pCellIdx: %p, nCells: %d", pPage->pFreeStart, + pPage->pCellIdx, nCells); + return -1; + } } for (; iOvfl < pPage->nOverflow; iOvfl++) { @@ -200,7 +232,10 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) { nCells = TDB_PAGE_NCELLS(pPage); - ASSERT(idx >= 0 && idx < nCells + pPage->nOverflow); + if (idx < 0 || idx >= nCells + pPage->nOverflow) { + tdbError("tdb/page-drop-cell: idx: %d out of range, nCells: %d, nOvfl: %d.", idx, nCells, pPage->nOverflow); + return -1; + } iOvfl = 0; for (; iOvfl < pPage->nOverflow; iOvfl++) { @@ -228,7 +263,10 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) { for (; iOvfl < pPage->nOverflow; iOvfl++) { pPage->aiOvfl[iOvfl]--; - ASSERT(pPage->aiOvfl[iOvfl] > 0); + if (pPage->aiOvfl[iOvfl] <= 0) { + tdbError("tdb/page-drop-cell: invalid ai idx: %d", pPage->aiOvfl[iOvfl]); + return -1; + } } return 0; @@ -240,12 +278,19 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { pToPage->pFreeStart = pToPage->pPageHdr + (pFromPage->pFreeStart - pFromPage->pPageHdr); pToPage->pFreeEnd = (u8 *)(pToPage->pPageFtr) - ((u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); - ASSERT(pToPage->pFreeEnd >= pToPage->pFreeStart); + if (pToPage->pFreeEnd < pToPage->pFreeStart) { + tdbError("tdb/page-copy: invalid to page, pFreeStart: %p, pFreeEnd: %p", pToPage->pFreeStart, pToPage->pFreeEnd); + return; + } memcpy(pToPage->pPageHdr, pFromPage->pPageHdr, pFromPage->pFreeStart - pFromPage->pPageHdr); memcpy(pToPage->pFreeEnd, pFromPage->pFreeEnd, (u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); - ASSERT(TDB_PAGE_CCELLS(pToPage) == pToPage->pFreeEnd - pToPage->pData); + if (TDB_PAGE_CCELLS(pToPage) != pToPage->pFreeEnd - pToPage->pData) { + tdbError("tdb/page-copy: invalid to page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pToPage), + pToPage->pFreeEnd - pToPage->pData); + return; + } delta = (pToPage->pPageHdr - pToPage->pData) - (pFromPage->pPageHdr - pFromPage->pData); if (delta != 0) { @@ -295,8 +340,16 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { *ppCell = NULL; nFree = TDB_PAGE_NFREE(pPage); - ASSERT(nFree >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)); - ASSERT(TDB_PAGE_CCELLS(pPage) == pPage->pFreeEnd - pPage->pData); + if (nFree < szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { + tdbError("tdb/page-allocate: invalid cell size, nFree: %d, szCell: %d, szOffset: %d", nFree, szCell, + TDB_PAGE_OFFSET_SIZE(pPage)); + return -1; + } + if (TDB_PAGE_CCELLS(pPage) != pPage->pFreeEnd - pPage->pData) { + tdbError("tdb/page-allocate: invalid page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pPage), + pPage->pFreeEnd - pPage->pData); + return -1; + } // 1. Try to allocate from the free space block area if (pPage->pFreeEnd - pPage->pFreeStart >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { @@ -308,7 +361,10 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { // 2. Try to allocate from the page free list cellFree = TDB_PAGE_FCELL(pPage); - ASSERT(cellFree == 0 || cellFree >= pPage->pFreeEnd - pPage->pData); + if (cellFree != 0 && cellFree < pPage->pFreeEnd - pPage->pData) { + tdbError("tdb/page-allocate: cellFree: %d, pFreeEnd: %p, pData: %p.", cellFree, pPage->pFreeEnd, pPage->pData); + return -1; + } if (cellFree && pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) { SCell *pPrevFreeCell = NULL; int szPrevFreeCell; @@ -353,16 +409,30 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { // 3. Try to dfragment and allocate again tdbPageDefragment(pPage); - ASSERT(pPage->pFreeEnd - pPage->pFreeStart == nFree); - ASSERT(nFree == TDB_PAGE_NFREE(pPage)); - ASSERT(pPage->pFreeEnd - pPage->pData == TDB_PAGE_CCELLS(pPage)); + if (pPage->pFreeEnd - pPage->pFreeStart != nFree) { + tdbError("tdb/page-allocate: nFree: %d, pFreeStart: %p, pFreeEnd: %p.", nFree, pPage->pFreeStart, pPage->pFreeEnd); + return -1; + } + if (TDB_PAGE_NFREE(pPage) != nFree) { + tdbError("tdb/page-allocate: nFree: %d, page free: %d.", nFree, TDB_PAGE_NFREE(pPage)); + return -1; + } + if (pPage->pFreeEnd - pPage->pData != TDB_PAGE_CCELLS(pPage)) { + tdbError("tdb/page-allocate: ccells: %d, pFreeStart: %p, pData: %p.", TDB_PAGE_CCELLS(pPage), pPage->pFreeStart, + pPage->pData); + return -1; + } pPage->pFreeEnd -= szCell; pCell = pPage->pFreeEnd; TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); _alloc_finish: - ASSERT(pCell); + if (NULL == pCell) { + tdbError("tdb/page-allocate: null ptr pCell."); + return -1; + } + pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); TDB_PAGE_NFREE_SET(pPage, nFree - szCell - TDB_PAGE_OFFSET_SIZE(pPage)); *ppCell = pCell; @@ -375,9 +445,18 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) { u8 *dest; u8 *src; - ASSERT(pCell >= pPage->pFreeEnd); - ASSERT(pCell + szCell <= (u8 *)(pPage->pPageFtr)); - ASSERT(pCell == TDB_PAGE_CELL_AT(pPage, idx)); + if (pCell < pPage->pFreeEnd) { + tdbError("tdb/page-free: invalid cell, cell: %p, free end: %p", pCell, pPage->pFreeEnd); + return -1; + } + if (pCell + szCell > (u8 *)(pPage->pPageFtr)) { + tdbError("tdb/page-free: cell crosses page footer, cell: %p, size: %d footer: %p", pCell, szCell, pPage->pFreeEnd); + return -1; + } + if (pCell != TDB_PAGE_CELL_AT(pPage, idx)) { + tdbError("tdb/page-free: cell pos incorrect, cell: %p, pos: %p", pCell, TDB_PAGE_CELL_AT(pPage, idx)); + return -1; + } nFree = TDB_PAGE_NFREE(pPage); @@ -390,7 +469,8 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) { pPage->pPageMethods->setFreeCellInfo(pCell, szCell, cellFree); TDB_PAGE_FCELL_SET(pPage, pCell - pPage->pData); } else { - ASSERT(0); + tdbError("tdb/page-free: invalid cell size: %d", szCell); + return -1; } } @@ -417,7 +497,10 @@ static int tdbPageDefragment(SPage *pPage) { nFree = TDB_PAGE_NFREE(pPage); nCells = TDB_PAGE_NCELLS(pPage); - ASSERT(pPage->pFreeEnd - pPage->pFreeStart < nFree); + if (pPage->pFreeEnd - pPage->pFreeStart >= nFree) { + tdbError("tdb/page-defragment: invalid free range, nFree: %d.", nFree); + return -1; + } // Loop to compact the page content // Here we use an O(n^2) algorithm to do the job since @@ -443,11 +526,19 @@ static int tdbPageDefragment(SPage *pPage) { } } - ASSERT(pCell != NULL); + if (NULL == pCell) { + tdbError("tdb/page-defragment: null ptr pCell."); + return -1; + } szCell = (*pPage->xCellSize)(pPage, pCell, 0, NULL, NULL); - ASSERT(pCell + szCell <= pNextCell); + if (pCell + szCell > pNextCell) { + tdbError("tdb/page-defragment: invalid cell range, pCell: %p, szCell: %d, pNextCell: %p.", pCell, szCell, + pNextCell); + return -1; + } + if (pCell + szCell < pNextCell) { memmove(pNextCell - szCell, pCell, szCell); } @@ -457,7 +548,11 @@ static int tdbPageDefragment(SPage *pPage) { TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pNextCell - pPage->pData); } - ASSERT(pPage->pFreeEnd - pPage->pFreeStart == nFree); + if (pPage->pFreeEnd - pPage->pFreeStart != nFree) { + tdbError("tdb/page-defragment: invalid free range, nFree: %d.", nFree); + return -1; + } + TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); TDB_PAGE_FCELL_SET(pPage, 0); @@ -483,39 +578,59 @@ typedef struct { // cellNum static inline int getPageCellNum(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellNum; } static inline void setPageCellNum(SPage *pPage, int cellNum) { - ASSERT(cellNum < 65536); + if (cellNum >= 65536) { + tdbError("tdb/page-set-cell-num: invalid cellNum: %d.", cellNum); + return; + } ((SPageHdr *)(pPage->pPageHdr))[0].cellNum = (u16)cellNum; } // cellBody static inline int getPageCellBody(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellBody; } static inline void setPageCellBody(SPage *pPage, int cellBody) { - ASSERT(cellBody < 65536); + if (cellBody >= 65536) { + tdbError("tdb/page-set-cell-body: invalid cellBody: %d.", cellBody); + return; + } ((SPageHdr *)(pPage->pPageHdr))[0].cellBody = (u16)cellBody; } // cellFree static inline int getPageCellFree(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellFree; } static inline void setPageCellFree(SPage *pPage, int cellFree) { - ASSERT(cellFree < 65536); + if (cellFree >= 65536) { + tdbError("tdb/page-set-cell-free: invalid cellFree: %d.", cellFree); + return; + } ((SPageHdr *)(pPage->pPageHdr))[0].cellFree = (u16)cellFree; } // nFree static inline int getPageNFree(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].nFree; } static inline void setPageNFree(SPage *pPage, int nFree) { - ASSERT(nFree < 65536); + if (nFree >= 65536) { + tdbError("tdb/page-set-nfree: invalid nFree: %d.", nFree); + return; + } ((SPageHdr *)(pPage->pPageHdr))[0].nFree = (u16)nFree; } // cell offset static inline int getPageCellOffset(SPage *pPage, int idx) { - ASSERT(idx >= 0 && idx < getPageCellNum(pPage)); + int cellNum = getPageCellNum(pPage); + if (idx < 0 || idx >= cellNum) { + tdbError("tdb/page-cell-offset: idx: %d out of range[%d, %d).", idx, 0, cellNum); + return -1; + } + return ((u16 *)pPage->pCellIdx)[idx]; } static inline void setPageCellOffset(SPage *pPage, int idx, int offset) { - ASSERT(offset < 65536); + if (offset >= 65536) { + tdbError("tdb/page-set-cell-offset: invalid offset: %d.", offset); + return; + } ((u16 *)pPage->pCellIdx)[idx] = (u16)offset; } @@ -590,7 +705,12 @@ static inline void setLPageNFree(SPage *pPage, int nFree) { // cell offset static inline int getLPageCellOffset(SPage *pPage, int idx) { - ASSERT(idx >= 0 && idx < getLPageCellNum(pPage)); + int cellNum = getLPageCellNum(pPage); + if (idx < 0 || idx >= cellNum) { + tdbError("tdb/lpage-cell-offset: idx: %d out of range[%d, %d).", idx, 0, cellNum); + return -1; + } + return TDB_GET_U24(pPage->pCellIdx + 3 * idx); } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 8d9933b160bd9b31ee97031e672e092bcfab5ecc..9131eb0909c2534a0955a1aa9c6c8e8c153271ca 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -14,7 +14,7 @@ */ #include "tdbInt.h" - +/* #pragma pack(push, 1) typedef struct { u8 hdrString[16]; @@ -26,7 +26,7 @@ typedef struct { #pragma pack(pop) TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct"); - +*/ struct hashset_st { size_t nbits; size_t mask; @@ -234,7 +234,6 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { int ret; SPage **ppPage; - // ASSERT(pPager->inTran); if (pPage->isDirty) return 0; // ref page one more time so the page will not be release @@ -243,23 +242,8 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { // Set page as dirty pPage->isDirty = 1; - /* - // Add page to dirty list(TODO: NOT use O(n^2) algorithm) - for (ppPage = &pPager->pDirty; (*ppPage) && TDB_PAGE_PGNO(*ppPage) < TDB_PAGE_PGNO(pPage); - ppPage = &((*ppPage)->pDirtyNext)) { - } - - if (*ppPage && TDB_PAGE_PGNO(*ppPage) == TDB_PAGE_PGNO(pPage)) { - tdbUnrefPage(pPage); - - return 0; - } - ASSERT(*ppPage == NULL || TDB_PAGE_PGNO(*ppPage) > TDB_PAGE_PGNO(pPage)); - pPage->pDirtyNext = *ppPage; - *ppPage = pPage; - */ - tdbTrace("put page: %p %d to dirty tree: %p", pPage, TDB_PAGE_PGNO(pPage), &pPager->rbt); + tdbTrace("tdb/pager-write: put page: %p %d to dirty tree: %p", pPage, TDB_PAGE_PGNO(pPage), &pPager->rbt); tRBTreePut(&pPager->rbt, (SRBTreeNode *)pPage); // Write page to journal if neccessary @@ -327,7 +311,11 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { while ((pNode = tRBTreeIterNext(&iter)) != NULL) { pPage = (SPage *)pNode; - ASSERT(pPage->nOverflow == 0); + if (pPage->nOverflow != 0) { + tdbError("tdb/pager-commit: %p, pPage: %p, ovfl: %d, commit page failed.", pPager, pPage, pPage->nOverflow); + return -1; + } + ret = tdbPagerPWritePageToDB(pPager, pPage); if (ret < 0) { tdbError("failed to write page to db since %s", tstrerror(terrno)); @@ -652,12 +640,15 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa loadPage = 0; ret = tdbPagerAllocPage(pPager, &pgno); if (ret < 0) { - ASSERT(0); + tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno); return -1; } } - ASSERT(pgno > 0); + if (pgno == 0) { + tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno); + return -1; + } // fetch a page container memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN); @@ -671,7 +662,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa if (!TDB_PAGE_INITIALIZED(pPage)) { ret = tdbPagerInitPage(pPager, pPage, initPage, arg, loadPage); if (ret < 0) { - ASSERT(0); + tdbError("tdb/pager: %p, pPage: %p, init page failed.", pPager, pPage); return -1; } } @@ -679,8 +670,14 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa // printf("thread %" PRId64 " pager fetch page %d pgno %d ppage %p\n", taosGetSelfPthreadId(), pPage->id, // TDB_PAGE_PGNO(pPage), pPage); - ASSERT(TDB_PAGE_INITIALIZED(pPage)); - ASSERT(pPage->pPager == pPager); + if (!TDB_PAGE_INITIALIZED(pPage)) { + tdbError("tdb/pager: %p, pPage: %p, fetch page uninited.", pPager, pPage); + return -1; + } + if (pPage->pPager != pPager) { + tdbError("tdb/pager: %p/%p, fetch page failed.", pPager, pPage->pPager); + return -1; + } *ppgno = pgno; *ppPage = pPage; @@ -722,8 +719,10 @@ int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno) { return -1; } - ASSERT(*ppgno != 0); - + if (*ppgno == 0) { + tdbError("tdb/pager:%p, alloc new page failed.", pPager); + return -1; + } return 0; } @@ -752,7 +751,6 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage nRead = tdbOsPRead(pPager->fd, pPage->pData, pPage->pageSize, ((i64)pPage->pageSize) * (pgno - 1)); tdbTrace("tdb/pager:%p, pgno:%d, nRead:%" PRId64, pPager, pgno, nRead); if (nRead < pPage->pageSize) { - ASSERT(0); tdbError("tdb/pager:%p, pgno:%d, nRead:%" PRId64 "pgSize:%" PRId32, pPager, pgno, nRead, pPage->pageSize); TDB_UNLOCK_PAGE(pPage); return -1; @@ -763,7 +761,8 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage ret = (*initPage)(pPage, arg, init); if (ret < 0) { - ASSERT(0); + tdbError("tdb/pager:%p, pgno:%d, nRead:%" PRId64 "pgSize:%" PRId32 " init page failed.", pPager, pgno, nRead, + pPage->pageSize); TDB_UNLOCK_PAGE(pPage); return -1; } @@ -782,7 +781,8 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage } } } else { - ASSERT(0); + tdbError("tdb/pager:%p, pgno:%d, nRead:%" PRId64 "pgSize:%" PRId32 " lock page failed.", pPager, pgno, nRead, + pPage->pageSize); return -1; } @@ -869,11 +869,19 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) { return -1; } + if (tdbOsLSeek(jfd, 0L, SEEK_SET) < 0) { + tdbError("failed to lseek jfd due to %s. file:%s, offset:0", strerror(errno), pPager->dbFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + pageBuf = tdbOsCalloc(1, pPager->pageSize); if (pageBuf == NULL) { return -1; } + tdbDebug("pager/restore: %p, %d/%d, txnId:%s", pPager, pPager->dbOrigSize, pPager->dbFileSize, jFileName); + for (int pgIndex = 0; pgIndex < journalSize; ++pgIndex) { // read pgno & the page from journal SPgno pgno; @@ -884,6 +892,8 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) { return -1; } + tdbTrace("pager/restore: restore pgno:%d,", pgno); + ret = tdbOsRead(jfd, pageBuf, pPager->pageSize); if (ret < 0) { tdbOsFree(pageBuf); @@ -923,7 +933,7 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) { return -1; } - if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) { + if (tdbOsRemove(jFileName) < 0 && errno != ENOENT) { tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -943,7 +953,12 @@ int tdbPagerRestoreJournals(SPager *pPager) { while ((pDirEntry = tdbReadDir(pDir)) != NULL) { char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry)); if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) { - if (tdbPagerRestore(pPager, name) < 0) { + char jname[TD_PATH_MAX] = {0}; + int dirLen = strlen(pPager->pEnv->dbName); + memcpy(jname, pPager->pEnv->dbName, dirLen); + jname[dirLen] = '/'; + memcpy(jname + dirLen + 1, name, strlen(name)); + if (tdbPagerRestore(pPager, jname) < 0) { tdbCloseDir(&pDir); tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), name); @@ -969,7 +984,12 @@ int tdbPagerRollback(SPager *pPager) { char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry)); if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) { - if (tdbOsRemove(name) < 0 && errno != ENOENT) { + char jname[TD_PATH_MAX] = {0}; + int dirLen = strlen(pPager->pEnv->dbName); + memcpy(jname, pPager->pEnv->dbName, dirLen); + jname[dirLen] = '/'; + memcpy(jname + dirLen + 1, name, strlen(name)); + if (tdbOsRemove(jname) < 0 && errno != ENOENT) { tdbCloseDir(&pDir); tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), name); diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index 295016997945a0e22d1b135914a035e47da2cc1d..18d14fa474f51104442cf689bd52649598b9c396 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -105,16 +105,14 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF #endif - ASSERT(pPager != NULL); - if (rollback) { - tdbPagerRollback(pPager); - } else { ret = tdbPagerRestoreJournals(pPager); if (ret < 0) { tdbOsFree(pTb); return -1; } + } else { + tdbPagerRollback(pPager); } // pTb->pBt diff --git a/source/libs/tdb/src/db/tdbTxn.c b/source/libs/tdb/src/db/tdbTxn.c index 055d9c7f984b05003a7a69d996e1eb54a51bbe8c..bc23fdb75949ef97de0711a23f187f590adf14f5 100644 --- a/source/libs/tdb/src/db/tdbTxn.c +++ b/source/libs/tdb/src/db/tdbTxn.c @@ -18,7 +18,10 @@ int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg, int flags) { // not support read-committed version at the moment - ASSERT(flags == 0 || flags == (TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)); + if (flags != 0 && flags != (TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)) { + tdbError("tdb/txn: invalid txn flags: %" PRId32, flags); + return -1; + } pTxn->flags = flags; pTxn->txnId = txnid; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index d144a76eb035522272bb13b8db630e0042485519..1dc79e0cfb9d606ba0e7739e8cb6f047449365db 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1665,11 +1665,20 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { if (pCtx->retryCode != TSDB_CODE_SUCCESS) { int32_t code = pResp->code; // return internal code app - if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK) { + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED) { pResp->code = pCtx->retryCode; } } + // check whole vnodes is offline on this vgroup + if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps || pCtx->retryStep > 0) { + if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + pResp->code = TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED; + } else if (pResp->code == TSDB_CODE_RPC_BROKEN_LINK) { + pResp->code = TSDB_CODE_RPC_SOMENODE_BROKEN_LINK; + } + } + STraceId* trace = &pResp->info.traceId; bool hasEpSet = cliTryExtractEpSet(pResp, &pCtx->epSet); if (hasEpSet) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 2b1f68d5f64952cbd933e3c20c1c507c2f2be966..73848773137133551e90ca57e9d59a8ba1cbc0fa 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1001,6 +1001,13 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, uv_loop_init(srv->loop); char pipeName[PATH_MAX]; + + if (false == taosValidIpAndPort(srv->ip, srv->port)) { + terrno = TAOS_SYSTEM_ERROR(errno); + tError("invalid ip/port, %d:%d, reason:%s", srv->ip, srv->port, terrstr()); + goto End; + } + #if defined(WINDOWS) || defined(DARWIN) int ret = uv_pipe_init(srv->loop, &srv->pipeListen, 0); if (ret != 0) { @@ -1087,12 +1094,6 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, } #endif - if (false == taosValidIpAndPort(srv->ip, srv->port)) { - terrno = TAOS_SYSTEM_ERROR(errno); - tError("invalid ip/port, %d:%d, reason:%s", srv->ip, srv->port, terrstr()); - goto End; - } - if (false == addHandleToAcceptloop(srv)) { goto End; } @@ -1185,8 +1186,8 @@ void transCloseServer(void* arg) { // impl later SServerObj* srv = arg; - tDebug("send quit msg to accept thread"); if (srv->inited) { + tDebug("send quit msg to accept thread"); uv_async_send(srv->pAcceptAsync); taosThreadJoin(srv->thread, NULL); SRV_RELEASE_UV(srv->loop); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index be536dec3e9c232dd9e1288de08ee71cfcabcd93..d068ede52d217a389cce32bc8ff01b5a813bd47c 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -267,54 +267,55 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { return 0; } -static int32_t walFetchBodyNew(SWalReader *pRead) { - SWalCont *pReadHead = &pRead->pHead->head; +static int32_t walFetchBodyNew(SWalReader *pReader) { + SWalCont *pReadHead = &pReader->pHead->head; int64_t ver = pReadHead->version; - wDebug("vgId:%d, wal starts to fetch body, index:%" PRId64, pRead->pWal->cfg.vgId, ver); + wDebug("vgId:%d, wal starts to fetch body, ver:%" PRId64 " ,len:%d", pReader->pWal->cfg.vgId, ver, + pReadHead->bodyLen); - if (pRead->capacity < pReadHead->bodyLen) { - SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen); + if (pReader->capacity < pReadHead->bodyLen) { + SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen); if (ptr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pRead->pHead = ptr; - pReadHead = &pRead->pHead->head; - pRead->capacity = pReadHead->bodyLen; + pReader->pHead = ptr; + pReadHead = &pReader->pHead->head; + pReader->capacity = pReadHead->bodyLen; } - if (pReadHead->bodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, pReadHead->bodyLen)) { + if (pReadHead->bodyLen != taosReadFile(pReader->pLogFile, pReadHead->body, pReadHead->bodyLen)) { if (pReadHead->bodyLen < 0) { terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s", - pRead->pWal->cfg.vgId, pRead->pHead->head.version, ver, tstrerror(terrno)); + pReader->pWal->cfg.vgId, pReader->pHead->head.version, ver, tstrerror(terrno)); } else { wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since file corrupted", - pRead->pWal->cfg.vgId, pRead->pHead->head.version, ver); + pReader->pWal->cfg.vgId, pReader->pHead->head.version, ver); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } - pRead->curInvalid = 1; + pReader->curInvalid = 1; return -1; } if (pReadHead->version != ver) { - wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64, pRead->pWal->cfg.vgId, - pRead->pHead->head.version, ver); - pRead->curInvalid = 1; + wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64, pReader->pWal->cfg.vgId, + pReader->pHead->head.version, ver); + pReader->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } - if (walValidBodyCksum(pRead->pHead) != 0) { - wError("vgId:%d, wal fetch body error:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, ver); - pRead->curInvalid = 1; + if (walValidBodyCksum(pReader->pHead) != 0) { + wError("vgId:%d, wal fetch body error:%" PRId64 ", since body checksum not passed", pReader->pWal->cfg.vgId, ver); + pReader->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } - wDebug("vgId:%d, index:%" PRId64 " is fetched, cursor advance", pRead->pWal->cfg.vgId, ver); - pRead->curVersion = ver + 1; + wDebug("vgId:%d, index:%" PRId64 " is fetched, cursor advance", pReader->pWal->cfg.vgId, ver); + pReader->curVersion = ver + 1; return 0; } diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index b7cb20896bb79d5cee6eba6fed74dc96d87173f6..5d9b6ee5cc2a97f7b071a21b1d0c84fcba1b505d 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -41,9 +41,13 @@ target_link_libraries( ) if(TD_WINDOWS) target_link_libraries( - os PUBLIC ws2_32 iconv msvcregex wcwidth winmm crashdump + os PUBLIC ws2_32 iconv msvcregex wcwidth winmm crashdump dbghelp ) elseif(TD_DARWIN_64) + find_library(CORE_FOUNDATION_FRAMEWORK CoreFoundation) + target_link_libraries(os PUBLIC ${CORE_FOUNDATION_FRAMEWORK}) + find_library(SYSTEM_CONFIGURATION_FRAMEWORK SystemConfiguration) + target_link_libraries(os PUBLIC ${SYSTEM_CONFIGURATION_FRAMEWORK}) target_link_libraries( os PUBLIC dl m iconv ) diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index db2a9937b5861db87b17a21cc0efbd8a16683bf1..6611a937f2486693e5cf82eff7d9d1c7ec1394a1 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -988,7 +988,7 @@ int32_t taosGetFqdn(char *fqdn) { #endif char hostname[1024]; hostname[1023] = '\0'; - if (gethostname(hostname, 1023) == -1) { + if (taosGetlocalhostname(hostname, 1023) == -1) { #ifdef WINDOWS printf("failed to get hostname, reason:%s\n", strerror(WSAGetLastError())); #else @@ -998,30 +998,28 @@ int32_t taosGetFqdn(char *fqdn) { return -1; } - struct addrinfo hints = {0}; - struct addrinfo *result = NULL; #ifdef __APPLE__ // on macosx, hostname -f has the form of xxx.local // which will block getaddrinfo for a few seconds if AI_CANONNAME is set // thus, we choose AF_INET (ipv4 for the moment) to make getaddrinfo return // immediately - hints.ai_family = AF_INET; + // hints.ai_family = AF_INET; + strcpy(fqdn, hostname); + strcpy(fqdn+strlen(hostname), ".local"); #else // __APPLE__ + struct addrinfo hints = {0}; + struct addrinfo *result = NULL; hints.ai_flags = AI_CANONNAME; -#endif // __APPLE__ + int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { fprintf(stderr, "failed to get fqdn, code:%d, reason:%s\n", ret, gai_strerror(ret)); return -1; } - -#ifdef __APPLE__ - // refer to comments above - strcpy(fqdn, hostname); -#else // __APPLE__ strcpy(fqdn, result->ai_canonname); -#endif // __APPLE__ freeaddrinfo(result); +#endif // __APPLE__ + return 0; } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index e1abe8484188106b0a53238881ef55feb1fd2876..6c9bf40e4d427e9ab3ee4f0793090993c55c33e0 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -98,6 +98,9 @@ LONG WINAPI exceptionHandler(LPEXCEPTION_POINTERS exception); #include #include #include +#include +#include +#include #else @@ -1007,6 +1010,11 @@ SysNameInfo taosGetSysNameInfo() { tstrncpy(info.machine, uts.machine, sizeof(info.machine)); } + char localHostName[512]; + taosGetlocalhostname(localHostName, 512); + TdCmdPtr pCmd = taosOpenCmd("scutil --get LocalHostName"); + tstrncpy(info.nodename, localHostName, sizeof(info.nodename)); + return info; #else SysNameInfo info = {0}; @@ -1042,3 +1050,46 @@ bool taosCheckCurrentInDll() { return false; #endif } + +#ifdef _TD_DARWIN_64 +int taosGetMaclocalhostnameByCommand(char *hostname, size_t maxLen) { + TdCmdPtr pCmd = taosOpenCmd("scutil --get LocalHostName"); + if (pCmd != NULL) { + if (taosGetsCmd(pCmd, maxLen - 1, hostname) > 0) { + int len = strlen(hostname); + if (hostname[len - 1] == '\n') { + hostname[len - 1] = '\0'; + } + return 0; + } + taosCloseCmd(&pCmd); + } + return -1; +} + +int getMacLocalHostNameBySCD(char *hostname, size_t maxLen) { + SCDynamicStoreRef store = SCDynamicStoreCreate(NULL, CFSTR(""), NULL, NULL); + CFStringRef hostname_cfstr = SCDynamicStoreCopyLocalHostName(store); + if (hostname_cfstr != NULL) { + CFStringGetCString(hostname_cfstr, hostname, maxLen - 1, kCFStringEncodingMacRoman); + CFRelease(hostname_cfstr); + } else { + return -1; + } + CFRelease(store); + return 0; +} +#endif + +int taosGetlocalhostname(char *hostname, size_t maxLen) { +#ifdef _TD_DARWIN_64 + int res = getMacLocalHostNameBySCD(hostname, maxLen); + if (res != 0) { + return taosGetMaclocalhostnameByCommand(hostname, maxLen); + } else { + return 0; + } +#else + return gethostname(hostname, maxLen); +#endif +} diff --git a/source/os/test/CMakeLists.txt b/source/os/test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..21fb2ee630d2d0058ca48f94a1f39c10ce7abe40 --- /dev/null +++ b/source/os/test/CMakeLists.txt @@ -0,0 +1,24 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) +PROJECT(TDengine) + +FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) +FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib /usr/lib64) +FIND_LIBRARY(LIB_GTEST_SHARED_DIR libgtest.so /usr/lib/ /usr/local/lib /usr/lib64) + +IF (HEADER_GTEST_INCLUDE_DIR AND (LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR)) + MESSAGE(STATUS "gTest library found, build os test") + + INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) + AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ENDIF() + +INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/src/util/inc) + +# osTests +add_executable(osTests "osTests.cpp") +target_link_libraries(osTests os util gtest_main) +add_test( + NAME osTests + COMMAND osTests +) \ No newline at end of file diff --git a/source/os/test/osTests.cpp b/source/os/test/osTests.cpp index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..d79e1934defeecd360f4d79fc528baa6947984c3 100644 --- a/source/os/test/osTests.cpp +++ b/source/os/test/osTests.cpp @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + +#include "os.h" + +TEST(osTest, osSystem) { + const char *flags = "UTL FATAL "; + ELogLevel level = DEBUG_FATAL; + int32_t dflag = 255; // tsLogEmbedded ? 255 : uDebugFlag + taosPrintTrace(flags, level, dflag); +} + +#pragma GCC diagnostic pop diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 4bd829442372305501d1d571bdec910b2182fcfe..6920925e5f82b16ba6d213ffbe03d6fa40d21d06 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -191,6 +191,8 @@ void* taosArrayReserve(SArray* pArray, int32_t num) { void* dst = TARRAY_GET_ELEM(pArray, pArray->size); pArray->size += num; + memset(dst, 0, num * pArray->elemSize); + return dst; } @@ -226,7 +228,7 @@ size_t taosArrayGetSize(const SArray* pArray) { if (pArray == NULL) { return 0; } - return pArray->size; + return TARRAY_SIZE(pArray); } void taosArraySetSize(SArray* pArray, size_t size) { @@ -295,6 +297,20 @@ void taosArrayRemove(SArray* pArray, size_t index) { pArray->size -= 1; } +void taosArrayRemoveBatch(SArray* pArray, size_t index, size_t num, FDelete fp) { + ASSERT(index + num <= pArray->size); + + if (fp) { + for (int32_t i = 0; i < num; i++) { + fp(taosArrayGet(pArray, index + i)); + } + } + + memmove((char*)pArray->pData + index * pArray->elemSize, (char*)pArray->pData + (index + num) * pArray->elemSize, + (pArray->size - index - num) * pArray->elemSize); + pArray->size -= num; +} + SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize) { assert(src != NULL && elemSize > 0); SArray* pDst = taosArrayInit(size, elemSize); diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index d84a3d25c604d0089e1abc5ca2778a255a13c859..3edd067ce2aafa35e96cf632badf04f822ce2454 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -1120,7 +1120,7 @@ int32_t WCSPatternMatch(const TdUcs4 *patterStr, const TdUcs4 *str, size_t size, return TSDB_PATTERN_NOMATCH; } - return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; + return (j >= size || str[j] == 0) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; } int32_t compareStrRegexCompMatch(const void *pLeft, const void *pRight) { return compareStrRegexComp(pLeft, pRight); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index abd8a9055a12f4bb7778e5720958294f58f18eb4..7bf63abc5febd6b7c99e0ba9b5061748aa58ae8f 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -51,6 +51,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, "Unable to resolve FQD TAOS_DEFINE_ERROR(TSDB_CODE_RPC_PORT_EADDRINUSE, "Port already in use") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_BROKEN_LINK, "Conn is broken") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_TIMEOUT, "Conn read timeout") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED, "some vnode/qnode/mnode(s) out of service") //common & util TAOS_DEFINE_ERROR(TSDB_CODE_TIME_UNSYNCED, "Client and server's time is not synchronized") @@ -136,6 +137,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CLAUSE_ERROR, "not supported stmt cl TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_KILLED, "Query killed") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_EXEC_NODE, "No available execution node in current query policy configuration") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NOT_STABLE_ERROR, "Table is not a super table") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CACHE_ERROR, "Stmt cache error") // mnode-common TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS, "Insufficient privilege for operation") @@ -405,6 +407,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_PROPOSE_NOT_READY, "Sync not ready for pr TAOS_DEFINE_ERROR(TSDB_CODE_SYN_STANDBY_NOT_READY, "Sync not ready for standby") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_BATCH_ERROR, "Sync batch error") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_RESTORING, "Sync is restoring") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_SNAPSHOT_MSG, "Sync invalid snapshot msg") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_BUFFER_FULL, "Sync buffer is full") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INTERNAL_ERROR, "Sync internal error") //tq @@ -644,13 +648,10 @@ const char* tstrerror(int32_t err) { // this is a system errno if ((err & 0x00ff0000) == 0x00ff0000) { int32_t code = err & 0x0000ffff; - if (code >= 0 && code < 36) { - return strerror(code); - } else { - return "unknown err"; - } + // strerror can handle any invalid code + // invalid code return Unknown error + return strerror(code); } - int32_t s = 0; int32_t e = sizeof(errors) / sizeof(errors[0]); diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 863cee9b08ced50308afc11db48a5f15c49f8162..a9a84c186033f8b9574f3ef3f22fed10e6794207 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -36,7 +36,7 @@ int32_t tQWorkerInit(SQWorkerPool *pool) { worker->pool = pool; } - uDebug("worker:%s is initialized, min:%d max:%d", pool->name, pool->min, pool->max); + uInfo("worker:%s is initialized, min:%d max:%d", pool->name, pool->min, pool->max); return 0; } @@ -51,8 +51,10 @@ void tQWorkerCleanup(SQWorkerPool *pool) { for (int32_t i = 0; i < pool->max; ++i) { SQWorker *worker = pool->workers + i; if (taosCheckPthreadValid(worker->thread)) { + uInfo("worker:%s:%d is stopping", pool->name, worker->id); taosThreadJoin(worker->thread, NULL); taosThreadClear(&worker->thread); + uInfo("worker:%s:%d is stopped", pool->name, worker->id); } } @@ -60,7 +62,7 @@ void tQWorkerCleanup(SQWorkerPool *pool) { taosCloseQset(pool->qset); taosThreadMutexDestroy(&pool->mutex); - uDebug("worker:%s is closed", pool->name); + uInfo("worker:%s is closed", pool->name); } static void *tQWorkerThreadFp(SQWorker *worker) { @@ -119,7 +121,7 @@ STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) { taosThreadAttrDestroy(&thAttr); pool->num++; - uDebug("worker:%s:%d is launched, total:%d", pool->name, worker->id, pool->num); + uInfo("worker:%s:%d is launched, total:%d", pool->name, worker->id, pool->num); } while (pool->num < pool->min); } @@ -130,7 +132,134 @@ STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) { } void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue) { - uDebug("worker:%s, queue:%p is freed", pool->name, queue); + uInfo("worker:%s, queue:%p is freed", pool->name, queue); + taosCloseQueue(queue); +} + +int32_t tAutoQWorkerInit(SAutoQWorkerPool *pool) { + pool->qset = taosOpenQset(); + pool->workers = taosArrayInit(2, sizeof(SQWorker *)); + if (pool->workers == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + (void)taosThreadMutexInit(&pool->mutex, NULL); + + uInfo("worker:%s is initialized as auto", pool->name); + return 0; +} + +void tAutoQWorkerCleanup(SAutoQWorkerPool *pool) { + int32_t size = taosArrayGetSize(pool->workers); + for (int32_t i = 0; i < size; ++i) { + SQWorker *worker = taosArrayGetP(pool->workers, i); + if (taosCheckPthreadValid(worker->thread)) { + taosQsetThreadResume(pool->qset); + } + } + + for (int32_t i = 0; i < size; ++i) { + SQWorker *worker = taosArrayGetP(pool->workers, i); + if (taosCheckPthreadValid(worker->thread)) { + uInfo("worker:%s:%d is stopping", pool->name, worker->id); + taosThreadJoin(worker->thread, NULL); + taosThreadClear(&worker->thread); + uInfo("worker:%s:%d is stopped", pool->name, worker->id); + } + taosMemoryFree(worker); + } + + taosArrayDestroy(pool->workers); + taosCloseQset(pool->qset); + taosThreadMutexDestroy(&pool->mutex); + + uInfo("worker:%s is closed", pool->name); +} + +static void *tAutoQWorkerThreadFp(SQWorker *worker) { + SAutoQWorkerPool *pool = worker->pool; + SQueueInfo qinfo = {0}; + void *msg = NULL; + int32_t code = 0; + + taosBlockSIGPIPE(); + setThreadName(pool->name); + worker->pid = taosGetSelfPthreadId(); + uInfo("worker:%s:%d is running, thread:%08" PRId64, pool->name, worker->id, worker->pid); + + while (1) { + if (taosReadQitemFromQset(pool->qset, (void **)&msg, &qinfo) == 0) { + uInfo("worker:%s:%d qset:%p, got no message and exiting, thread:%08" PRId64, pool->name, worker->id, pool->qset, + worker->pid); + break; + } + + if (qinfo.fp != NULL) { + qinfo.workerId = worker->id; + qinfo.threadNum = taosArrayGetSize(pool->workers); + (*((FItem)qinfo.fp))(&qinfo, msg); + } + + taosUpdateItemSize(qinfo.queue, 1); + } + + return NULL; +} + +STaosQueue *tAutoQWorkerAllocQueue(SAutoQWorkerPool *pool, void *ahandle, FItem fp) { + STaosQueue *queue = taosOpenQueue(); + if (queue == NULL) return NULL; + + taosThreadMutexLock(&pool->mutex); + taosSetQueueFp(queue, fp, NULL); + taosAddIntoQset(pool->qset, queue, ahandle); + + int32_t queueNum = taosGetQueueNumber(pool->qset); + int32_t curWorkerNum = taosArrayGetSize(pool->workers); + int32_t dstWorkerNum = ceil(queueNum * pool->ratio); + if (dstWorkerNum < 1) dstWorkerNum = 1; + + // spawn a thread to process queue + while (curWorkerNum < dstWorkerNum) { + SQWorker *worker = taosMemoryCalloc(1, sizeof(SQWorker)); + if (worker == NULL || taosArrayPush(pool->workers, &worker) == NULL) { + uError("worker:%s:%d failed to create", pool->name, curWorkerNum); + taosMemoryFree(worker); + taosCloseQueue(queue); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + worker->id = curWorkerNum; + worker->pool = pool; + + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (taosThreadCreate(&worker->thread, &thAttr, (ThreadFp)tAutoQWorkerThreadFp, worker) != 0) { + uError("worker:%s:%d failed to create thread, total:%d", pool->name, worker->id, curWorkerNum); + (void)taosArrayPop(pool->workers); + taosMemoryFree(worker); + taosCloseQueue(queue); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + taosThreadAttrDestroy(&thAttr); + uInfo("worker:%s:%d is launched, total:%d", pool->name, worker->id, (int32_t)taosArrayGetSize(pool->workers)); + + curWorkerNum++; + } + + taosThreadMutexUnlock(&pool->mutex); + uInfo("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle); + + return queue; +} + +void tAutoQWorkerFreeQueue(SAutoQWorkerPool *pool, STaosQueue *queue) { + uInfo("worker:%s, queue:%p is freed", pool->name, queue); taosCloseQueue(queue); } @@ -152,7 +281,7 @@ int32_t tWWorkerInit(SWWorkerPool *pool) { worker->pool = pool; } - uDebug("worker:%s is initialized, max:%d", pool->name, pool->max); + uInfo("worker:%s is initialized, max:%d", pool->name, pool->max); return 0; } @@ -169,17 +298,19 @@ void tWWorkerCleanup(SWWorkerPool *pool) { for (int32_t i = 0; i < pool->max; ++i) { SWWorker *worker = pool->workers + i; if (taosCheckPthreadValid(worker->thread)) { + uInfo("worker:%s:%d is stopping", pool->name, worker->id); taosThreadJoin(worker->thread, NULL); taosThreadClear(&worker->thread); taosFreeQall(worker->qall); taosCloseQset(worker->qset); + uInfo("worker:%s:%d is stopped", pool->name, worker->id); } } taosMemoryFreeClear(pool->workers); taosThreadMutexDestroy(&pool->mutex); - uDebug("worker:%s is closed", pool->name); + uInfo("worker:%s is closed", pool->name); } static void *tWWorkerThreadFp(SWWorker *worker) { @@ -235,7 +366,7 @@ STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); if (taosThreadCreate(&worker->thread, &thAttr, (ThreadFp)tWWorkerThreadFp, worker) != 0) goto _OVER; - uDebug("worker:%s:%d is launched, max:%d", pool->name, worker->id, pool->max); + uInfo("worker:%s:%d is launched, max:%d", pool->name, worker->id, pool->max); pool->nextId = (pool->nextId + 1) % pool->max; taosThreadAttrDestroy(&thAttr); @@ -259,13 +390,14 @@ _OVER: } else { while (worker->pid <= 0) taosMsleep(10); queue->threadId = worker->pid; - uInfo("worker:%s, queue:%p is allocated, ahandle:%p thread:%08" PRId64, pool->name, queue, ahandle, queue->threadId); + uInfo("worker:%s, queue:%p is allocated, ahandle:%p thread:%08" PRId64, pool->name, queue, ahandle, + queue->threadId); return queue; } } void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue) { - uDebug("worker:%s, queue:%p is freed", pool->name, queue); + uInfo("worker:%s, queue:%p is freed", pool->name, queue); taosCloseQueue(queue); } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 8f25dfa0642754821584a6c715a99876c1ac0ea7..ac207003b5d95a4811b2d2f26fb2db1926eaec85 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -681,8 +681,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/create_wrong_topic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/basic5.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb0.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb0.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb2.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb3.py @@ -733,7 +733,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-19201.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 @@ -1037,7 +1037,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py #develop test -,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py +#,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/demo.py @@ -1046,7 +1046,7 @@ ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/json_tag.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/query_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py -,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py +#,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 60df188d7b84c2f202b6ec11665013da143393e9..88dada44accf588357314843d2a7d09964c896c0 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -232,8 +232,8 @@ CaseCtrl gCaseCtrl = { .printCreateTblSql = true, .printQuerySql = true, .printStmtSql = true, - .printVerbose = false, - .printRes = false, + .printVerbose = true, + .printRes = true, .autoCreateTbl = false, .numericParam = false, .rowNum = 0, diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 662c4a1a6c1fb5dba105f4996203885f77e61247..217bd66ef61d937a575145e36c645db37b20f046 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -134,6 +134,7 @@ echo "mDebugFlag 143" >> $TAOS_CFG echo "wDebugFlag 143" >> $TAOS_CFG echo "sDebugFlag 143" >> $TAOS_CFG echo "tsdbDebugFlag 143" >> $TAOS_CFG +echo "tdbDebugFlag 143" >> $TAOS_CFG echo "tqDebugFlag 143" >> $TAOS_CFG echo "fsDebugFlag 143" >> $TAOS_CFG echo "idxDebugFlag 143" >> $TAOS_CFG diff --git a/tests/script/test.sh b/tests/script/test.sh index a7a5d34fbe98dee38e30dec3749db82cf069222d..19180382fd30de26667fef44a5ebc54d19d5b6a7 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -10,13 +10,11 @@ set +e #set -x FILE_NAME= -RELEASE=0 -ASYNC=0 VALGRIND=0 -UNIQUE=0 +TEST=0 UNAME_BIN=`which uname` OS_TYPE=`$UNAME_BIN` -while getopts "f:agvum" arg +while getopts "f:tgv" arg do case $arg in f) @@ -25,8 +23,8 @@ do v) VALGRIND=1 ;; - u) - UNIQUE=1 + t) + TEST=1 ;; g) VALGRIND=2 @@ -140,6 +138,11 @@ if [ -n "$FILE_NAME" ]; then result=$? echo "Execute result:" $result + if [ $TEST -eq 1 ]; then + echo "Exit without check asan errors" + exit 1 + fi + if [ $result -eq 0 ]; then $CODE_DIR/sh/sigint_stop_dnodes.sh $CODE_DIR/sh/checkAsan.sh diff --git a/tests/script/tsim/dnode/balance1.sim b/tests/script/tsim/dnode/balance1.sim index 2b0154c8e5fdb975ec993b959d365a04ec7a7c5d..3bb9c4f3ebd6cefab513f0c6fc35cbdc2d7aee89 100644 --- a/tests/script/tsim/dnode/balance1.sim +++ b/tests/script/tsim/dnode/balance1.sim @@ -74,10 +74,10 @@ sql insert into d2.t2 values(now+5s, 21) sql select * from information_schema.ins_dnodes print dnode1 openVnodes $data(1)[2] print dnode2 openVnodes $data(2)[2] -if $data(1)[2] != 0 then +if $data(1)[2] != 1 then return -1 endi -if $data(2)[2] != 2 then +if $data(2)[2] != 1 then return -1 endi diff --git a/tests/script/tsim/dnode/balance2.sim b/tests/script/tsim/dnode/balance2.sim index 3f5a42d4d33b1c29acc054d816dc213c573108d0..9eeb7e125158eafd27bfe31d39586bf2a116d9ee 100644 --- a/tests/script/tsim/dnode/balance2.sim +++ b/tests/script/tsim/dnode/balance2.sim @@ -161,13 +161,13 @@ print dnode1 openVnodes $data(1)[2] print dnode3 openVnodes $data(3)[2] print dnode4 openVnodes $data(4)[2] print dnode5 openVnodes $data(5)[2] -if $data(1)[2] != 2 then +if $data(1)[2] != 3 then return -1 endi if $data(3)[2] != 3 then return -1 endi -if $data(4)[2] != 4 then +if $data(4)[2] != 3 then return -1 endi if $data(5)[2] != 3 then diff --git a/tests/script/tsim/dnode/balance3.sim b/tests/script/tsim/dnode/balance3.sim index ce328f10bd6561fb4277decdb804ef235f51cb09..2fb284b46614e37ee6ac27bd13a16eb7429e9d86 100644 --- a/tests/script/tsim/dnode/balance3.sim +++ b/tests/script/tsim/dnode/balance3.sim @@ -127,10 +127,10 @@ print dnode1 openVnodes $data(1)[2] print dnode2 openVnodes $data(2)[2] print dnode3 openVnodes $data(3)[2] print dnode4 openVnodes $data(4)[2] -if $data(1)[2] != 0 then +if $data(1)[2] != 1 then return -1 endi -if $data(2)[2] != 2 then +if $data(2)[2] != 1 then return -1 endi if $data(3)[2] != 2 then @@ -228,10 +228,10 @@ print dnode1 openVnodes $data(1)[2] print dnode3 openVnodes $data(3)[2] print dnode4 openVnodes $data(4)[2] print dnode5 openVnodes $data(5)[2] -if $data(1)[2] != 1 then +if $data(1)[2] != 2 then return -1 endi -if $data(3)[2] != 3 then +if $data(3)[2] != 2 then return -1 endi if $data(4)[2] != 3 then diff --git a/tests/script/tsim/dnode/balancex.sim b/tests/script/tsim/dnode/balancex.sim index 6b16e8b5696e8279337605bea44c04e001bce531..0cfc64a954f10457ab720990c758cfe4032e900a 100644 --- a/tests/script/tsim/dnode/balancex.sim +++ b/tests/script/tsim/dnode/balancex.sim @@ -142,10 +142,10 @@ print dnode1 openVnodes $data(1)[2] print dnode2 openVnodes $data(2)[2] print dnode2 openVnodes $data(3)[2] print dnode2 openVnodes $data(4)[2] -if $data(1)[2] != 0 then +if $data(1)[2] != 1 then return -1 endi -if $data(2)[2] != 2 then +if $data(2)[2] != 1 then return -1 endi if $data(3)[2] != 2 then diff --git a/tests/script/tsim/dnode/vnode_clean.sim b/tests/script/tsim/dnode/vnode_clean.sim index 112e5f28a45c367dc3df301df01efc356185fc9c..ba1d083c68e069fdbfecbd13144f31750a81eb81 100644 --- a/tests/script/tsim/dnode/vnode_clean.sim +++ b/tests/script/tsim/dnode/vnode_clean.sim @@ -71,10 +71,10 @@ sql insert into d2.t2 values(now+5s, 21) sql select * from information_schema.ins_dnodes print dnode1 openVnodes $data(1)[2] print dnode2 openVnodes $data(2)[2] -if $data(1)[2] != 0 then +if $data(1)[2] != 1 then return -1 endi -if $data(2)[2] != 2 then +if $data(2)[2] != 1 then return -1 endi @@ -181,10 +181,10 @@ sql select * from information_schema.ins_dnodes print dnode1 openVnodes $data(1)[2] print dnode2 openVnodes $data(3)[2] print dnode2 openVnodes $data(4)[2] -if $data(1)[2] != 0 then +if $data(1)[2] != 1 then return -1 endi -if $data(3)[2] != 2 then +if $data(3)[2] != 1 then return -1 endi if $data(4)[2] != 1 then @@ -204,10 +204,10 @@ sql select * from information_schema.ins_dnodes print dnode1 openVnodes $data(1)[2] print dnode2 openVnodes $data(3)[2] print dnode2 openVnodes $data(4)[2] -if $data(1)[2] != 0 then +if $data(1)[2] != 1 then return -1 endi -if $data(3)[2] != 2 then +if $data(3)[2] != 1 then return -1 endi if $data(4)[2] != 2 then @@ -220,13 +220,13 @@ sql select * from information_schema.ins_dnodes print dnode1 openVnodes $data(1)[2] print dnode2 openVnodes $data(3)[2] print dnode2 openVnodes $data(4)[2] -if $data(1)[2] != 1 then +if $data(1)[2] != 2 then return -1 endi if $data(3)[2] != null then return -1 endi -if $data(4)[2] != 3 then +if $data(4)[2] != 2 then return -1 endi diff --git a/tests/script/tsim/stream/basic3.sim b/tests/script/tsim/stream/basic3.sim index 48fb860a72272077051f16d0df7723a46706e0b3..41e19b19af0e388ff8a736723d9d74df356e2a6a 100644 --- a/tests/script/tsim/stream/basic3.sim +++ b/tests/script/tsim/stream/basic3.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c debugflag -v 131 -system sh/exec.sh -n dnode1 -s start -v +system sh/cfg.sh -n dnode1 -c debugflag 131 +system sh/exec.sh -n dnode1 -s start sleep 5000 diff --git a/tests/script/tsim/stream/sliding.sim b/tests/script/tsim/stream/sliding.sim index c9a1ddd922b4c6d1194b87902ef6bdf22826165e..8287274cd2a008f264365ea06d049428ae645d2f 100644 --- a/tests/script/tsim/stream/sliding.sim +++ b/tests/script/tsim/stream/sliding.sim @@ -672,6 +672,123 @@ if $data61 != 1 then goto loop5 endi +print step 8 + +sql drop stream IF EXISTS streams4; +sql drop database IF EXISTS test4; + +sql create database test4 vgroups 6; +sql use test4; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams4 trigger at_once into streamt4 as select _wstart as ts, count(*),min(a) c1 from st interval(10s) sliding(5s); + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791243000,2,1,1,1.0); + +sql insert into t2 values(1648791273000,3,1,1,1.0); +sql insert into t2 values(1648791313000,4,1,1,1.0); + +$loop_count = 0 + +loop6: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt4 order by 1; + +# row 0 +if $rows != 8 then + print ====loop6=rows=$rows + goto loop6 +endi + +if $data01 != 1 then + print ====loop6=data01=$data01 + goto loop6 +endi + +if $data02 != 1 then + print ====loop6=data02=$data02 + return -1 +endi + +if $data11 != 1 then + print ====loop6=data11=$data11 + goto loop6 +endi + +if $data12 != 1 then + print ====loop6=data12=$data12 + return -1 +endi + +if $data21 != 1 then + print ====loop6=data21=$data21 + goto loop6 +endi + +if $data22 != 2 then + print ====loop6=data22=$data22 + return -1 +endi + +if $data31 != 1 then + print ====loop6=data31=$data31 + goto loop6 +endi + +if $data32 != 2 then + print ====loop6=data32=$data32 + return -1 +endi + +if $data41 != 1 then + print ====loop6=data41=$data41 + goto loop6 +endi + +if $data42 != 3 then + print ====loop6=data42=$data42 + return -1 +endi + +if $data51 != 1 then + print ====loop6=data51=$data51 + goto loop6 +endi + +if $data52 != 3 then + print ====loop6=data52=$data52 + return -1 +endi + +if $data61 != 1 then + print ====loop6=data61=$data61 + return -1 +endi + +if $data62 != 4 then + print ====loop6=data62=$data62 + return -1 +endi + +if $data71 != 1 then + print ====loop6=data71=$data71 + return -1 +endi + +if $data72 != 4 then + print ====loop6=data72=$data72 + return -1 +endi + $loop_all = $loop_all + 1 print ============loop_all=$loop_all diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 722008836964c8b8fcc6ed72960928d2cf8dd10e..1834432bc9a9364e039df2c95ce47fb01e945eab 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -12,6 +12,7 @@ from util.dnodes import * from util.dnodes import TDDnodes from util.dnodes import TDDnode from util.cluster import * +import subprocess BASEVERSION = "3.0.1.8" class TDTestCase: @@ -26,8 +27,21 @@ class TDTestCase: self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) + + def checkProcessPid(self,processName): + i=0 + while i<60: + print(f"wait stop {processName}") + processPid = subprocess.getstatusoutput(f'ps aux|grep {processName} |grep -v "grep"|awk \'{{print $2}}\'')[1] + print(f"times:{i},{processName}-pid:{processPid}") + if(processPid == ""): + break + i += 1 + sleep(1) + else: + print(f'this processName is not stoped in 60s') - + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -115,15 +129,15 @@ class TDTestCase: # tdsqlF.query(f"select count(*) from {stb}") # tdsqlF.checkData(0,0,tableNumbers*recordNumbers1) os.system("pkill taosd") - sleep(2) + self.checkProcessPid("taosd") print(f"start taosd: nohup taosd -c {cPath} & ") os.system(f" nohup taosd -c {cPath} & " ) sleep(10) tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y") - os.system("pkill -9 taosd") - + os.system("pkill taosd") # make sure all the data are saved in disk. + self.checkProcessPid("taosd") tdLog.printNoPrefix("==========step2:update new version ") diff --git a/tests/system-test/1-insert/opentsdb_json_taosc_insert.py b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py index 44243fe029233dc3458a7b6822600bc198e11824..93f3c7410df5bfc0ffef836d8353249f8de0c965 100644 --- a/tests/system-test/1-insert/opentsdb_json_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py @@ -1719,6 +1719,7 @@ class TDTestCase: print(err.errno) def runAll(self): + """ for value_type in ["obj", "default"]: self.initCheckCase(value_type) self.symbolsCheckCase(value_type) @@ -1771,7 +1772,7 @@ class TDTestCase: # self.sStbStbDdataDtsMtInsertMultiThreadCheckCase() # self.sStbDtbDdataDtsMtInsertMultiThreadCheckCase() # self.lengthIcreaseCrashCheckCase() - + """ def run(self): print("running {}".format(__file__)) self.createDb() diff --git a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py index f58882720610b32794fc48e4cd652ae70e63b5d7..a17a2d9514a50a06f594fca2990048b1876ef441 100644 --- a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py @@ -1416,8 +1416,8 @@ class TDTestCase: self.symbolsCheckCase() self.tsCheckCase() self.openTstbTelnetTsCheckCase() - self.idSeqCheckCase() - self.idLetterCheckCase() + #self.idSeqCheckCase() + #self.idLetterCheckCase() self.noIdCheckCase() self.maxColTagCheckCase() self.stbTbNameCheckCase() @@ -1450,7 +1450,7 @@ class TDTestCase: self.spellCheckCase() self.pointTransCheckCase() self.defaultTypeCheckCase() - self.tbnameTagsColsNameCheckCase() + #self.tbnameTagsColsNameCheckCase() # # # MultiThreads # self.stbInsertMultiThreadCheckCase() # self.sStbStbDdataInsertMultiThreadCheckCase() diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index b764edebd732ad7d35fea98f0e75c08307991ff9..8ad1982b55506621606a92840e5496f29094da75 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -18,7 +18,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), True) #tdSql.init(conn.cursor(), logSql) # output sql.txt file def checkFileContent(self, dbname="sml_db"): @@ -76,13 +76,14 @@ class TDTestCase: tdSql.query(f"select * from {dbname}.`sys.cpu.nice` order by _ts") tdSql.checkRows(2) tdSql.checkData(0, 1, 9.000000000) - tdSql.checkData(0, 2, "lga") - tdSql.checkData(0, 3, "web02") - tdSql.checkData(0, 4, None) + tdSql.checkData(0, 2, "web02") + tdSql.checkData(0, 3, None) + tdSql.checkData(0, 4, "lga") + tdSql.checkData(1, 1, 18.000000000) - tdSql.checkData(1, 2, "lga") - tdSql.checkData(1, 3, "web01") - tdSql.checkData(1, 4, "t1") + tdSql.checkData(1, 2, "web01") + tdSql.checkData(1, 3, "t1") + tdSql.checkData(0, 4, "lga") tdSql.query(f"select * from {dbname}.macylr") tdSql.checkRows(2) diff --git a/tests/system-test/7-tmq/subscribeDb.py b/tests/system-test/7-tmq/subscribeDb.py index fd06eedefdcd899fb71261bd4e97324ee0170a1b..0fa9bcfbd461c0498d301e7b702a2ee3f8dd8aa4 100644 --- a/tests/system-test/7-tmq/subscribeDb.py +++ b/tests/system-test/7-tmq/subscribeDb.py @@ -61,7 +61,7 @@ class TDTestCase: def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'): sql = "insert into %s.consumeinfo values "%cdbName - sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) + sql += "(now + %ds, %d, '%s', '%s', %d, %d, %d)"%(consumerId, consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) tdLog.info("consume info sql: %s"%sql) tdSql.query(sql) @@ -174,12 +174,13 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -271,12 +272,13 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -337,6 +339,7 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath @@ -406,12 +409,13 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() diff --git a/tests/system-test/7-tmq/subscribeDb0.py b/tests/system-test/7-tmq/subscribeDb0.py index d4c5e2f87fd6b06eb874f8398ce64ee6eb5947cc..50ef52cb15a5336395154813619976c7933f2dc7 100644 --- a/tests/system-test/7-tmq/subscribeDb0.py +++ b/tests/system-test/7-tmq/subscribeDb0.py @@ -174,12 +174,13 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -191,6 +192,7 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath @@ -254,12 +256,13 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 141d013270e5d9a545332a1cad173ee066841e73..4cda0624010305743e4249dd87594f1cf255b037 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -60,7 +60,7 @@ class TMQCom: def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'): sql = "insert into %s.consumeinfo values "%cdbName - sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) + sql += "(now + %ds, %d, '%s', '%s', %d, %d, %d)"%(consumerId, consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) tdLog.info("consume info sql: %s"%sql) tdSql.query(sql) diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index f2fbd8486570e385332be5280f58f25af7ae4d75..7ca18f293cddf415a2b4ac3354f5848171e7b736 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -123,6 +123,11 @@ class TDTestCase: def checkData(self): tdSql.execute('use db_taosx') + tdSql.query("select * from tb1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 0) + tdSql.checkData(0, 2, 1) + tdSql.query("select * from ct3 order by c1 desc") tdSql.checkRows(2) tdSql.checkData(0, 1, 51) diff --git a/tests/system-test/compatibilityAllTest.sh b/tests/system-test/compatibilityAllTest.sh new file mode 100755 index 0000000000000000000000000000000000000000..8b599afd8659e9ce4ed2f92408523f9de8dda917 --- /dev/null +++ b/tests/system-test/compatibilityAllTest.sh @@ -0,0 +1,46 @@ +#!/bin/bash +ulimit -c unlimited +#======================p1-insert=============== + +python3 ./test.py -f 0-others/taosShell.py +python3 ./test.py -f 0-others/taosShellError.py +python3 ./test.py -f 0-others/taosShellNetChk.py +python3 ./test.py -f 1-insert/alter_database.py +python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py +python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py +python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py +python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py +python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py +python3 ./test.py -f 1-insert/alter_stable.py +python3 ./test.py -f 1-insert/alter_table.py +python3 ./test.py -f 1-insert/boundary.py +python3 ./test.py -f 2-query/top.py +python3 ./test.py -f 2-query/top.py -R +python3 ./test.py -f 2-query/tsbsQuery.py +python3 ./test.py -f 2-query/tsbsQuery.py -R +python3 ./test.py -f 2-query/ttl_comment.py +python3 ./test.py -f 2-query/ttl_comment.py -R +python3 ./test.py -f 2-query/twa.py +python3 ./test.py -f 2-query/twa.py -R +python3 ./test.py -f 2-query/union.py +python3 ./test.py -f 2-query/union.py -R +python3 ./test.py -f 6-cluster/5dnode1mnode.py +python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 -i False +python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 -i False +python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 -n 3 +python3 ./test.py -f 7-tmq/subscribeStb4.py +python3 ./test.py -f 7-tmq/db.py +python3 ./test.py -f 7-tmq/tmqError.py +python3 ./test.py -f 7-tmq/schema.py +python3 ./test.py -f 7-tmq/stbFilter.py +python3 ./test.py -f 7-tmq/tmqCheckData.py +python3 ./test.py -f 7-tmq/tmqCheckData1.py +python3 ./test.py -f 7-tmq/tmqConsumerGroup.py +python3 ./test.py -f 7-tmq/tmqShow.py +python3 ./test.py -f 7-tmq/tmqAlterSchema.py +python3 ./test.py -f 99-TDcase/TD-20582.py diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index df416b3822a890f9b6e6e5d3790521686587f50b..49885c0aea8098b9ea8160b0cd89c93d1f5b20aa 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -78,11 +78,20 @@ int smlProcess_telnet_Test() { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - const char *sql[] = {"sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0", + char *sql[4] = {0}; + sql[0] = taosMemoryCalloc(1, 128); + sql[1] = taosMemoryCalloc(1, 128); + sql[2] = taosMemoryCalloc(1, 128); + sql[3] = taosMemoryCalloc(1, 128); + const char *sql1[] = {"sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0", "sys.if.bytes.out 1479496101 1.3E1 interface=eth0 host=web01 ", "sys.if.bytes.out 1479496102 1.3E3 network=tcp", " sys.procs.running 1479496100 42 host=web01 "}; + for(int i = 0; i < 4; i++){ + strncpy(sql[i], sql1[i], 128); + } + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -147,28 +156,7 @@ int smlProcess_json3_Test() { taos_free_result(pRes); const char *sql[] = { - "{\"metric\":\"meter_current1\",\"timestamp\":{\"value\":1662344042,\"type\":\"s\"},\"value\":{\"value\":10.3,\"type\":\"i64\"},\"tags\":{\"t1\":{\"value\":2,\"type\":\"bigint\"},\"t2\":{\"value\":2,\"type\":\"int\"},\"t3\":{\"value\":2,\"type\":\"i16\"},\"t4\":{\"value\":2,\"type\":\"i8\"},\"t5\":{\"value\":2,\"type\":\"f32\"},\"t6\":{\"value\":2,\"type\":\"double\"},\"t7\":{\"value\":\"8323\",\"type\":\"binary\"},\"t8\":{\"value\":\"北京\",\"type\":\"nchar\"},\"t9\":{\"value\":true,\"type\":\"bool\"},\"id\":\"d1001\"}}"}; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, - TSDB_SML_TIMESTAMP_NANO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - taos_close(taos); - - return code; -} - -int smlProcess_json4_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "{\"metric\":\"meter_current2\",\"timestamp\":{\"value\":1662344042000,\"type\":\"ms\"},\"value\":\"ni\",\"tags\":{\"t1\":{\"value\":20,\"type\":\"i64\"},\"t2\":{\"value\":25,\"type\":\"i32\"},\"t3\":{\"value\":2,\"type\":\"smallint\"},\"t4\":{\"value\":2,\"type\":\"tinyint\"},\"t5\":{\"value\":2,\"type\":\"float\"},\"t6\":{\"value\":0.2,\"type\":\"f64\"},\"t7\":\"nsj\",\"t8\":{\"value\":\"北京\",\"type\":\"nchar\"},\"t9\":false,\"id\":\"d1001\"}}" + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":0,\"value\":\"18\",\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":\"lga\"}}]" }; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -677,52 +665,6 @@ int sml_oom_Test() { return code; } -int sml_16368_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "[{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639000, \"type\": \"us\"}, \"value\": 1, " - "\"tags\": {\"t1\": 3, \"t2\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t3\", \"type\": " - "\"binary\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833739000, \"type\": \"us\"}, \"value\": 2, " - "\"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " - "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}," - "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639100, \"type\": \"us\"}, \"value\": 3, " - "\"tags\": {\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t3\": {\"value\": \"ste\", \"type\": \"nchar\"}}}," - "{\"metric\": \"stf567890\", \"timestamp\": {\"value\": 1626006833639200, \"type\": \"us\"}, \"value\": 4, " - "\"tags\": {\"t1\": {\"value\": 4, \"type\": \"bigint\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " - "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639300, \"type\": \"us\"}, \"value\": " - "{\"value\": 5, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t2\": 5.0, " - "\"t3\": {\"value\": \"t4\", \"type\": \"binary\"}}}," - "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639400, \"type\": \"us\"}, \"value\": " - "{\"value\": 6, \"type\": \"double\"}, \"tags\": {\"t2\": 5.0, \"t3\": {\"value\": \"ste2\", \"type\": " - "\"nchar\"}}}," - "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006834639400, \"type\": \"us\"}, \"value\": " - "{\"value\": 7, \"type\": \"double\"}, \"tags\": {\"t2\": {\"value\": 5.0, \"type\": \"double\"}, \"t3\": " - "{\"value\": \"ste2\", \"type\": \"nchar\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833839006, \"type\": \"us\"}, \"value\": " - "{\"value\": 8, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": " - "{\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, " - "\"type\": \"double\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833939007, \"type\": \"us\"}, \"value\": " - "{\"value\": 9, \"type\": \"double\"}, \"tags\": {\"t1\": 4, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " - "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}]"}; - pRes = taos_schemaless_insert(taos, (char **)sql, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MICRO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - taos_close(taos); - - return code; -} - int sml_dup_time_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -762,214 +704,6 @@ int sml_dup_time_Test() { return code; } -int sml_16960_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "[" - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955000, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": 830525384, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955001, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -588348364, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955002, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -370310823, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955003, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -811250191, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955004, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -330340558, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}" - "]"}; - - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, - TSDB_SML_TIMESTAMP_MILLI_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - taos_close(taos); - - return code; -} - int sml_add_tag_col_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -1004,10 +738,10 @@ int sml_add_tag_col_Test() { int smlProcess_18784_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + TAOS_RES *pRes = taos_query(taos, "create database if not exists db_18784 schemaless 1"); taos_free_result(pRes); - pRes = taos_query(taos, "use sml_db"); + pRes = taos_query(taos, "use db_18784"); taos_free_result(pRes); const char *sql[] = { @@ -1018,7 +752,7 @@ int smlProcess_18784_Test() { printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes)); int code = taos_errno(pRes); ASSERT(!code); - ASSERT(taos_affected_rows(pRes) == 2); + ASSERT(taos_affected_rows(pRes) == 1); taos_free_result(pRes); pRes = taos_query(taos, "select * from disk"); @@ -1092,9 +826,10 @@ int sml_ts2164_Test() { taos_free_result(pRes); const char *sql[] = { +// "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", + "meters,location=la,groupid=ca current=11.8,voltage=221", "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", - "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", - "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27", +// "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27", }; pRes = taos_query(taos, "use line_test"); @@ -1119,6 +854,9 @@ int sml_ttl_Test() { const char *sql[] = { "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" 1626006833739000000", }; + const char *sql1[] = { + "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" 1626006833339000000", + }; pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); @@ -1128,6 +866,11 @@ int sml_ttl_Test() { printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes)); taos_free_result(pRes); + pRes = taos_schemaless_insert_ttl(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, 20); + + printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes)); + taos_free_result(pRes); + pRes = taos_query(taos, "select `ttl` from information_schema.ins_tables where table_name='t_be97833a0e1f523fcdaeb6291d6fdf27'"); printf("%s result2:%s\n", __FUNCTION__, taos_errstr(pRes)); TAOS_ROW row = taos_fetch_row(pRes); @@ -1141,7 +884,46 @@ int sml_ttl_Test() { return code; } +//char *str[] ={ +// "", +// "f64", +// "F64", +// "f32", +// "F32", +// "i", +// "I", +// "i64", +// "I64", +// "u", +// "U", +// "u64", +// "U64", +// "i32", +// "I32", +// "u32", +// "U32", +// "i16", +// "I16", +// "u16", +// "U16", +// "i8", +// "I8", +// "u8", +// "U8", +//}; +//uint8_t smlCalTypeSum(char* endptr, int32_t left){ +// uint8_t sum = 0; +// for(int i = 0; i < left; i++){ +// sum += endptr[i]; +// } +// return sum; +//} + int main(int argc, char *argv[]) { + +// for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){ +// printf("str:%s \t %d\n", str[i], smlCalTypeSum(str[i], strlen(str[i]))); +// } int ret = 0; ret = sml_ttl_Test(); ASSERT(!ret); @@ -1154,11 +936,9 @@ int main(int argc, char *argv[]) { ret = smlProcess_json1_Test(); ASSERT(!ret); ret = smlProcess_json2_Test(); - ASSERT(!ret); + ASSERT(ret); ret = smlProcess_json3_Test(); - ASSERT(!ret); - ret = smlProcess_json4_Test(); - ASSERT(!ret); + ASSERT(ret); ret = sml_TD15662_Test(); ASSERT(!ret); ret = sml_TD15742_Test(); @@ -1167,12 +947,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_oom_Test(); ASSERT(!ret); - ret = sml_16368_Test(); - ASSERT(!ret); ret = sml_dup_time_Test(); ASSERT(!ret); - ret = sml_16960_Test(); - ASSERT(!ret); ret = sml_add_tag_col_Test(); ASSERT(!ret); ret = smlProcess_18784_Test(); diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 024a253a2e2aad97b0639468b79ff04cb177b8a4..69bf52bb1ac1c80015e12a6ecf6701dc73f8d139 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -78,28 +78,27 @@ static void msg_process(TAOS_RES* msg) { int buildDatabase(TAOS* pConn, TAOS_RES* pRes){ /* test for TD-20612 start*/ -// pRes = taos_query(pConn,"create table tb1 (ts timestamp, c1 int, c2 int)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn,"insert into tb1 (ts, c1) values(1669092069069, 0)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn,"insert into tb1 (ts, c2) values(1669092069069, 1)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// return 0; + pRes = taos_query(pConn,"create table tb1 (ts timestamp, c1 int, c2 int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn,"insert into tb1 (ts, c1) values(1669092069069, 0)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn,"insert into tb1 (ts, c2) values(1669092069069, 1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + /* test for TD-20612 end*/ pRes = taos_query(pConn, @@ -614,6 +613,7 @@ void initLogFile() { } }else{ char *result[] = { + "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"tb1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":4}],\"tags\":[]}", "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":64},{\"name\":\"c4\",\"type\":5}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1},{\"name\":\"t2\",\"type\":8,\"length\":64}]}", "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}", @@ -652,6 +652,7 @@ void initLogFile() { } }else{ char *result[] = { + "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"tb1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":4}],\"tags\":[]}", "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}", "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}",